Skip to main content

範例與常用技巧

檔頭常用宣告
    Use Bash Strict Mode 
    # quickly syntax
set -euo pipefail

# let script exit if a command fails
set -o errexit 
# OR
set -e

# let script exit if an unsed variable is used
set -o nounset
# OR
set -u

# This setting prevents errors in a pipeline from being masked.
set -o pipefail

# for Debug
set -x

# setting IFS
IFS=$'\n\t'

    How set -o pipefail works

    $ grep some-string /non/existent/file | sort
grep: /non/existent/file: No such file or directory
$ echo $?
0

$ set -o pipefail
$ grep some-string /non/existent/file | sort
grep: /non/existent/file: No such file or directory
$ echo $?
2

    How the IFS  works

    #!/bin/bash
names=(
  "Aaron Maxwell"
  "Wayne Gretzky"
  "David Beckham"
  "Anderson da Silva"
)

echo "With default IFS value..."
for name in ${names[@]}; do
  echo "$name"
done

echo ""
echo "With strict-mode IFS value..."
IFS=$'\n\t'
for name in ${names[@]}; do
  echo "$name"
done

############### Output #############
With default IFS value...
Aaron
Maxwell
Wayne
Gretzky
David
Beckham
Anderson
da
Silva

With strict-mode IFS value...
Aaron Maxwell
Wayne Gretzky
David Beckham
Anderson da Silva
    檔案的目錄位置
    # 指定的檔案
readlink -f <file.name>

# 目前檔案
WORKDIR=$(readlink -f "$0") ;目前檔案的絕對路經
WORKDIR=$( cd $( dirname "$0" ) && pwd )
    Script 檔案名稱
    $ echo $0
./test.sh

$ echo `basename $0`
test.sh
    刪除 * 天前的舊備份檔
    cd $BACKUP_DIR
for backup in `find . -ctime +$BACKUP_KEEP_DAYS -name "cacti.*.tar.gz"`; do rm -f $backup; done;
    輸出多行的文字訊息
    cat <<EOF
Welcome .....
	
Here are the messages that you want to show up
		
EOF
    將執行後的所有輸出訊息導入一個檔案

    Sample #1

    #!/bin/sh
LOG="my.log"
(
....
) 2>&1 | tee -a $LOG

    不適用在內容裡有 python 指令的訊息輸入，輸入等待的畫面會無法顯示。

    Sample #2

    temp=$(mktemp)
exec &> ${temp}

echo "All outputs will be saved into the file ${temp}."

    Sample #3

    #!/bin/bash
set -eu 
exec 3>&1 4>&2
trap 'exec 2>&4 1>&3' 0 1 2 3
exec 1>/path/to/script.log 2>&1

# rest of the script below
dnf -y in foo bar
# firewall rules goes here 
....
...
    快速修改大量檔案的副檔名
    ## *.old -> *.new
for fname in $(ls *.old);do echo "mv $fname ->"; echo $(echo $fname |sed 's/.old/.new/');mv $fname $(echo $fname | sed 's/.old/.new/');done
    快速新增或列出多個指定目錄
    mkdir {AAA,BBB,CCC}
    快速備份設定檔
    cp my.cfg{,.bak}
    計算指定目錄底下的檔案總數量

    用 find

    # 計算目前目錄底下所有檔案數
find . -type f | wc -l 

# 計算第一層所有子目錄的檔案數並排序清單
find . -maxdepth 1 -type d -print0 | xargs -0 -I {} sh -c 'echo $(find {} -type f | wc -l) {}' | sort -n
NOTE: 第一層子目錄名稱不可包含空格

    用 rsync

    rsync --stats --dry-run -ax /usr /tmp

Number of files: 326,373 (reg: 211,698, dir: 24,284, link: 90,391)
Number of created files: 326,373 (reg: 211,698, dir: 24,284, link: 90,391)
Number of deleted files: 0
Number of regular files transferred: 211,698
Total file size: 7,180,685,730 bytes
Total transferred file size: 7,178,524,818 bytes

NOTE: 這指令實際不會作檔案複製，/tmp 只是一個假目錄，回傳結果是 reg: 211698 就是檔案總數

    用 tree

    tree /mydir -a | tail -n 1

5 directories, 56 files

NOTE: 當目錄底下有包含 symbolic link 也會被計算
    在 SHELL 內執行另一個外部 SHELL 或指令

    1. 使用 pipe line

    echo"md5sum $X > $X.sum "| bash

    2. 使用 eval

    get_arch="uname -p"
if [ "`eval "$get_arch"`" = "i686" ]; then
....
fi
    命令模式提示字元的路徑名稱太長

    加上這變數

    PROMPT_DIRTRIM=2
    處理 CSV 檔
    #!/bin/bash
INPUT=data.cvs
OLDIFS=$IFS
IFS=','
[ ! -f $INPUT ] && { echo "$INPUT file not found"; exit 99; }
while read flname dob ssn tel status
do
	echo "Name : $flname"
	echo "DOB : $dob"
	echo "SSN : $ssn"
	echo "Telephone : $tel"
	echo "Status : $status"
done < $INPUT
IFS=$OLDIFS

    CSV and JSON

    # JSON to CSV
cat df.json | jq -r '.[]| join(",")'
cat bingbot.json | jq -r '.prefixes[] | {cidr: .ipv4Prefix, comment: "BingBot"} | join(",")' > bingbot.csv
    JSON 檔
    建立暫存檔
    tmpfile1=$(mktemp)
tmpfile2="/tmp/$(basename $0).$$.tmp"
    Get my public IP
    curl ifconfig.me
curl ifconfig.me/ip
curl ifconfig.co
curl checkip.amazonaws.com
curl icanhazip.com
curl ipecho.net/plain

dig +short myip.opendns.com @resolver1.opendns.com
dig TXT +short o-o.myaddr.l.google.com @ns1.google.com
dig TXT +short o-o.myaddr.l.google.com @ns1.google.com | awk -F'"' '{print $2}'
    主機 IP
    # On Linux
hostname -I
hostip=$(/sbin/ip a | awk '/eth[012]:|ens192:|bond0:/,/^$/' | grep -E "inet [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}" | head -1 | awk -F " " '{print $2}' | cut -d"/" -f1)

# On AIX
hostip=$( ifconfig -a | grep inet | awk '{print $2}' | head -1 )
hostip=$( ifconfig -a | grep inet | awk '{print $2}' | sed -n '/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/p' | head -1 )
    找出最大的目錄或檔案
    # Way 1
du -a /var | sort -n -r | head -n 10

# Way 2
cd /path/to/some/where
du -hsx * | sort -rh | head -10
du -hsx -- * | sort -rh | head -10

# Way 3
find /path/to/dir/ -printf '%s %p\n'| sort -nr | head -10
find . -printf '%s %p\n'| sort -nr | head -10
## Skip directories and only display files
find /path/to/search/ -type f -printf '%s %p\n'| sort -nr | head -10

# Create a shell alias
## shell alias ##  
alias ducks='du -cks * | sort -rn | head'
## deal with special files names ##
alias ducks='du -cks -- * | sort -rn | head'
    shuf: 隨機排序
    curl -s https://www.imdb.com/list/ls020046354/export | cut -d ',' -f 6  | shuf
    tree: 顯示專案目錄的檔案樹
    tree --dirsfirst --filelimit 10 --sort=name

# Display size of files
tree -s

# Display permissions of files
tree -p

# Display directory only
tree -d

# Display till a certain level/depth
tree -L 1

# List only those files that match pattern given
tree -P *screenshot*
    sort : 資料排序
    sort -t ',' -k5,5 -k1,1 -k9,9 -k3,3 -k11,11 my.csv
    •  -t 分隔符號
    • -k5,5 排序第 5 欄，以字串類型排列
    • 欄位排序先後依序為第 5, 1, 9, 3, 11 欄
    • 欄位排序若要以數值方式來排，改成 -k5,5n
    timeout : 自動停止執行
    timeout 10 tail -f /var/log/httpd/access.log
timeout 5m ping 8.8.8.8
timeout 300 tcpdump -n -w data.pcap

# Sending specific signal
# To get a list of all available signals, use the command kill -l .
timeout -s SIGKILL ping 8.8.8.8
    variables : 變數
    變數 說明
     $0  腳本檔名
     $1  第1個參數
     $2  第2個參數
     ${10}  第10個參數 #10
     $#  參數的個數
     $*  顯示所有參數 (作為一個字串)
     $@  顯示所有參數 (每個為一個獨立字串)
     ${$*}  傳遞到腳本中的參數的個數
     ${$@}  傳遞到腳本中的參數的個數
     $?  腳本結束後的傳回值
     $$  腳本的程序 ID
     $_  前個命令的最後一個參數
     $!  最後一個執行程序的 ID
    u=${1:-root}  如果 $1 未指定，就賦予值 root
    u=${USER:-foo}  如果 $USER 未指定，就賦予值 foo
    len=${#var}  計算 $var 字串的長度

    如果 $2 未指定或空值，輸出錯誤訊息

    ${varName?Error varName is not defined}
${varName:?Error varName is not defined or is empty}
    cut: 切割文字
    # AAA = BBB, 取出 BBB
cut -d= -f2

# 111 2222 33 444444 555, 取出 33 以後的所有內容
cut -d ' ' -f3-

# 檢視超長的文字
head -n 1 data.csv | wc  # 計算行的字元數
head data.csv | cut -c -30  # 列出前幾行的前 30 個字元, 如果要顯示後 30 個字元，可以用 30-
    tr: 置換
    # aaa bbb ccc
# 換成
# aaa
# bbb
# ccc
echo "aaa bbb ccc" | tr " " "\n"
    完整複製 Home 目錄

    由於 User 的 Home 目錄內有許多隱藏檔，若要完整複製它們，有兩個方法：

    方法一：可以複製成一個新目錄

    cd /home
cp -a user1/ user1_new/

    方法二：複製到一個現有目錄內

    cd /home
cp -a user1/.[^.]* user1_new/

    建立一個新的 Home 目錄

    cp -r /etc/skel /home/user1
chown -R user1.group1 /home/user1
chmod 0700 /home/user1
    xargs

    接入(Pipe) 不支援 stdin 指令的替代方法

    printf: 格式化輸出
    • %s 字串
    • %d 整數
    printf "%-40s ..................%s\n" "Disable the service $1" "[$2]"

Disable the service apmd                 ..................[OK]
Disable the service bluetooth            ..................[OK]
Disable the service hidd                 ..................[OK]
Disable the service cups                 ..................[OK]
Disable the service firstboot            ..................[OK]
Disable the service readahead_early      ..................[OK]

printf "%40s ..................%s\n" "Disable the service $1" "[$2]"

                Disable the service apmd ..................[OK]
           Disable the service bluetooth ..................[OK]
                Disable the service hidd ..................[OK]
                Disable the service cups ..................[OK]
           Disable the service firstboot ..................[OK]
     Disable the service readahead_early ..................[OK]
    印出一個長符號
    printf -- '-%.0s' {1..80}
printf -- '=%.0s' {1..80}
    ping: 掃描一個 IP 範圍
    { for p in {1..254}; do ping -c1 -w1 10.22.9.$p & done } | grep "64 bytes"
    nice: Reduce CPU and Disk load of backup scripts
    # Reduce the I/O priority of the /usr/local/bin/backup.sh script so that it does not interfere with other processes
# The -n parameter must be between 0 and 7, where lower numbers mean higher priority
/usr/bin/ionice -c2 -n7 /usr/local/bin/backup.sh

# To reduce the CPU priority, use the command nice
# The -n parameter can range from -20 to 19, where lower numbers mean higher priority
/usr/bin/nice -n 19 /usr/local/bin/backup.sh

# Nice and ionice can also be combined, to run a script at low I/O and CPU priority
/usr/bin/nice -n 19 /usr/bin/ionice -c2 -n7 /usr/local/bin/backup.sh
    Hex to ASCII
    # hex = 54657374696e672031203220330
# ascii = Testing 1 2 3

# xxd
echo 54657374696e672031203220330 | xxd -r -p && echo ''

# printf
printf '\x54\x65\x73\x74\x69\x6e\x67\x20\x31\x20\x32\x20\x33\x0' && echo ''

# sed
echo -n 54657374696e67203120322033 | sed 's/\([0-9A-F]\{2\}\)/\\\\\\x\1/gI' | xargs printf && echo ''
    隨機密碼
    genpasswd() { 
	local l=$1
       	[ "$l" == "" ] && l=16
      	tr -dc A-Za-z0-9_ < /dev/urandom | head -c ${l} | xargs 
}
    tr -dc A-Za-z0-9_ < /dev/urandom | head -c 16 | xargs

# Generate more than one
tr -dc A-Za-z0-9_ < /dev/urandom | fold -16 | head -5
    密碼加密

    Install mkpasswd

    sudo apt install whois      # [On Debian, Ubuntu and Mint]
sudo yum install expect     # [On RHEL/CentOS/Fedora and Rocky/AlmaLinux]
sudo emerge -a whois        # [On Gentoo Linux]
sudo apk add mkpasswd       # [On Alpine Linux]
sudo pacman -S whois        # [On Arch Linux]
sudo zypper install whois   # [On OpenSUSE]

    Usage

    # Generate MD5 Password Hash With a Salt
mkpasswd -m md5 -S UMsalt12 UbuntuMintPassword

# Generate SHA-256 Password Hash Without Salt
mkpasswd -m sha-256 UbuntuMintPassword

# Generate SHA-256 Password Hash With Salt and Number of Rounds
mkpasswd -m sha-512 -S UMsalt12 -R 10000 UbuntuMintPassword

# Generate SHA-256 Hash from Password File
echo "UbuntuMintPassword" | mkpasswd -m sha-256 -s

     

    ls: 進階技巧
    # Find the biggest zip file
ls -lSrh ~/Downloads/*.zip
    stat: 檔案屬性
    ❯ stat --printf='Name: %n\nPermissions: %a\n' my.log
Name: my.log
Permissions: 777

❯ stat --format="%F" my.log
regular file

# Symlink
❯ stat ~/bin/FoxitReader
  File: /home/alang/bin/FoxitReader -> /home/alang/opt/foxitsoftware/foxitreader/FoxitReader.sh
  Size: 56        	Blocks: 0          IO Block: 4096   symbolic link
Device: 10302h/66306d	Inode: 787474      Links: 1
Access: (0777/lrwxrwxrwx)  Uid: ( 1000/   alang)   Gid: ( 1000/   alang)
Access: 2023-07-16 10:26:13.412193581 +0800
Modify: 2023-02-26 12:13:50.234374171 +0800
Change: 2023-02-26 12:13:50.234374171 +0800
 Birth: 2023-02-26 12:13:50.234374171 +0800

❯ stat -L ~/bin/FoxitReader
  File: /home/alang/bin/FoxitReader
  Size: 120       	Blocks: 8          IO Block: 4096   regular file
Device: 10302h/66306d	Inode: 1457222     Links: 1
Access: (0755/-rwxr-xr-x)  Uid: ( 1000/   alang)   Gid: ( 1000/   alang)
Access: 2023-07-02 14:34:09.957428285 +0800
Modify: 2023-02-26 12:13:50.246374250 +0800
Change: 2023-02-26 12:13:50.246374250 +0800
 Birth: 2023-02-26 12:13:48.530362986 +0800

    Back to top