Parallel and Multi-thread

GNU Parallel

Official: https://www.gnu.org/software/parallel/ 

Install

# CentOS 7
yum group install "Development Tools"
wget http://ftp.gnu.org/gnu/parallel/parallel-latest.tar.bz2
tar xjf parallel-latest.tar.bz2
cd parallel-*
./configure /opt/parallel

Usage

 

With Bash

# Multiple files  *.lst with lots of file paths such as
# cat 1.lst
# /path/to/file1 
# /path/to/file2
# /path/to/file3
#

files="$@"
## An arbitrary limiting factor so that there are some free processes
## in case I want to run something else
num_processes=3
#
echo "Parallel Processing with $num_processes threads (`date "+%F %T"`)"
for lst in $files
do
    for f in $(cat $lst | sed '/^#/d')
    do
        ((i=i%num_processes)); ((i++==0)) && wait
        yourshell.sh $f &
    done
    sleep 120
done