Skip to main content

Parallel and Multi-thread

GNU Parallel

https://www.gnu.org/software/parallel/ 

With Bash
# Multiple files  *.lst with lots of file paths such as
# cat 1.lst
# /path/to/file1 
# /path/to/file2
# /path/to/file3
#

files="$@"
## An arbitrary limiting factor so that there are some free processes
## in case I want to run something else
num_processes=3
#
echo "Parallel Processing with $num_processes threads (`date "+%F %T"`)"
for lst in $files
do
    for f in $(cat $lst | sed '/^#/d')
    do
        ((i=i%num_processes)); ((i++==0)) && wait
        yourshell.sh $f &
    done
    sleep 120
done