#!/bin/sh
# http://www.thiscoolsite.com/?p=6

# Modified by A.Lang 2011-10-09 

# Version 2.0 - 2011-10-09
# - Added -b option to sync the datas without doing virtual IP change. 
# Updated by 2011-10-15
# - modified some words
# - added SYNCMODE for syncing the whole system
#
# This script is setup for an Asterisk/FreePBX system.  If used on another type of
# Asterisk box, the function: Asterisk may need to be modified to start/stop asterisk
# Also, if amportal is not in the regular path, the script will have to be modified.
#

################################
# These are the only three lines which MUST be changed in this script
################################
# The Master and Slave IP Addresses for Replication / Testing
MASTERIP="192.168.7.62"
SLAVEIP="192.168.7.70"

# The IP address that will float between Master and Slave
FLOAT="192.168.7.71"


#
# These are used to make sure that only one copy of this script
# runs at a time.  LOCKDIR is also used to store the old crontab file
# and used to determine if the crontab needs to be sync'd
#
LOCKDIR=/var/run/asterisk
LCK_FILE=$LOCKDIR/`basename $0`.lck
CACHEDIR=/var/cache/asterisk

#
# Directories to sync
#
# These are in addition to the standard Asterisk directories
# Elastix uses /tftpboot, especially when using Aastra scripts
# FTP is used for phones outside the network, to avoid having a tftp server on the internet
#
# If you have MySQL databases on the servers, they will need to be set up in a master-master
# replication mode.  
#
#
# Define the SYNCMODE:
# 1: SYNCDIRS + ASTDIRS + SYNCFILES would be synced only
# 2: full system syncing except EXCLUDEDIRS
SYNCMODE=1

RSYNC_EXCLUDE="--exclude=/proc --exclude=/tmp --exclude=/mnt --exclude=/sys --exclude=/lost+found --exclude=/dev --exclude=/etc/sysconfig/network --exclude=/etc/sysconfig/network-scripts --exclude=/root/.ssh"

#
# Some people would also want to sync /usr/src/ here
# Make sure that all directories end with a trailing slash
#SYNCDIRS="/var/www/ /tftpboot/ /var/ftp/ /etc/vsftpd/ /etc/fail2ban/ /etc/postfix/ /home/"
SYNCDIRS="/var/www/ /home/"

# Perl directories
PERLCONFIG="/usr/lib/perl5/ /root/.cpan/"
SYNCDIRS="$SYNCDIRS $PERLCONFIG"

# Apache config, Redhat based dists use /etc/httpd, Debian/Ubuntu use /etc/apache or /etc/apache2
# Comment these two lines out if you don't want the apache configs sync'd
#APACHECONFIG="/etc/httpd/ /etc/pki/"
#APACHECTL="service httpd "

#SYNCDIRS="$SYNCDIRS $APACHECONFIG"

#
# Individual files to synchronize
#
#SYNCFILES="/etc/passwd /etc/shadow /etc/group /etc/amportal.conf /usr/local/sbin/syncftp.sh /etc/php.ini"
SYNCFILES="/etc/passwd /etc/shadow /etc/group"

#
# Asterisk directories to sync
#
#ASTDIRS="/etc/asterisk/ /var/spool/asterisk/voicemail/ /var/lib/asterisk/moh/ /var/lib/asterisk/sounds/ /var/cache/aastra/"
ASTDIRS="/etc/asterisk/ /var/spool/asterisk/voicemail/ /var/spool/asterisk/monitor/ /var/lib/asterisk/"

#
# SYNCCRON has the following values:
#	0	Do not sync any crontabs
#	1	sync root crontab only
#	2	sync /etc/crontab only
#	3	sync both the root crontab and the /etc/crontab files
SYNCCRON=3

#
# Services to start/stop upon switchover
#
#SERVICES="amportal iptables fail2ban vsftpd"
#SERVICES="amportal vsftpd"
SERVICES="amportal"


# The device on which the floating interface exists
# If set to "1" this is the Master server.
# If commented out, or set to anything else, this will act as if it is a slave
# Now set dynamically below
#MASTER="1"

# Now set dynamically below
# DEVICE="eth0"

###########################################################
# Determine at run time if this is the master or slave
# by looking at all the configured network devices
#
# Obtain the list of network devices using the ip command
#
###########################################################
devs=`ip -o link | cut -f 2 --delimiter=' ' -`

MASTER="-1"
DEVICE=""
for i in $devs; do
   d=$(echo ${i%:})
   x=$(ifconfig $d) && x=${x#*inet addr:} && x=${x%% *}
   if [ "$x" = "$MASTERIP" ]; then
      MASTER="1"
      DEVICE=$d
      break;
   fi
   if [ "$x" = "$SLAVEIP" ]; then
      MASTER="0"
      DEVICE=$d
      break;
   fi
done
#
# If no device found with either the master or slave IP, then exit
#
[ "$MASTER" = "-1" ] && exit 0

# The specific interface alias on the device
IFACE="$DEVICE:1"

#LOGFILE=/var/log/flip1405.log
LOGFILE=/var/log/`basename $0`.log

#
# Addition by Jonathan Bayer
#
# Set VERBOSE=1 to see the rsync commands
VERBOSE=0

# The following allows any of the above variables to be overwritten at
# run time by reading the values from a config file.
# This is useful so you don't need to edit the file on each system.
[ -f /etc/sysconfig/ast-ha-rsync ] && . /etc/sysconfig/ast-ha-rsync



#
# usage
#
function usage {
	cat <<_EOF_


Usage: $0 <options>

	Options:
	-v	Verbose (write to log file)
	-s	copy current file to the other system in the same location
	-r	force current system to release the virtualIP
	-b	sync the datas only but without doing virtualIP change(NOTE:this is used on Master node only)

_EOF_
	exit 0
}


############# Get the options, if any ########################
COPYTO=0
# VERBOSE=0
RELEASE=0
BACKUP=0
while getopts ":svrbg" OPT
do
  case $OPT in
      s)
         COPYTO=1
         ;;
      # Allow a -v if called by hand for debugging
      v)
         VERBOSE=1
         ;;
      r)
         RELEASE=1
         ;;
      b)
 	 BACKUP=1
  	 ;;
      ?)
         usage
         ;;
  esac
done



# copies current batch file to the other system
if [ $COPYTO -eq 1 ]; then
   MY_PATH=$(readlink -f "$0")
   [ "$MASTER" = "1" ] && scp $MY_PATH root@$SLAVEIP:$MY_PATH
   [ "$MASTER" = "0" ] && scp $MY_PATH root@$MASTERIP:$MY_PATH
   exit
fi

if [ $VERBOSE -eq 1 ]; then
   echo -- `date +"%Y-%m-%d %H:%M:%S"` START -- >>$LOGFILE
   echo "Master: $MASTER" >>$LOGFILE
   echo "Device: $DEVICE" >>$LOGFILE
fi

#
# Rsync command
#
#RSYNC="rsync -avzr --rsh=ssh"
RSYNC="/usr/bin/rsync -azr --delete"

############# Functions below ####################

#
# logit
#
function logit {
    [ $VERBOSE -eq 1 ] && echo -e $1 >>$LOGFILE
}

#
# syncCrontabTo
#
function syncCrontabTo {
    [ $SYNCCRON -eq 0 ] && return

    [ ! -d $CACHEDIR ] && mkdir -p $CACHEDIR
    oldCron=$CACHEDIR/root.crontab

    if [ $SYNCCRON -eq 1 -o $SYNCCRON -eq 3 ]; then
       tmpfile=/tmp/$$.cron
       crontab -u root -l >$tmpfile
       rc=1
       if [ -f $oldCron ]; then
          cmp -s $tmpfile $oldCron
          rc=$?
       fi
       if [ $rc -ne 0 ]; then
          $RSYNC $tmpfile root@$1:/tmp
          ssh root@$1 crontab -uroot  $tmpfile
          cp $tmpfile $oldCron
       fi
       rm $tmpfile
    fi
    oldCron=$CACHEDIR/etc.crontab

    if [ $SYNCCRON -eq 2 -o $SYNCCRON -eq 3 ]; then
       rc=1
       if [ -f $oldCron ]; then
          cmp /etc/crontab $oldCron
          rc=$?
       fi
       if [ $rc -ne 0 ]; then
          # No need to restart crond after this, crond automatically
          # checks the file to see if it's changed.
          $RSYNC /etc/crontab  root@$1:/etc/crontab
          cp /etc/crontab $oldCron
       fi
    fi
}


#
# Check if running
#

function checkIfRunning {
    PDIR=${0%`basename $0`}

    if [ "$CHECK_LOCK" != "1" ]; then
       # ------------------------------------------------------------
       # Am I Running
       # ------------------------------------------------------------
       if [ -f "${LCK_FILE}" ]; then
	    
          # The file exists so read the PID
          # to see if it is still running
          MYPID=`head -n 1 "${LCK_FILE}"`
	    
          TEST_RUNNING=`ps -p ${MYPID} | grep ${MYPID}`
	    
          if [ -z "${TEST_RUNNING}" ]; then
             # The process is not running
             # Echo current PID into lock file
             echo $$ > "${LCK_FILE}"
          else
             logit "`basename $0` is already running [${MYPID}]"
             exit 0
          fi
	    
       else
	  echo $$ > "${LCK_FILE}"
       fi
    fi
}

#
# services - start/stop services, special handling for asterisk and amportal
#
function services {
    echo "services $1"
    for i in $SERVICES; do
       if [ "$i" = "amportal" -a "$1" = "restart" ]; then
          /usr/sbin/amportal $1
       else
          [ "$i" != "amportal" ] && service $i $1
       fi
    done
    #[ "$APACHECTL" != "" ] && $APACHECTL $1
}

#
# Asterisk start/stop
#
function Asterisk {
    f=`which amportal`
    a=`which asterisk`
    if [ "$f" != "" ]; then
       amportal $1
    else
       if [ "$a" != "" ]; then
          asterisk $1
       else
          logit "Unable to $1 Asterisk"
       fi
    fi
}


#
# syncto - Sync specified directories and files to the destination 
#
function syncto {
    DEST=$1
    # Check to see if slave is up before starting rsync
    ping -c 1 $DEST 2>&1 >/dev/null
    rc=$?
    if [ $rc -eq 0 ]; then
       if [ $SYNCMODE -eq 1 ]; then
          logit "Syncing files: SYNCMODE #1"    
          # Configuration File Replication from Master to Slave 
          # Basic Asterisk files 
          for i in $ASTDIRS; do
			    # Make sure directory ends in a slash
			    [[ $i != */ ]] && i="$i"/
			    logit "$RSYNC $i root@$DEST:$i"
			    $RSYNC $i root@$DEST:$i
		    done 

          for i in $SYNCDIRS; do 
             # Make sure directory ends in a slash 
             [[ $i != */ ]] && i="$i"/    
             logit "$RSYNC $i root@$DEST:$i"  
             $RSYNC $i root@$DEST:$i 
          done 

          for i in $SYNCFILES; do 
             logit "$RSYNC $i root@$DEST:$i" 
             $RSYNC $i root@$DEST:$i 
          done  

          syncCrontabTo $DEST 
       elif [ $SYNCMODE -eq 2 ]; then
          # Syncing the whole system
          logit "Syncing files: SYNCMODE #2 - Full System"
          logit "$RSYNC $RSYNC_EXCLUDE / root@$DEST:/"
          $RSYNC $RSYNC_EXCLUDE / root@$DEST:/ 
       fi
    fi
}
                          
# syncfrom - Sync specified directories and files from the destination 
#
function syncfrom {
    SRC=$1
    DEST=$2
    # Check to see if slave is up before starting rsync
    ping -c 1 $DEST 2>&1 >/dev/null
    rc=$?
    if [ $rc -eq 0 ]; then
       # Configuration File Replication from Master to Slave
       # Basic Asterisk files
       for i in $ASTDIRS; do
          logit "$RSYNC root@$DEST:$i $i"
          $RSYNC root@$DEST:$i $i
       done

       for i in $SYNCDIRS; do
          logit "$RSYNC root@$DEST:$i $i"
          $RSYNC root@$DEST:$i $i
       done

       for i in $SYNCFILES; do
          logit "$RSYNC root@$DEST:$i $i"
          $RSYNC root@$DEST:$i $i
       done
		
    fi
}

#
# checkSysUptime used by slave to delay any action for 5 minutes, thereby giving the master a chance to 
# boot and become active.  This is needed in case of a complete power outage which brought
# down both servers
#
function checkSysUptime {
    upSeconds=`cat /proc/uptime  | cut -f1 -d"."`
    mins=$((upSeconds/60))
    logit "Uptime in minutes: $mins"
    if [ $mins -lt 5 ]; then
       rm $LCK_FILE
       exit 0
    fi
}

checkIfRunning

#
# checkStatus - Check status of Asterisk on the specified addresses, flip if necessary
#
function checkStatus {
    local MASTERIP=$1
    local SLAVEIP=$2
    local FLOAT=$3
    local DEVICE=$4

    #if Local Asterisk up = 'open|filtered'
    STATUS=$(nmap --system-dns -p 4569 -sU 127.0.0.1 | awk '{print $2}' | grep open)
    #if primary owns virtual 
    PRIMARYIP=$(ifconfig "$IFACE" | grep "$FLOAT" | awk '{print $2}' | sed 's/addr://g')
    #if virtual is not pingable = 'down.'
    #VIRTUALIP=$(nmap --system-dns -sP "$FLOAT" | grep down | awk '{print $4}')
    ping -c 1 $FLOAT >/dev/null
    VIRTUALIP=$?

    logit "Status: $STATUS"
    logit "FLOAT: $FLOAT"
    logit "VirtualIP: $PRIMARYIP"
    logit "VirtualIP Up/Down[0/1]: $VIRTUALIP"

    #syncto $SLAVEIP
    if [ "$STATUS" == "open|filtered" ] ; then  ###is primary asterisk up?
       logit "open|filtered"
       if [ "$PRIMARYIP" != "$FLOAT" ] ; then  ###does primary not own virtual ip?
          logit "Current host does not own virtual IP" 
          #if [ "$VIRTUALIP" == "down." ] ; then  ###is the virtual IP not pingable?
          if [ $VIRTUALIP -eq 1 -a $RELEASE -eq 0 ] ; then  ###is the virtual IP not pingable?
             logit "Virtual IP not pingable" 
             logit "Startup the Virtual IP:$FLOAT on $IFACE"
             ifconfig $IFACE $FLOAT/24 up
             arping -U -c 5 -I $DEVICE $FLOAT  ###Gratuitous ARP request
             # Restart necessary services.  Asterisk will be reloaded
             services restart

             # Need to sync from slave here
             #syncfrom $SLAVEIP $MASTERIP
          fi
       else
          if [ $RELEASE -eq 1 ]; then
             logit "Releasing virtual IP" 
             # Stop necessary services
             services stop
             ifconfig $IFACE down
             # Force a long sleep here to allow the other system to grab the virtualIP
             Asterisk stop
             echo "Run this script on the other system to force an immediate failover."
             echo -e "\nWaiting for 60 seconds\n"
             for i in `seq 1 60`; do
                sleep 1
                echo -n "."
             done
             Asterisk start
          else
             # Only sync if we own the virtual IP
             syncto $SLAVEIP
          fi
       fi
    else
       logit "Asterisk is Not up" 
       # Stop necessary services
       services stop
       ifconfig $IFACE down
    fi
}

# Main Code
if [ "$MASTER" == "1" ] ; then
   if [ $BACKUP -eq 1 ]; then
      syncto $SLAVEIP
   else 
      logit "\nStarted as Master" >>$LOGFILE
      checkStatus $MASTERIP $SLAVEIP $FLOAT $DEVICE
   fi
else # We must be running as Slave node
   logit "\nStarted as Slave" 
   checkSysUptime
   checkStatus $SLAVEIP $MASTERIP $FLOAT $DEVICE
fi
rm $LCK_FILE