@ -3,8 +3,8 @@
PROGRAM = "Osync" # Rsync based two way sync engine with fault tolerance
PROGRAM = "Osync" # Rsync based two way sync engine with fault tolerance
AUTHOR = "(L) 2013-2015 by Orsiris \"Ozy\" de Jong"
AUTHOR = "(L) 2013-2015 by Orsiris \"Ozy\" de Jong"
CONTACT = "http://www.netpower.fr/osync - ozy@netpower.fr"
CONTACT = "http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION = 0.99RC4
PROGRAM_VERSION = 1.00pre
PROGRAM_BUILD = 3003201503
PROGRAM_BUILD = 0104201502
## type doesn't work on platforms other than linux (bash). If if doesn't work, always assume output is not a zero exitcode
## type doesn't work on platforms other than linux (bash). If if doesn't work, always assume output is not a zero exitcode
if ! type -p " $BASH " > /dev/null
if ! type -p " $BASH " > /dev/null
@ -17,7 +17,7 @@ fi
if [ ! " $DEBUG " = = "yes" ]
if [ ! " $DEBUG " = = "yes" ]
then
then
DEBUG = no
DEBUG = no
SLEEP_TIME = 0 .1
SLEEP_TIME = .1
else
else
SLEEP_TIME = 3
SLEEP_TIME = 3
fi
fi
@ -36,10 +36,7 @@ else
fi
fi
## Default directory where to store temporary run files
## Default directory where to store temporary run files
if [ -w /dev/shm ]
if [ -w /tmp ]
then
RUN_DIR = /dev/shm
elif [ -w /tmp ]
then
then
RUN_DIR = /tmp
RUN_DIR = /tmp
elif [ -w /var/tmp ]
elif [ -w /var/tmp ]
@ -60,6 +57,11 @@ export LC_ALL=C
ALERT_LOG_FILE = $RUN_DIR /osync_lastlog
ALERT_LOG_FILE = $RUN_DIR /osync_lastlog
function Dummy
{
sleep .1
}
function Log
function Log
{
{
if [ $sync_on_changes -eq 1 ]
if [ $sync_on_changes -eq 1 ]
@ -400,14 +402,14 @@ function WaitForTaskCompletion
then
then
Log "Current task still running."
Log "Current task still running."
fi
fi
if [ $EXEC_TIME -gt $2 ]
if [ $EXEC_TIME -gt " $2 " ]
then
then
if [ $soft_alert -eq 0 ] && [ $2 != 0 ]
if [ $soft_alert -eq 0 ] && [ " $2 " != 0 ]
then
then
LogError "Max soft execution time exceeded for task."
LogError "Max soft execution time exceeded for task."
soft_alert = 1
soft_alert = 1
fi
fi
if [ $EXEC_TIME -gt $3 ] && [ $3 != 0 ]
if [ $EXEC_TIME -gt " $3 " ] && [ " $3 " != 0 ]
then
then
LogError "Max hard execution time exceeded for task. Stopping task execution."
LogError "Max hard execution time exceeded for task. Stopping task execution."
kill -s SIGTERM $1
kill -s SIGTERM $1
@ -431,7 +433,7 @@ function WaitForTaskCompletion
return $?
return $?
}
}
# Waits for pid $1 to complete. Will log an alert if $2 seconds passed since script start unless $2 equals 0.
# Waits for pid $1 to complete. Will log an alert if $2 seconds passed since script start unless $2 equals 0.
# Will stop task and log alert if $3 seconds passed since script start unless $3 equals 0.
# Will stop task and log alert if $3 seconds passed since script start unless $3 equals 0.
function WaitForCompletion
function WaitForCompletion
{
{
@ -443,14 +445,14 @@ function WaitForCompletion
then
then
Log "Current task still running."
Log "Current task still running."
fi
fi
if [ $SECONDS -gt $2 ]
if [ $SECONDS -gt " $2 " ]
then
then
if [ $soft_alert -eq 0 ] && [ $2 != 0 ]
if [ $soft_alert -eq 0 ] && [ " $2 " != 0 ]
then
then
LogError "Max soft execution time exceeded for script."
LogError "Max soft execution time exceeded for script."
soft_alert = 1
soft_alert = 1
fi
fi
if [ $SECONDS -gt $3 ] && [ $3 != 0 ]
if [ $SECONDS -gt " $3 " ] && [ " $3 " != 0 ]
then
then
LogError "Max hard execution time exceeded for script. Stopping current task execution."
LogError "Max hard execution time exceeded for script. Stopping current task execution."
kill -s SIGTERM $1
kill -s SIGTERM $1
@ -1501,157 +1503,129 @@ function SoftDelete
{
{
if [ " $CONFLICT_BACKUP " != "no" ] && [ $CONFLICT_BACKUP_DAYS -ne 0 ]
if [ " $CONFLICT_BACKUP " != "no" ] && [ $CONFLICT_BACKUP_DAYS -ne 0 ]
then
then
if [ -d " $MASTER_SYNC_DIR $MASTER_BACKUP_DIR " ]
Log "Running conflict backup cleanup."
_SoftDelete $CONFLICT_BACKUP_DAYS " $MASTER_SYNC_DIR $MASTER_BACKUP_DIR " " $SLAVE_SYNC_DIR $SLAVE_BACKUP_DIR "
fi
if [ " $SOFT_DELETE " != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]
then
Log "Running soft deletion cleanup."
_SoftDelete $SOFT_DELETE_DAYS " $MASTER_SYNC_DIR $MASTER_DELETE_DIR " " $SLAVE_SYNC_DIR $SLAVE_DELETE_DIR "
fi
}
# Takes 3 arguments
# $1 = ctime (CONFLICT_BACKUP_DAYS or SOFT_DELETE_DAYS), $2 = MASTER_(BACKUP/DELETED)_DIR, $3 = SLAVE_(BACKUP/DELETED)_DIR
function _SoftDelete
{
if [ -d " $2 " ]
then
if [ $dryrun -eq 1 ]
then
then
if [ $dryrun -eq 1 ]
Log " Listing files older than $1 days on master replica. Won't remove anything. "
then
else
Log " Listing backups older than $CONFLICT_BACKUP_DAYS days on master replica. Won't remove anything. "
Log " Removing files older than $1 days on master replica. "
if [ $verbose -eq 1 ]
fi
then
if [ $verbose -eq 1 ]
$FIND_CMD " $MASTER_SYNC_DIR $MASTER_BACKUP_DIR / " -ctime +$CONFLICT_BACKUP_DAYS &
else
$FIND_CMD " $MASTER_SYNC_DIR $MASTER_BACKUP_DIR / " -ctime +$CONFLICT_BACKUP_DAYS > /dev/null &
fi
else
Log " Removing backups older than $CONFLICT_BACKUP_DAYS days on master replica. "
$FIND_CMD " $MASTER_SYNC_DIR $MASTER_BACKUP_DIR / " -ctime +$CONFLICT_BACKUP_DAYS -exec rm -rf '{}' \; &
fi
child_pid = $!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
retval = $?
if [ $retval -ne 0 ]
then
LogError "Error while executing conflict backup cleanup on master replica."
else
Log "Conflict backup cleanup complete on master replica."
fi
elif [ -d " $MASTER_SYNC_DIR $MASTER_BACKUP_DIR " ] && ! [ -w " $MASTER_SYNC_DIR $MASTER_BACKUP_DIR " ]
then
then
LogError " Warning: Master replica conflict backup dir [ $MASTER_SYNC_DIR $MASTER_BACKUP_DIR ] isn't writable. Cannot clean old files. "
# Cannot launch log function from xargs, ugly hack
$FIND_CMD " $2 / " -type f -ctime +$1 -print0 | xargs -0 -I { } echo "Will delete file {}" > $RUN_DIR /osync_soft_delete_master_$SCRIPT_PID
Log " Command output:\n $( cat $RUN_DIR /osync_soft_delete_master_$SCRIPT_PID ) "
$FIND_CMD " $2 / " -type d -empty -ctime +$1 -print0 | xargs -0 -I { } echo "Will delete directory {}" > $RUN_DIR /osync_soft_delete_master_$SCRIPT_PID
Log " Command output:\n $( cat $RUN_DIR /osync_soft_delete_master_$SCRIPT_PID ) "
fi
fi
if [ $dryrun -ne 1 ]
if [ " $REMOTE_SYNC " = = "yes" ]
then
then
CheckConnectivity3rdPartyHosts
$FIND_CMD " $2 / " -type f -ctime +$1 -print0 | xargs -0 -I { } rm -f "{}" && $FIND_CMD " $2 / " -type d -empty -ctime +$1 -print0 | xargs -0 -I { } rm -rf "{}" > $RUN_DIR /osync_soft_delete_master_$SCRIPT_PID &
CheckConnectivityRemoteHost
if [ $dryrun -eq 1 ]
then
Log " Listing backups older than $CONFLICT_BACKUP_DAYS days on slave replica. Won't remove anything. "
eval " $SSH_CMD \"if [ -w \\\" $SLAVE_SYNC_DIR $SLAVE_BACKUP_DIR \\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\" $SLAVE_SYNC_DIR $SLAVE_BACKUP_DIR /\\\" -ctime + $CONFLICT_BACKUP_DAYS ; fi\" " &
else
Log " Removing backups older than $CONFLICT_BACKUP_DAYS days on remote slave replica. "
eval " $SSH_CMD \"if [ -w \\\" $SLAVE_SYNC_DIR $SLAVE_BACKUP_DIR \\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\" $SLAVE_SYNC_DIR $SLAVE_BACKUP_DIR /\\\" -ctime + $CONFLICT_BACKUP_DAYS -exec rm -rf '{}' \;; fi\" " &
fi
child_pid = $!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
retval = $?
if [ $retval -ne 0 ]
then
LogError "Error while executing conflict backup cleanup on slave replica."
else
Log "Conflict backup cleanup complete on slave replica."
fi
else
else
if [ -w " $SLAVE_SYNC_DIR $SLAVE_BACKUP_DIR " ]
Dummy &
then
if [ $dryrun -eq 1 ]
then
Log " Listing backups older than $CONFLICT_BACKUP_DAYS days on slave replica. Won't remove anything. "
$FIND_CMD " $SLAVE_SYNC_DIR $SLAVE_BACKUP_DIR / " -ctime +$CONFLICT_BACKUP_DAYS &
else
Log " Removing backups older than $CONFLICT_BACKUP_DAYS days on slave replica. "
$FIND_CMD " $SLAVE_SYNC_DIR $SLAVE_BACKUP_DIR / " -ctime +$CONFLICT_BACKUP_DAYS -exec rm -rf '{}' \; &
fi
child_pid = $!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
retval = $?
if [ $retval -ne 0 ]
then
LogError "Error while executing conflict backup cleanup on slave replica."
else
Log "Conflict backup cleanup complete on slave replica."
fi
elif [ -d " $SLAVE_SYNC_DIR $SLAVE_BACKUP_DIR " ] && ! [ -w " $SLAVE_SYNC_DIR $SLAVE_BACKUP_DIR " ]
then
LogError " Warning: Slave replica conflict backup dir [ $SLAVE_SYNC_DIR $SLAVE_BACKUP_DIR ] isn't writable. Cannot clean old files. "
fi
fi
fi
child_pid = $!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
retval = $?
if [ $retval -ne 0 ]
then
LogError "Error while executing cleanup on master replica."
Log " Command output:\n $( cat $RUN_DIR /osync_soft_delete_master_$SCRIPT_PID ) "
else
Log "Cleanup complete on master replica."
fi
elif [ -d " $2 " ] && ! [ -w " $2 " ]
then
LogError " Warning: Master replica dir [ $2 ] isn't writable. Cannot clean old files. "
fi
fi
if [ " $SOFT_DELETE " != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]
if [ " $REMOTE_SYNC " = = "yes" ]
then
then
if [ -d " $MASTER_SYNC_DIR $MASTER_DELETE_DIR " ]
CheckConnectivity3rdPartyHosts
then
CheckConnectivityRemoteHost
if [ $dryrun -eq 1 ]
if [ $dryrun -eq 1 ]
then
Log " Listing soft deleted items older than $SOFT_DELETE_DAYS days on master replica. Won't remove anything. "
$FIND_CMD " $MASTER_SYNC_DIR $MASTER_DELETE_DIR / " -ctime +$SOFT_DELETE_DAYS &
else
Log " Removing soft deleted items older than $SOFT_DELETE_DAYS days on master replica. "
$FIND_CMD " $MASTER_SYNC_DIR $MASTER_DELETE_DIR / " -ctime +$SOFT_DELETE_DAYS -exec rm -rf '{}' \; &
fi
child_pid = $!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
retval = $?
if [ $retval -ne 0 ]
then
LogError "Error while executing soft delete cleanup on master replica."
else
Log "Soft delete cleanup complete on master replica."
fi
elif [ -d " $MASTER_SYNC_DIR $MASTER_DELETE_DIR " ] && ! [ -w " $MASTER_SYNC_DIR $MASTER_DELETE_DIR " ]
then
then
LogError " Warning: Master replica deletion backup dir [ $MASTER_SYNC_DIR $MASTER_DELETE_DIR ] isn't writable. Cannot clean old files. "
Log " Listing files older than $1 days on slave replica. Won't remove anything. "
else
Log " Removing files older than $1 days on slave replica. "
fi
if [ $verbose -eq 1 ]
then
# Cannot launch log function from xargs, ugly hack
eval " $SSH_CMD \"if [ -w \\\" $3 \\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\" $3 /\\\" -type f -ctime + $1 -print0 | xargs -0 -I {} echo Will delete file {} && $REMOTE_FIND_CMD \\\" $3 /\\\" -type d -empty -ctime $1 -print0 | xargs -0 -I {} echo Will delete directory {}; fi\" " > $RUN_DIR /osync_soft_delete_slave_$SCRIPT_PID
Log " Command output:\n $( cat $RUN_DIR /osync_soft_delete_slave_$SCRIPT_PID ) "
fi
fi
if [ $dryrun -ne 1 ]
if [ " $REMOTE_SYNC " = = "yes" ]
then
then
CheckConnectivity3rdPartyHosts
eval " $SSH_CMD \"if [ -w \\\" $3 \\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\" $3 /\\\" -type f -ctime + $1 -print0 | xargs -0 -I {} rm -f \\\"{}\\\" && $REMOTE_FIND_CMD \\\" $3 /\\\" -type d -empty -ctime $1 -print0 | xargs -0 -I {} rm -rf \\\"{}\\\"; fi\" " > $RUN_DIR /osync_soft_delete_slave_$SCRIPT_PID &
CheckConnectivityRemoteHost
else
if [ $dryrun -eq 1 ]
Dummy &
then
fi
Log " Listing soft deleted items older than $SOFT_DELETE_DAYS days on slave replica. Won't remove anything. "
child_pid = $!
eval " $SSH_CMD \"if [ -w \\\" $SLAVE_SYNC_DIR $SLAVE_DELETE_DIR \\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\" $SLAVE_SYNC_DIR $SLAVE_DELETE_DIR /\\\" -ctime + $SOFT_DELETE_DAYS ; fi\" " &
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
else
retval = $?
Log " Removing soft deleted items older than $SOFT_DELETE_DAYS days on remote slave replica. "
if [ $retval -ne 0 ]
eval " $SSH_CMD \"if [ -w \\\" $SLAVE_SYNC_DIR $SLAVE_DELETE_DIR \\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\" $SLAVE_SYNC_DIR $SLAVE_DELETE_DIR /\\\" -ctime + $SOFT_DELETE_DAYS -exec rm -rf '{}' \;; fi\" " &
then
fi
LogError "Error while executing cleanup on slave replica."
child_pid = $!
Log " Command output:\n $( cat $RUN_DIR /osync_soft_delete_slave_$SCRIPT_PID ) "
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
retval = $?
if [ $retval -ne 0 ]
then
LogError "Error while executing soft delete cleanup on slave replica."
else
else
Log "Soft delete cleanup complete on slave replica."
Log "Cleanup complete on slave replica."
fi
fi
else
if [ -w " $3 " ]
then
if [ $dryrun -eq 1 ]
then
Log " Listing files older than $1 days on slave replica. Won't remove anything. "
else
else
if [ -w " $SLAVE_SYNC_DIR $SLAVE_DELETE_DIR " ]
Log " Removing files older than $1 days on slave replica. "
then
fi
if [ $dryrun -eq 1 ]
if [ $verbose -eq 1 ]
then
then
Log " Listing soft deleted items older than $SOFT_DELETE_DAYS days on slave replica. Won't remove anything. "
# Cannot launch log function from xargs, ugly hack
$FIND_CMD " $SLAVE_SYNC_DIR $SLAVE_DELETE_DIR / " -ctime +$SOFT_DELETE_DAYS &
$FIND_CMD " $3 / " -type f -ctime +$1 -print0 | xargs -0 -I { } echo "Will delete file {}" > $RUN_DIR /osync_soft_delete_slave_$SCRIPT_PID
else
Log " Command output:\n $( cat $RUN_DIR /osync_soft_delete_slave_$SCRIPT_PID ) "
Log " Removing soft deleted items older than $SOFT_DELETE_DAYS days on slave replica. "
$FIND_CMD " $3 / " -type d -empty -ctime +$1 -print0 | xargs -0 -I { } echo "Will delete directory {}" > $RUN_DIR /osync_soft_delete_slave_$SCRIPT_PID
$FIND_CMD " $SLAVE_SYNC_DIR $SLAVE_DELETE_DIR / " -ctime +$SOFT_DELETE_DAYS -exec rm -rf '{}' \; &
Log " Command output:\n $( cat $RUN_DIR /osync_conflict_backup_slave_$SCRIPT_PID ) "
fi
Dummy &
child_pid = $!
fi
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
if [ $dryrun -ne 1 ]
retval = $?
then
if [ $retval -ne 0 ]
$FIND_CMD " $3 / " -type f -ctime +$1 -print0 | xargs -0 -I { } rm -f "{}" && $FIND_CMD " $3 / " -type d -empty -ctime +$1 -print0 | xargs -0 -I { } rm -rf "{}" > $RUN_DIR /osync_soft_delete_slave_$SCRIPT_PID &
then
fi
LogError "Error while executing soft delete cleanup on slave replica."
child_pid = $!
else
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
Log "Soft delete cleanup complete on slave replica."
retval = $?
fi
if [ $retval -ne 0 ]
elif [ -d " $SLAVE_SYNC_DIR $SLAVE_DELETE_DIR " ] && ! [ -w " $SLAVE_SYNC_DIR $SLAVE_DELETE_DIR " ]
then
then
LogError "Error while executing cleanup on slave replica."
LogError " Warning: Slave replica deletion backup dir [ $SLAVE_SYNC_DIR $SLAVE_DELETE_DIR ] isn't writable. Cannot clean old files. "
Log " Command output:\n $( cat $RUN_DIR /osync_soft_delete_slave_$SCRIPT_PID ) "
fi
else
Log "Cleanup complete on slave replica."
fi
elif [ -d " $3 " ] && ! [ -w " $3 " ]
then
LogError " Warning: Slave replica dir [ $3 ] isn't writable. Cannot clean old files. "
fi
fi
fi
fi
}
}
function Init
function Init
@ -1956,9 +1930,9 @@ function Usage
echo $CONTACT
echo $CONTACT
echo ""
echo ""
echo "You may use Osync with a full blown configuration file, or use its default options for quick command line sync."
echo "You may use Osync with a full blown configuration file, or use its default options for quick command line sync."
echo "Usage: osync /path/to/config/file [OPTIONS]"
echo "Usage: osync .sh /path/to/config/file [OPTIONS]"
echo "or osync --master=/path/to/master/replica --slave=/path/to/slave/replica [OPTIONS] [QUICKSYNC OPTIONS]"
echo "or osync .sh --master=/path/to/master/replica --slave=/path/to/slave/replica [OPTIONS] [QUICKSYNC OPTIONS]"
echo "or osync --master=/path/to/master/replica --slave=ssh://backupuser@remotehost.com[:portnumber]//path/to/slave/replica [OPTIONS] [QUICKSYNC OPTIONS]"
echo "or osync .sh --master=/path/to/master/replica --slave=ssh://[ backupuser] @remotehost.com[:portnumber]//path/to/slave/replica [OPTIONS] [QUICKSYNC OPTIONS]"
echo ""
echo ""
echo "[OPTIONS]"
echo "[OPTIONS]"
echo "--dry Will run osync without actually doing anything; just testing"
echo "--dry Will run osync without actually doing anything; just testing"
@ -1975,6 +1949,10 @@ function Usage
echo "--slave=\"\" Local or remote slave replica path. Can be a ssh uri like ssh://user@host.com:22//path/to/slave/replica (is mandatory)"
echo "--slave=\"\" Local or remote slave replica path. Can be a ssh uri like ssh://user@host.com:22//path/to/slave/replica (is mandatory)"
echo "--rsakey=\"\" Alternative path to rsa private key for ssh connection to slave replica"
echo "--rsakey=\"\" Alternative path to rsa private key for ssh connection to slave replica"
echo "--sync-id=\"\" Optional sync task name to identify this synchronization task when using multiple slaves"
echo "--sync-id=\"\" Optional sync task name to identify this synchronization task when using multiple slaves"
echo ""
echo "Additionnaly, you may set most osync options at runtime. eg:"
echo "SOFT_DELETE_DAYS=365 osync.sh --master=/path --slave=/other/path"
echo ""
exit 128
exit 128
}
}
@ -2140,12 +2118,41 @@ then
then
then
SYNC_ID = "quicksync task"
SYNC_ID = "quicksync task"
fi
fi
MINIMUM_SPACE = 1024
REMOTE_SYNC = no
# Let the possibility to initialize those values directly via command line like SOFT_DELETE_DAYS=60 ./osync.sh
CONFLICT_BACKUP_DAYS = 30
SOFT_DELETE_DAYS = 30
if [ " $MINIMUM_SPACE " = = "" ]
RESUME_TRY = 1
then
MINIMUM_SPACE = 1024
fi
if [ " $CONFLICT_BACKUP_DAYS " = = "" ]
then
CONFLICT_BACKUP_DAYS = 30
fi
if [ " $SOFT_DELETE_DAYS " = = "" ]
then
SOFT_DELETE_DAYS = 30
fi
if [ " $RESUME_TRY " = = "" ]
then
RESUME_TRY = 1
fi
if [ " $SOFT_MAX_EXEC_TIME " = = "" ]
then
SOFT_MAX_EXEC_TIME = 0
fi
if [ " $HARD_MAX_EXEC_TIME " = = "" ]
then
HARD_MAX_EXEC_TIME = 0
fi
MIN_WAIT = 30
MIN_WAIT = 30
REMOTE_SYNC = no
else
else
ConfigFile = " $1 "
ConfigFile = " $1 "
LoadConfigFile " $ConfigFile "
LoadConfigFile " $ConfigFile "