@ -3,8 +3,8 @@
PROGRAM = "Osync" # Rsync based two way sync engine with fault tolerance
AUTHOR = "(L) 2013-2015 by Orsiris \"Ozy\" de Jong"
CONTACT = "http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION = 0.99RC4
PROGRAM_BUILD = 3003201503
PROGRAM_VERSION = 1.00pre
PROGRAM_BUILD = 0104201502
## type doesn't work on platforms other than linux (bash). If if doesn't work, always assume output is not a zero exitcode
if ! type -p " $BASH " > /dev/null
@ -17,7 +17,7 @@ fi
if [ ! " $DEBUG " = = "yes" ]
then
DEBUG = no
SLEEP_TIME = 0 .1
SLEEP_TIME = .1
else
SLEEP_TIME = 3
fi
@ -36,10 +36,7 @@ else
fi
## Default directory where to store temporary run files
if [ -w /dev/shm ]
then
RUN_DIR = /dev/shm
elif [ -w /tmp ]
if [ -w /tmp ]
then
RUN_DIR = /tmp
elif [ -w /var/tmp ]
@ -60,6 +57,11 @@ export LC_ALL=C
ALERT_LOG_FILE = $RUN_DIR /osync_lastlog
function Dummy
{
sleep .1
}
function Log
{
if [ $sync_on_changes -eq 1 ]
@ -400,14 +402,14 @@ function WaitForTaskCompletion
then
Log "Current task still running."
fi
if [ $EXEC_TIME -gt $2 ]
if [ $EXEC_TIME -gt " $2 " ]
then
if [ $soft_alert -eq 0 ] && [ $2 != 0 ]
if [ $soft_alert -eq 0 ] && [ " $2 " != 0 ]
then
LogError "Max soft execution time exceeded for task."
soft_alert = 1
fi
if [ $EXEC_TIME -gt $3 ] && [ $3 != 0 ]
if [ $EXEC_TIME -gt " $3 " ] && [ " $3 " != 0 ]
then
LogError "Max hard execution time exceeded for task. Stopping task execution."
kill -s SIGTERM $1
@ -431,7 +433,7 @@ function WaitForTaskCompletion
return $?
}
# Waits for pid $1 to complete. Will log an alert if $2 seconds passed since script start unless $2 equals 0.
# Waits for pid $1 to complete. Will log an alert if $2 seconds passed since script start unless $2 equals 0.
# Will stop task and log alert if $3 seconds passed since script start unless $3 equals 0.
function WaitForCompletion
{
@ -443,14 +445,14 @@ function WaitForCompletion
then
Log "Current task still running."
fi
if [ $SECONDS -gt $2 ]
if [ $SECONDS -gt " $2 " ]
then
if [ $soft_alert -eq 0 ] && [ $2 != 0 ]
if [ $soft_alert -eq 0 ] && [ " $2 " != 0 ]
then
LogError "Max soft execution time exceeded for script."
soft_alert = 1
fi
if [ $SECONDS -gt $3 ] && [ $3 != 0 ]
if [ $SECONDS -gt " $3 " ] && [ " $3 " != 0 ]
then
LogError "Max hard execution time exceeded for script. Stopping current task execution."
kill -s SIGTERM $1
@ -1501,157 +1503,129 @@ function SoftDelete
{
if [ " $CONFLICT_BACKUP " != "no" ] && [ $CONFLICT_BACKUP_DAYS -ne 0 ]
then
if [ -d " $MASTER_SYNC_DIR $MASTER_BACKUP_DIR " ]
Log "Running conflict backup cleanup."
_SoftDelete $CONFLICT_BACKUP_DAYS " $MASTER_SYNC_DIR $MASTER_BACKUP_DIR " " $SLAVE_SYNC_DIR $SLAVE_BACKUP_DIR "
fi
if [ " $SOFT_DELETE " != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]
then
Log "Running soft deletion cleanup."
_SoftDelete $SOFT_DELETE_DAYS " $MASTER_SYNC_DIR $MASTER_DELETE_DIR " " $SLAVE_SYNC_DIR $SLAVE_DELETE_DIR "
fi
}
# Takes 3 arguments
# $1 = ctime (CONFLICT_BACKUP_DAYS or SOFT_DELETE_DAYS), $2 = MASTER_(BACKUP/DELETED)_DIR, $3 = SLAVE_(BACKUP/DELETED)_DIR
function _SoftDelete
{
if [ -d " $2 " ]
then
if [ $dryrun -eq 1 ]
then
if [ $dryrun -eq 1 ]
then
Log " Listing backups older than $CONFLICT_BACKUP_DAYS days on master replica. Won't remove anything. "
if [ $verbose -eq 1 ]
then
$FIND_CMD " $MASTER_SYNC_DIR $MASTER_BACKUP_DIR / " -ctime +$CONFLICT_BACKUP_DAYS &
else
$FIND_CMD " $MASTER_SYNC_DIR $MASTER_BACKUP_DIR / " -ctime +$CONFLICT_BACKUP_DAYS > /dev/null &
fi
else
Log " Removing backups older than $CONFLICT_BACKUP_DAYS days on master replica. "
$FIND_CMD " $MASTER_SYNC_DIR $MASTER_BACKUP_DIR / " -ctime +$CONFLICT_BACKUP_DAYS -exec rm -rf '{}' \; &
fi
child_pid = $!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
retval = $?
if [ $retval -ne 0 ]
then
LogError "Error while executing conflict backup cleanup on master replica."
else
Log "Conflict backup cleanup complete on master replica."
fi
elif [ -d " $MASTER_SYNC_DIR $MASTER_BACKUP_DIR " ] && ! [ -w " $MASTER_SYNC_DIR $MASTER_BACKUP_DIR " ]
Log " Listing files older than $1 days on master replica. Won't remove anything. "
else
Log " Removing files older than $1 days on master replica. "
fi
if [ $verbose -eq 1 ]
then
LogError " Warning: Master replica conflict backup dir [ $MASTER_SYNC_DIR $MASTER_BACKUP_DIR ] isn't writable. Cannot clean old files. "
# Cannot launch log function from xargs, ugly hack
$FIND_CMD " $2 / " -type f -ctime +$1 -print0 | xargs -0 -I { } echo "Will delete file {}" > $RUN_DIR /osync_soft_delete_master_$SCRIPT_PID
Log " Command output:\n $( cat $RUN_DIR /osync_soft_delete_master_$SCRIPT_PID ) "
$FIND_CMD " $2 / " -type d -empty -ctime +$1 -print0 | xargs -0 -I { } echo "Will delete directory {}" > $RUN_DIR /osync_soft_delete_master_$SCRIPT_PID
Log " Command output:\n $( cat $RUN_DIR /osync_soft_delete_master_$SCRIPT_PID ) "
fi
if [ " $REMOTE_SYNC " = = "yes" ]
if [ $dryrun -ne 1 ]
then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
if [ $dryrun -eq 1 ]
then
Log " Listing backups older than $CONFLICT_BACKUP_DAYS days on slave replica. Won't remove anything. "
eval " $SSH_CMD \"if [ -w \\\" $SLAVE_SYNC_DIR $SLAVE_BACKUP_DIR \\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\" $SLAVE_SYNC_DIR $SLAVE_BACKUP_DIR /\\\" -ctime + $CONFLICT_BACKUP_DAYS ; fi\" " &
else
Log " Removing backups older than $CONFLICT_BACKUP_DAYS days on remote slave replica. "
eval " $SSH_CMD \"if [ -w \\\" $SLAVE_SYNC_DIR $SLAVE_BACKUP_DIR \\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\" $SLAVE_SYNC_DIR $SLAVE_BACKUP_DIR /\\\" -ctime + $CONFLICT_BACKUP_DAYS -exec rm -rf '{}' \;; fi\" " &
fi
child_pid = $!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
retval = $?
if [ $retval -ne 0 ]
then
LogError "Error while executing conflict backup cleanup on slave replica."
else
Log "Conflict backup cleanup complete on slave replica."
fi
$FIND_CMD " $2 / " -type f -ctime +$1 -print0 | xargs -0 -I { } rm -f "{}" && $FIND_CMD " $2 / " -type d -empty -ctime +$1 -print0 | xargs -0 -I { } rm -rf "{}" > $RUN_DIR /osync_soft_delete_master_$SCRIPT_PID &
else
if [ -w " $SLAVE_SYNC_DIR $SLAVE_BACKUP_DIR " ]
then
if [ $dryrun -eq 1 ]
then
Log " Listing backups older than $CONFLICT_BACKUP_DAYS days on slave replica. Won't remove anything. "
$FIND_CMD " $SLAVE_SYNC_DIR $SLAVE_BACKUP_DIR / " -ctime +$CONFLICT_BACKUP_DAYS &
else
Log " Removing backups older than $CONFLICT_BACKUP_DAYS days on slave replica. "
$FIND_CMD " $SLAVE_SYNC_DIR $SLAVE_BACKUP_DIR / " -ctime +$CONFLICT_BACKUP_DAYS -exec rm -rf '{}' \; &
fi
child_pid = $!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
retval = $?
if [ $retval -ne 0 ]
then
LogError "Error while executing conflict backup cleanup on slave replica."
else
Log "Conflict backup cleanup complete on slave replica."
fi
elif [ -d " $SLAVE_SYNC_DIR $SLAVE_BACKUP_DIR " ] && ! [ -w " $SLAVE_SYNC_DIR $SLAVE_BACKUP_DIR " ]
then
LogError " Warning: Slave replica conflict backup dir [ $SLAVE_SYNC_DIR $SLAVE_BACKUP_DIR ] isn't writable. Cannot clean old files. "
fi
Dummy &
fi
child_pid = $!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
retval = $?
if [ $retval -ne 0 ]
then
LogError "Error while executing cleanup on master replica."
Log " Command output:\n $( cat $RUN_DIR /osync_soft_delete_master_$SCRIPT_PID ) "
else
Log "Cleanup complete on master replica."
fi
elif [ -d " $2 " ] && ! [ -w " $2 " ]
then
LogError " Warning: Master replica dir [ $2 ] isn't writable. Cannot clean old files. "
fi
if [ " $SOFT_DELETE " != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]
if [ " $REMOTE_SYNC " = = "yes" ]
then
if [ -d " $MASTER_SYNC_DIR $MASTER_DELETE_DIR " ]
then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
if [ $dryrun -eq 1 ]
then
Log " Listing soft deleted items older than $SOFT_DELETE_DAYS days on master replica. Won't remove anything. "
$FIND_CMD " $MASTER_SYNC_DIR $MASTER_DELETE_DIR / " -ctime +$SOFT_DELETE_DAYS &
else
Log " Removing soft deleted items older than $SOFT_DELETE_DAYS days on master replica. "
$FIND_CMD " $MASTER_SYNC_DIR $MASTER_DELETE_DIR / " -ctime +$SOFT_DELETE_DAYS -exec rm -rf '{}' \; &
fi
child_pid = $!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
retval = $?
if [ $retval -ne 0 ]
then
LogError "Error while executing soft delete cleanup on master replica."
else
Log "Soft delete cleanup complete on master replica."
fi
elif [ -d " $MASTER_SYNC_DIR $MASTER_DELETE_DIR " ] && ! [ -w " $MASTER_SYNC_DIR $MASTER_DELETE_DIR " ]
then
LogError " Warning: Master replica deletion backup dir [ $MASTER_SYNC_DIR $MASTER_DELETE_DIR ] isn't writable. Cannot clean old files. "
Log " Listing files older than $1 days on slave replica. Won't remove anything. "
else
Log " Removing files older than $1 days on slave replica. "
fi
if [ $verbose -eq 1 ]
then
# Cannot launch log function from xargs, ugly hack
eval " $SSH_CMD \"if [ -w \\\" $3 \\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\" $3 /\\\" -type f -ctime + $1 -print0 | xargs -0 -I {} echo Will delete file {} && $REMOTE_FIND_CMD \\\" $3 /\\\" -type d -empty -ctime $1 -print0 | xargs -0 -I {} echo Will delete directory {}; fi\" " > $RUN_DIR /osync_soft_delete_slave_$SCRIPT_PID
Log " Command output:\n $( cat $RUN_DIR /osync_soft_delete_slave_$SCRIPT_PID ) "
fi
if [ " $REMOTE_SYNC " = = "yes" ]
if [ $dryrun -ne 1 ]
then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
if [ $dryrun -eq 1 ]
then
Log " Listing soft deleted items older than $SOFT_DELETE_DAYS days on slave replica. Won't remove anything. "
eval " $SSH_CMD \"if [ -w \\\" $SLAVE_SYNC_DIR $SLAVE_DELETE_DIR \\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\" $SLAVE_SYNC_DIR $SLAVE_DELETE_DIR /\\\" -ctime + $SOFT_DELETE_DAYS ; fi\" " &
else
Log " Removing soft deleted items older than $SOFT_DELETE_DAYS days on remote slave replica. "
eval " $SSH_CMD \"if [ -w \\\" $SLAVE_SYNC_DIR $SLAVE_DELETE_DIR \\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\" $SLAVE_SYNC_DIR $SLAVE_DELETE_DIR /\\\" -ctime + $SOFT_DELETE_DAYS -exec rm -rf '{}' \;; fi\" " &
fi
child_pid = $!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
retval = $?
if [ $retval -ne 0 ]
then
LogError "Error while executing soft delete cleanup on slave replica."
eval " $SSH_CMD \"if [ -w \\\" $3 \\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\" $3 /\\\" -type f -ctime + $1 -print0 | xargs -0 -I {} rm -f \\\"{}\\\" && $REMOTE_FIND_CMD \\\" $3 /\\\" -type d -empty -ctime $1 -print0 | xargs -0 -I {} rm -rf \\\"{}\\\"; fi\" " > $RUN_DIR /osync_soft_delete_slave_$SCRIPT_PID &
else
Dummy &
fi
child_pid = $!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
retval = $?
if [ $retval -ne 0 ]
then
LogError "Error while executing cleanup on slave replica."
Log " Command output:\n $( cat $RUN_DIR /osync_soft_delete_slave_$SCRIPT_PID ) "
else
Log "Soft delete cleanup complete on slave replica."
fi
Log "Cleanup complete on slave replica."
fi
else
if [ -w " $3 " ]
then
if [ $dryrun -eq 1 ]
then
Log " Listing files older than $1 days on slave replica. Won't remove anything. "
else
if [ -w " $SLAVE_SYNC_DIR $SLAVE_DELETE_DIR " ]
then
if [ $dryrun -eq 1 ]
then
Log " Listing soft deleted items older than $SOFT_DELETE_DAYS days on slave replica. Won't remove anything. "
$FIND_CMD " $SLAVE_SYNC_DIR $SLAVE_DELETE_DIR / " -ctime +$SOFT_DELETE_DAYS &
else
Log " Removing soft deleted items older than $SOFT_DELETE_DAYS days on slave replica. "
$FIND_CMD " $SLAVE_SYNC_DIR $SLAVE_DELETE_DIR / " -ctime +$SOFT_DELETE_DAYS -exec rm -rf '{}' \; &
fi
child_pid = $!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
retval = $?
if [ $retval -ne 0 ]
then
LogError "Error while executing soft delete cleanup on slave replica."
else
Log "Soft delete cleanup complete on slave replica."
fi
elif [ -d " $SLAVE_SYNC_DIR $SLAVE_DELETE_DIR " ] && ! [ -w " $SLAVE_SYNC_DIR $SLAVE_DELETE_DIR " ]
then
LogError " Warning: Slave replica deletion backup dir [ $SLAVE_SYNC_DIR $SLAVE_DELETE_DIR ] isn't writable. Cannot clean old files. "
fi
Log " Removing files older than $1 days on slave replica. "
fi
if [ $verbose -eq 1 ]
then
# Cannot launch log function from xargs, ugly hack
$FIND_CMD " $3 / " -type f -ctime +$1 -print0 | xargs -0 -I { } echo "Will delete file {}" > $RUN_DIR /osync_soft_delete_slave_$SCRIPT_PID
Log " Command output:\n $( cat $RUN_DIR /osync_soft_delete_slave_$SCRIPT_PID ) "
$FIND_CMD " $3 / " -type d -empty -ctime +$1 -print0 | xargs -0 -I { } echo "Will delete directory {}" > $RUN_DIR /osync_soft_delete_slave_$SCRIPT_PID
Log " Command output:\n $( cat $RUN_DIR /osync_conflict_backup_slave_$SCRIPT_PID ) "
Dummy &
fi
if [ $dryrun -ne 1 ]
then
$FIND_CMD " $3 / " -type f -ctime +$1 -print0 | xargs -0 -I { } rm -f "{}" && $FIND_CMD " $3 / " -type d -empty -ctime +$1 -print0 | xargs -0 -I { } rm -rf "{}" > $RUN_DIR /osync_soft_delete_slave_$SCRIPT_PID &
fi
child_pid = $!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
retval = $?
if [ $retval -ne 0 ]
then
LogError "Error while executing cleanup on slave replica."
Log " Command output:\n $( cat $RUN_DIR /osync_soft_delete_slave_$SCRIPT_PID ) "
else
Log "Cleanup complete on slave replica."
fi
elif [ -d " $3 " ] && ! [ -w " $3 " ]
then
LogError " Warning: Slave replica dir [ $3 ] isn't writable. Cannot clean old files. "
fi
fi
}
function Init
@ -1956,9 +1930,9 @@ function Usage
echo $CONTACT
echo ""
echo "You may use Osync with a full blown configuration file, or use its default options for quick command line sync."
echo "Usage: osync /path/to/config/file [OPTIONS]"
echo "or osync --master=/path/to/master/replica --slave=/path/to/slave/replica [OPTIONS] [QUICKSYNC OPTIONS]"
echo "or osync --master=/path/to/master/replica --slave=ssh://backupuser@remotehost.com[:portnumber]//path/to/slave/replica [OPTIONS] [QUICKSYNC OPTIONS]"
echo "Usage: osync .sh /path/to/config/file [OPTIONS]"
echo "or osync .sh --master=/path/to/master/replica --slave=/path/to/slave/replica [OPTIONS] [QUICKSYNC OPTIONS]"
echo "or osync .sh --master=/path/to/master/replica --slave=ssh://[ backupuser] @remotehost.com[:portnumber]//path/to/slave/replica [OPTIONS] [QUICKSYNC OPTIONS]"
echo ""
echo "[OPTIONS]"
echo "--dry Will run osync without actually doing anything; just testing"
@ -1975,6 +1949,10 @@ function Usage
echo "--slave=\"\" Local or remote slave replica path. Can be a ssh uri like ssh://user@host.com:22//path/to/slave/replica (is mandatory)"
echo "--rsakey=\"\" Alternative path to rsa private key for ssh connection to slave replica"
echo "--sync-id=\"\" Optional sync task name to identify this synchronization task when using multiple slaves"
echo ""
echo "Additionnaly, you may set most osync options at runtime. eg:"
echo "SOFT_DELETE_DAYS=365 osync.sh --master=/path --slave=/other/path"
echo ""
exit 128
}
@ -2140,12 +2118,41 @@ then
then
SYNC_ID = "quicksync task"
fi
MINIMUM_SPACE = 1024
REMOTE_SYNC = no
CONFLICT_BACKUP_DAYS = 30
SOFT_DELETE_DAYS = 30
RESUME_TRY = 1
# Let the possibility to initialize those values directly via command line like SOFT_DELETE_DAYS=60 ./osync.sh
if [ " $MINIMUM_SPACE " = = "" ]
then
MINIMUM_SPACE = 1024
fi
if [ " $CONFLICT_BACKUP_DAYS " = = "" ]
then
CONFLICT_BACKUP_DAYS = 30
fi
if [ " $SOFT_DELETE_DAYS " = = "" ]
then
SOFT_DELETE_DAYS = 30
fi
if [ " $RESUME_TRY " = = "" ]
then
RESUME_TRY = 1
fi
if [ " $SOFT_MAX_EXEC_TIME " = = "" ]
then
SOFT_MAX_EXEC_TIME = 0
fi
if [ " $HARD_MAX_EXEC_TIME " = = "" ]
then
HARD_MAX_EXEC_TIME = 0
fi
MIN_WAIT = 30
REMOTE_SYNC = no
else
ConfigFile = " $1 "
LoadConfigFile " $ConfigFile "