Archives
/
osync
mirror of https://github.com/deajan/osync
2
0
Fork 0
Code Issues Releases Wiki Activity

Fixed various glitches

Browse Source
pull/2/head
deajan 11 years ago
parent d6ebcbd55f
commit 50c4fe50e1
2 changed files with 61 additions and 41 deletions
Show Stats Download Patch File Download Diff File

88
osync.sh
Unescape Escape View File

@ -1,9 +1,10 @@
#!/bin/bash
###### Osync - Rsync based two way sync engine with fault tolerance
###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr)
OSYNC_VERSION=0.99
OSYNC_BUILD=1808201302
OSYNC_VERSION=0.99preRC2
OSYNC_BUILD=2408201301
DEBUG=no
SCRIPT_PID=$$
@ -153,8 +154,6 @@ function CleanUp
function SendAlert
{
CheckConnectivityRemoteHost
CheckConnectivity3rdPartyHosts
cat "$LOG_FILE" | gzip -9 > /tmp/osync_lastlog.gz
if type -p mutt > /dev/null 2>&1
then
@ -333,30 +332,21 @@ function RunLocalCommand
function RunRemoteCommand
{
CheckConnectivity3rdPartyHosts
if [ "$REMOTE_SYNC" == "yes" ]
CheckConnectivityRemoteHost
eval "$SSH_CMD \"$1\" > /dev/shm/osync_run_remote_$SCRIPT_PID 2>&1 &"
child_pid=$!
WaitForTaskCompletion $child_pid 0 $2
retval=$?
if [ $retval -eq 0 ]
then
CheckConnectivityRemoteHost
if [ $? != 0 ]
then
LogError "Connectivity test failed. Cannot run remote command."
return 1
else
eval "$SSH_CMD \"$1\" > /dev/shm/osync_run_remote_$SCRIPT_PID 2>&1 &"
fi
child_pid=$!
WaitForTaskCompletion $child_pid 0 $2
retval=$?
if [ $retval -eq 0 ]
then
Log "Running command [$1] succeded."
else
LogError "Running command [$1] failed."
fi
Log "Running command [$1] succeded."
else
LogError "Running command [$1] failed."
fi
if [ -f /dev/shm/osync_run_remote_$SCRIPT_PID ] && [ $verbose -eq 1 ]
then
Log "Command output:\n$(cat /dev/shm/osync_run_remote_$SCRIPT_PID)"
fi
if [ -f /dev/shm/osync_run_remote_$SCRIPT_PID ] && [ $verbose -eq 1 ]
then
Log "Command output:\n$(cat /dev/shm/osync_run_remote_$SCRIPT_PID)"
fi
}
@ -394,9 +384,9 @@ function CheckConnectivityRemoteHost
if [ $? != 0 ]
then
LogError "Cannot ping $REMOTE_HOST"
return 1
fi
fi
exit 1
fi
fi
}
function CheckConnectivity3rdPartyHosts
@ -404,7 +394,7 @@ function CheckConnectivity3rdPartyHosts
if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]
then
remote_3rd_party_success=0
for $i in $REMOTE_3RD_PARTY_HOSTS
for i in $REMOTE_3RD_PARTY_HOSTS
do
ping $i -c 2 > /dev/null 2>&1
if [ $? != 0 ]
@ -417,8 +407,8 @@ function CheckConnectivity3rdPartyHosts
if [ $remote_3rd_party_success -ne 1 ]
then
LogError "No remote 3rd party host responded to ping. No internet ?"
return 1
fi
exit 1
fi
fi
}
@ -436,6 +426,8 @@ function CreateOsyncDirs
if [ "$REMOTE_SYNC" == "yes" ]
then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
eval "$SSH_CMD \"if ! [ -d \\\"$SLAVE_STATE_DIR\\\" ]; then $COMMAND_SUDO mkdir --parents \\\"$SLAVE_STATE_DIR\\\"; fi\"" &
child_pid=$!
WaitForTaskCompletion $child_pid 0 1800
@ -470,6 +462,8 @@ function CheckMasterSlaveDirs
if [ "$REMOTE_SYNC" == "yes" ]
then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
if [ "$CREATE_DIRS" == "yes" ]
then
eval "$SSH_CMD \"if ! [ -d \\\"$SLAVE_SYNC_DIR\\\" ]; then $COMMAND_SUDO mkdir --parents \\\"$SLAVE_SYNC_DIR\\\"; fi"\" &
@ -524,6 +518,8 @@ function CheckMinimumSpace
if [ "$REMOTE_SYNC" == "yes" ]
then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
eval "$SSH_CMD \"$COMMAND_SUDO df -P \\\"$SLAVE_SYNC_DIR\\\"\"" > /dev/shm/osync_slave_space_$SCRIPT_PID &
child_pid=$!
WaitForTaskCompletion $child_pid 0 1800
@ -567,6 +563,8 @@ function WriteLockFiles
if [ "$REMOTE_SYNC" == "yes" ]
then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
eval "$SSH_CMD \"$COMMAND_SUDO echo $SCRIPT_PID@$SYNC_ID > \\\"$SLAVE_STATE_DIR/lock\\\"\"" &
child_pid=$!
WaitForTaskCompletion $child_pid 0 1800
@ -618,6 +616,8 @@ function LockDirectories
if [ "$REMOTE_SYNC" == "yes" ]
then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
eval "$SSH_CMD \"if [ -f \\\"$SLAVE_STATE_DIR/lock\\\" ]; then cat \\\"$SLAVE_STATE_DIR/lock\\\"; fi\" > /dev/shm/osync_remote_slave_lock_$SCRIPT_PID" &
child_pid=$!
WaitForTaskCompletion $child_pid 0 1800
@ -667,6 +667,8 @@ function UnlockDirectories
{
if [ "$REMOTE_SYNC" == "yes" ]
then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
eval "$SSH_CMD \"if [ -f \\\"$SLAVE_STATE_DIR/lock\\\" ]; then $COMMAND_SUDO rm \\\"$SLAVE_STATE_DIR/lock\\\"; fi\"" &
child_pid=$!
WaitForTaskCompletion $child_pid 0 1800
@ -723,6 +725,8 @@ function slave_tree_current
Log "Creating slave replica file list."
if [ "$REMOTE_SYNC" == "yes" ]
then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
rsync_cmd="$(which $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -rlptgoDE8 $RSYNC_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$ESC_SLAVE_SYNC_DIR/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > /dev/shm/osync_slave-tree-current_$SCRIPT_PID &"
else
rsync_cmd="$(which $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -rlptgoDE8 $RSYNC_ARGS --exclude \"$OSYNC_DIR\" $RSNYC_EXCLUDE --list-only \"$SLAVE_SYNC_DIR/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > /dev/shm/osync_slave-tree-current_$SCRIPT_PID &"
@ -777,6 +781,8 @@ function sync_update_slave
Log "Updating slave replica."
if [ "$REMOTE_SYNC" == "yes" ]
then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
rsync_cmd="$(which $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats -e \"$RSYNC_SSH_CMD\" $SLAVE_BACKUP --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from \"$MASTER_STATE_DIR/master-deleted-list\" --exclude-from \"$MASTER_STATE_DIR/slave-deleted-list\" \"$MASTER_SYNC_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SLAVE_SYNC_DIR/\" > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 &"
else
rsync_cmd="$(which $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats $SLAVE_BACKUP --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from \"$MASTER_STATE_DIR/master-deleted-list\" --exclude-from \"$MASTER_STATE_DIR/slave-deleted-list\" \"$MASTER_SYNC_DIR/\" \"$SLAVE_SYNC_DIR/\" > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 &"
@ -813,7 +819,8 @@ function sync_update_master
Log "Updating master replica."
if [ "$REMOTE_SYNC" == "yes" ]
then
#rsync_cmd="$(which $RSYNC_EXECUTABLE) $RSYNC_ARGS -rlptgoDEui --stats -e \"$RSYNC_SSH_CMD\" $MASTER_BACKUP --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from \"$MASTER_STATE_DIR/slave-deleted-list\" --exclude-from \"$MASTER_STATE_DIR/master-deleted-list\" $REMOTE_USER@$REMOTE_HOST:\"$SLAVE_SYNC_DIR/\" \"$MASTER_SYNC_DIR\" > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 &"
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
rsync_cmd="$(which $RSYNC_EXECUTABLE) $RSYNC_ARGS -rlptgoDEui --stats -e \"$RSYNC_SSH_CMD\" $MASTER_BACKUP --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from \"$MASTER_STATE_DIR/slave-deleted-list\" --exclude-from \"$MASTER_STATE_DIR/master-deleted-list\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SLAVE_SYNC_DIR/\" \"$MASTER_SYNC_DIR\" > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 &"
else
rsync_cmd="$(which $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats $MASTER_BACKUP --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from \"$MASTER_STATE_DIR/slave-deleted-list\" --exclude-from \"$MASTER_STATE_DIR/master-deleted-list\" \"$SLAVE_SYNC_DIR/\" \"$MASTER_SYNC_DIR/\" > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 &"
@ -850,6 +857,8 @@ function delete_on_slave
Log "Propagating deletions to slave replica."
if [ "$REMOTE_SYNC" == "yes" ]
then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
rsync_cmd="$(which $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats -e \"$RSYNC_SSH_CMD\" $SLAVE_DELETE --delete --exclude \"$OSYNC_DIR\" --include-from \"$MASTER_STATE_DIR/master-deleted-list\" --exclude=\"*\" \"$MASTER_SYNC_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SLAVE_SYNC_DIR/\" > /dev/shm/osync_deletion_on_slave_$SCRIPT_PID 2>&1 &"
else
#rsync_cmd="$(which $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats $SLAVE_DELETE --delete --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from \"$MASTER_STATE_DIR/slave-deleted-list\" --include-from \"$MASTER_STATE_DIR/master-deleted-list\" \"$MASTER_SYNC_DIR/\" \"$SLAVE_SYNC_DIR/\" > /dev/shm/osync_deletion_on_slave_$SCRIPT_PID 2>&1 &"
@ -886,9 +895,10 @@ function delete_on_master
Log "Propagating deletions to master replica."
if [ "$REMOTE_SYNC" == "yes" ]
then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
rsync_cmd="$(which $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats -e \"$RSYNC_SSH_CMD\" $MASTER_DELETE --delete --exclude \"$OSYNC_DIR\" --include-from \"$MASTER_STATE_DIR/slave-deleted-list\" --exclude=\"*\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SLAVE_SYNC_DIR/\" \"$MASTER_SYNC_DIR/\" > /dev/shm/osync_deletion_on_master_$SCRIPT_PID 2>&1 &"
else
#rsync_cmd="$(which $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats $MASTER_DELETE --delete --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from \"$MASTER_STATE_DIR/master-deleted-list\" --include-from \"$MASTER_STATE_DIR/slave-deleted-list\" \"$SLAVE_SYNC_DIR/\" \"$MASTER_SYNC_DIR/\" > /dev/shm/osync_deletion_on_master_$SCRIPT_PID 2>&1 &"
rsync_cmd="$(which $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats $MASTER_DELETE --delete --exclude \"$OSYNC_DIR\" --include-from \"$MASTER_STATE_DIR/slave-deleted-list\" --exclude=\"*\" \"$SLAVE_SYNC_DIR/\" \"$MASTER_SYNC_DIR/\" > /dev/shm/osync_deletion_on_master_$SCRIPT_PID 2>&1 &"
fi
if [ "$DEBUG" == "yes" ]
@ -955,6 +965,8 @@ function slave_tree_after
Log "Creating after run slave replica file list."
if [ "$REMOTE_SYNC" == "yes" ]
then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
rsync_cmd="$(which $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -rlptgoDE8 $RSYNC_ARGS -e \"$RSYNC_SSH_CMD\" --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --list-only $REMOTE_USER@$REMOTE_HOST:\"$ESC_SLAVE_SYNC_DIR/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > /dev/shm/osync_slave-tree-after_$SCRIPT_PID &"
else
rsync_cmd="$(which $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -rlptgoDE8 $RSYNC_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --list-only \"$SLAVE_SYNC_DIR/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > /dev/shm/osync_slave-tree-after_$SCRIPT_PID &"
@ -982,6 +994,8 @@ function slave_tree_after
function Sync
{
Log "Starting synchronization task."
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
if [ -f "$MASTER_STATE_DIR/last-action" ] && [ "$RESUME_SYNC" == "yes" ]
then
@ -1171,6 +1185,8 @@ function SoftDelete
if [ "$REMOTE_SYNC" == "yes" ]
then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
Log "Removing backups older than $CONFLICT_BACKUP_DAYS days on remote slave replica."
if [ $dryrun -eq 1 ]
then
@ -1234,6 +1250,8 @@ function SoftDelete
if [ "$REMOTE_SYNC" == "yes" ]
then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
Log "Removing soft deleted items older than $SOFT_DELETE_DAYS days on remote slave replica."
if [ $dryrun -eq 1 ]
then
@ -1482,7 +1500,7 @@ then
Log "-------------------------------------------------------------"
Log "$DRY_WARNING $DATE - Osync v$OSYNC_VERSION script begin."
Log "-------------------------------------------------------------"
Log "Backup task [$BACKUP_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)"
Log "Sync task [$SYNC_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)"
if [ $no_maxtime -eq 1 ]
then
SOFT_MAX_EXEC_TIME=0

14
sync.conf
Unescape Escape View File

@ -2,12 +2,12 @@
###### Osync - Rsync based two way sync engine with fault tolerance
###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr)
#### Config file rev 0408201301
#### Config file rev 2408201302
## Sync job identification, any string you want, no spaces
SYNC_ID="sync_test"
## Leaving this empty will create a logfile at /var/log/obackup_version_BACKUP_ID.log
## Leaving this empty will create a logfile at /var/log/osync_version_SYNC_ID.log
LOGFILE=""
## Directories to synchronize
@ -36,15 +36,17 @@ RSYNC_EXECUTABLE=rsync
##Remote options (will sync slave through ssh tunnel, needs RSA key. See documentation for remote sync.
REMOTE_SYNC=no
SSH_RSA_PRIVATE_KEY=~/.ssh/id_rsa
REMOTE_USER=backupuser
REMOTE_USER=syncuser
REMOTE_HOST=your-remote-host.tld
REMOTE_PORT=22
## ssh compression should be used unless your remote connection is good enough (LAN)
SSH_COMPRESSION=yes
## Check for connectivity to remote host before launching remote backup tasks. Be sure the hosts responds to ping. Failing to ping will skip current task.
## Check for connectivity to remote host before launching remote sync task. Be sure the hosts responds to ping. Failing to ping will stop sync.
REMOTE_HOST_PING=no
## Check for internet access by pinging one or more 3rd party hosts before remote backup tasks. Leave empty if you don't want this check to be be performed. Failing to ping will skip current task.
REMOTE_3RD_PARTY_HOST="www.kernel.org www.google.fr"
## Check for internet access by pinging one or more 3rd party hosts before remote sync task. Leave empty if you don't want this check to be be performed. Failing to ping will stop sync.
## If you use this function, you should set more than one 3rd party host, and be sure you can ping them.
## Be aware some DNS like opendns redirect false hostnames. Also, this adds an extra execution time of a bit less than a minute.
REMOTE_3RD_PARTY_HOSTS="www.kernel.org www.google.fr"
## Preserve ACLS. Make sure target FS can hold ACLs or you'll get loads of errors.
PRESERVE_ACL=no

Write Preview
Loading…
Cancel
Save