From 1f9b84dae321c9cca1c1e6713de41480a0d73956 Mon Sep 17 00:00:00 2001 From: deajan Date: Mon, 22 Jul 2013 11:34:58 +0200 Subject: [PATCH] Code cleanup, preparing sudo rsync and rsync executable support. --- CHANGELOG.md | 16 ++++++++-- osync.sh | 88 ++++++++++++++++++++++++++++++---------------------- sync.conf | 34 +++++++++++++++----- 3 files changed, 91 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 672cf18..326f3e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,17 @@ -! Tag as v1.0RC1 -! verify spaces in sync dirs -! Support for ACL and Xattr +TODO LIST +--------- +! Tag as v1.0 beta +! verify spaces in sync dir names +! sudo support for rsync and generic cmds +! add osync support in ssh_filter.sh +! minimum space checks +! different rsync executable + +RECENT CHANGES +-------------- + +- Added support for ACL and xattr - Added --force-unlock parameter to bypass any existing locks on replicas - Added full remote support for slave replica - Improved error detection diff --git a/osync.sh b/osync.sh index ff43755..6103e09 100755 --- a/osync.sh +++ b/osync.sh @@ -4,7 +4,7 @@ ###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr) OSYNC_VERSION=0.8 -OSYNC_BUILD=2107201304 +OSYNC_BUILD=2207201301 DEBUG=no SCRIPT_PID=$$ @@ -380,24 +380,6 @@ function RunAfterHook fi } -function SetSudoOptions -{ - ## Add this to support prior config files without RSYNC_EXECUTABLE option - if [ "$RSYNC_EXECUTABLE" == "" ] - then - RSYNC_EXECUTABLE=rsync - fi - - if [ "$SUDO_EXEC" == "yes" ] - then - RSYNC_PATH="sudo $(which $RSYNC_EXECUTABLE)" - COMMAND_SUDO="sudo" - else - RSYNC_PATH="$(which $RSYNC_EXECUTABLE)" - COMMAND_SUDO="" - fi -} - function CheckConnectivityRemoteHost { if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_SYNC" != "no" ] @@ -623,13 +605,13 @@ function LockDirectories then if [ "$slave_lock_id" == "$SYNC_ID" ] then - Log "There is a dead osync lock on slave replica that corresponds to this master. Instance $slave_lock_pid no longer running. Resuming." + Log "There is a dead osync lock on slave replica that corresponds to this master sync-id. Instance $slave_lock_pid no longer running. Resuming." else if [ "$FORCE_STRANGER_LOCK_RESUME" == "yes" ] then - LogError "WARNING: There is a dead osync lock on slave replica that does not correspond to this master. Forcing resume." + LogError "WARNING: There is a dead osync lock on slave replica that does not correspond to this master sync-id. Forcing resume." else - LogError "There is a dead osync lock on slave replica that does not correspond to this master. Will not resume." + LogError "There is a dead osync lock on slave replica that does not correspond to this master sync-id. Will not resume." exit 1 fi fi @@ -676,7 +658,8 @@ function UnlockDirectories function master_tree_current { Log "Creating master replica file list." - rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $MASTER_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_master-tree-current_$SCRIPT_PID & + rsync_cmd="$(which $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -rlptgodE --exclude \"$OSYNC_DIR\" --list-only \"$MASTER_SYNC_DIR/\" | grep \"^-\|^d\" | awk '{print $5}' | (grep -v \"^\.$\" || :) > /dev/shm/osync_master-tree-current_$SCRIPT_PID &" + eval $rsync_cmd child_pid=$! WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME if [ $? == 0 ] && [ -f /dev/shm/osync_master-tree-current_$SCRIPT_PID ] @@ -695,9 +678,9 @@ function slave_tree_current Log "Creating slave replica file list." if [ "$REMOTE_SYNC" == "yes" ] then - rsync -rlptgodE --exclude "$OSYNC_DIR" -e "$RSYNC_SSH_CMD" --list-only $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-current_$SCRIPT_PID & + $(which $RSYNC_EXECUTABLE) -rlptgodE --exclude "$OSYNC_DIR" -e "$RSYNC_SSH_CMD" --list-only $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-current_$SCRIPT_PID & else - rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-current_$SCRIPT_PID & + $(which $RSYNC_EXECUTABLe) -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-current_$SCRIPT_PID & fi child_pid=$! WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME @@ -744,9 +727,9 @@ function sync_update_slave Log "Updating slave replica." if [ "$REMOTE_SYNC" == "yes" ] then - rsync $DRY_OPTION -rlptgodEui -e "$RSYNC_SSH_CMD" $SLAVE_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 & + rsync $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $SLAVE_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 & else - rsync $DRY_OPTION -rlptgodEui $SLAVE_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 & + rsync $RSYNC_ARGS -rlptgodEui $SLAVE_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 & fi child_pid=$! WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME @@ -772,9 +755,9 @@ function sync_update_master Log "Updating master replica." if [ "$REMOTE_SYNC" == "yes" ] then - rsync $DRY_OPTION -rlptgodEui -e "$RSYNC_SSH_CMD" $MASTER_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --exclude-from "$MASTER_STATE_DIR/master-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 & + rsync $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $MASTER_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --exclude-from "$MASTER_STATE_DIR/master-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 & else - rsync $DRY_OPTION -rlptgodEui $MASTER_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --exclude-from "$MASTER_STATE_DIR/master-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 & + rsync $RSYNC_ARGS -rlptgodEui $MASTER_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --exclude-from "$MASTER_STATE_DIR/master-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 & fi child_pid=$! WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME @@ -800,9 +783,9 @@ function delete_on_slave Log "Propagating deletitions to slave replica." if [ "$REMOTE_SYNC" == "yes" ] then - rsync $DRY_OPTION -rlptgodEui -e "$RSYNC_SSH_CMD" $SLAVE_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --include-from "$MASTER_STATE_DIR/master-deleted-list" $MASTER_SYNC_DIR/ $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ > /dev/shm/osync_deletition_on_slave_$SCRIPT_PID 2>&1 & + rsync $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $SLAVE_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --include-from "$MASTER_STATE_DIR/master-deleted-list" $MASTER_SYNC_DIR/ $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ > /dev/shm/osync_deletition_on_slave_$SCRIPT_PID 2>&1 & else - rsync $DRY_OPTION -rlptgodEui $SLAVE_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --include-from "$MASTER_STATE_DIR/master-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ > /dev/shm/osync_deletition_on_slave_$SCRIPT_PID 2>&1 & + rsync $RSYNC_ARGS -rlptgodEui $SLAVE_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --include-from "$MASTER_STATE_DIR/master-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ > /dev/shm/osync_deletition_on_slave_$SCRIPT_PID 2>&1 & fi child_pid=$! WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0 @@ -827,9 +810,9 @@ function delete_on_master Log "Propagating deletitions to master replica." if [ "$REMOTE_SYNC" == "yes" ] then - rsync $DRY_OPTION -rlptgodEui -e "$RSYNC_SSH_CMD" $MASTER_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --include-from "$MASTER_STATE_DIR/slave-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_deletition_on_master_$SCRIPT_PID 2>&1 & + rsync $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $MASTER_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --include-from "$MASTER_STATE_DIR/slave-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_deletition_on_master_$SCRIPT_PID 2>&1 & else - rsync $DRY_OPTION -rlptgodEui $MASTER_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --include-from "$MASTER_STATE_DIR/slave-deleted-list" $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_deletition_on_master_$SCRIPT_PID 2>&1 & + rsync $RSYNC_ARGS -rlptgodEui $MASTER_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --include-from "$MASTER_STATE_DIR/slave-deleted-list" $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_deletition_on_master_$SCRIPT_PID 2>&1 & fi child_pid=$! WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0 @@ -1070,13 +1053,44 @@ function Init RSYNC_SSH_CMD="$(which ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -p $REMOTE_PORT" fi - ## Dryrun option + ## Set rsync executable and rsync path (for remote sudo rsync) + if [ "$RSYNC_EXECUTABLE" == "" ] + then + RSYNC_EXECUTABLE=rsync + fi + + if [ "$SUDO_EXEC" == "yes" ] + then + RSYNC_PATH="sudo $(which $RSYNC_EXECUTABLE)" + COMMAND_SUDO="sudo" + else + RSYNC_PATH="$(which $RSYNC_EXECUTABLE)" + COMMAND_SUDO="" + fi + + + ## Set rsync options + RSYNC_ARGS="-" + if [ "$PRESERVE_ACLS" == "yes" ] + then + RSYNC_ARGS=$RSYNC_ARGS"A" + fi + if [ "$PRESERVE_XATTR" == "yes" ] + then + RSYNC_ARGS=$RSYNC_ARGS"X" + fi + if [ "$RSYNC_COMPRESS" == "yes" ] + then + RSYNC_ARGS=$RSYNC_ARGS"z" + fi if [ $dryrun -eq 1 ] then - DRY_OPTION=--dry-run + RSYNC_ARGS=$RSYNC_ARGS"n" DRY_WARNING="/!\ DRY RUN" - else - DRY_OPTION= + fi + if [ "$RSYNC_ARGS" == "-" ] + then + RSYNC_ARGS="" fi ## Conflict options diff --git a/sync.conf b/sync.conf index b133cf7..d498a2f 100755 --- a/sync.conf +++ b/sync.conf @@ -2,16 +2,16 @@ ###### Osync - Rsync based two way sync engine with fault tolerance ###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr) -#### Config file rev 2107201302 +#### Config file rev 2207201301 -## Sync job identification, any string you want +## Sync job identification, any string you want, no spaces SYNC_ID="sync_test" ## Directories to synchronize MASTER_SYNC_DIR="/home/git/osync/test/dir1" SLAVE_SYNC_DIR="/home/git/osync/test/dir2" -## Create directories if they do not exist +## Create sync directories if they do not exist CREATE_DIRS=yes ## List of directories to exclude in sync on both sides (rsync patterns, wildcards work). Must be relative paths. List is separated by PATH SEPARATOR CHAR defined below (semicolon by default). @@ -33,37 +33,57 @@ SSH_RSA_PRIVATE_KEY=~/.ssh/id_rsa REMOTE_USER=backupmaster REMOTE_HOST=badministrateur.com REMOTE_PORT=48884 +## ssh compression should be used unless your remote connection is good enough (LAN) SSH_COMPRESSION=yes +## Check for connectivity to remote host before launching remote backup tasks. Be sure the hosts responds to ping. Failing to ping will skip current task. REMOTE_HOST_PING=no +## Check for internet access by pinging one or more 3rd party hosts before remote backup tasks. Leave empty if you don't want this check to be be performed. Failing to ping will skip current task. REMOTE_3RD_PARTY_HOST="www.kernel.org" +## Preserve ACLS. Make sure target FS can hold ACLs or you'll get loads of errors. PRESERVE_ACL=yes +## Preserve Xattr PRESERVE_XATTR=yes +## Let RSYNC compress file transfers. Do not use if you already enabled SSH compression. RSYNC_COMPRESS=yes +## Maximum execution time for sync process. Soft exec time only generates warning. Hard exec time will generate warning and stop sync process. SOFT_MAX_EXEC_TIME=30000 HARD_MAX_EXEC_TIME=36000 +## If the same file exists on both sides, newer version will be used. If both files have the same timestamp but differ, CONFILCT_PREVALANCE sets winner CONFLICT_PREVALANCE=master +## Keep a backup of a file if gets updated from remote side CONFLICT_BACKUP=yes -CONFLICT_BACKUP_DAYS=30 -# This can be very space consuming +## Keep multiple backups of a file if it gets updated from remote side. This can be very space consuming CONFLICT_BACKUP_MULTIPLE=yes +## Number of days to keep backups +CONFLICT_BACKUP_DAYS=30 +## On deletition propagation to sync partner, keep a backup of deleted files on sync partner SOFT_DELETE=yes +## Number of days to keep deleted files SOFT_DELETE_DAYS=30 -RESUME_SYNC=no +## Resume an aborted sync task +RESUME_SYNC=yes +## Number of times to try resuming before initating a new sync RESUME_TRY=2 -FORCE_STRANGER_LOCK_RESUME=yes +## When a dead pidlock exists on slave that does not correspond to master's sync-id, force pidlock removal +FORCE_STRANGER_LOCK_RESUME=no +## List of alert mails separated by spaces DESTINATION_MAILS="ozy@badministrateur.com" +## Run local commands before and after sync task LOCAL_RUN_BEFORE_CMD="" LOCAL_RUN_AFTER_CMD="" +## Run commands on remote slave befre and after sync task REMOTE_RUN_BEFORE_CMD="du /var/log" REMOTE_RUN_AFTER_CMD="du /tmp" +## Maximum execution time for commands before sync task. Commands get killed if not finished after MAX_EXC_TIMEe. Set this to 0 to disable killing. MAX_EXEC_TIME_PER_CMD_BEFORE=0 +## Maximum execution time for commands after sync task. Commands get killed if not finished after MAX_EXEC_TIME. Set this to 0 to disable killing command. MAX_EXEC_TIME_PER_CMD_AFTER=0