diff --git a/CHANGELOG.md b/CHANGELOG.md index b9b25b9..4baacdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,22 +1,24 @@ FUTURE IMPROVEMENTS ------------------- -- Merge master and slave functions -- Merge tree current and after functions -- Tree functions execute piped commands (grep, awk) on master when launched on remote slave, can cause more bandwith usage +- Sync function merge (master and slave functions are more more or less the same) +- Tree function merge (current and after tree functions are the same except for output filename and logging) +- Tree functions execute piped commands (grep, awk) on master when launched on remote slave which can cause more bandwith usage KNOWN ISSUES ------------ -(v?)- Cannot write pidlock on remote slave with SUDO_EXEC=yes but insufficient rights (sudo does not work for command echo) -- If master and remote slave aren't the same distros and rsync binary isn't in the same path, execution may fail (RSYNC_PATH should be configurable) -- Possible non deleted file with space in name on master replica from slave remote replica -- can load configuration files that don't have .conf extension... -- Softdelete functions do not honor maximum execution time +- If master and remote slave systems don't have rsync in the same path, execution may fail (RSYNC_PATH is configured on master but also executed on slave) RECENT CHANGES -------------- +- Fixed a possible error upon master replica lock check +- Fixed exclude directorires with spaces in names generate errros on master replica tree functions +- Dryruns won't create after run tree lists and therefore not prevent building real run delete lists +- Softdelete and conflict backup functions are now time controlled +- Added bandwidth limit +- Update and delete functions now run rsync with --stats parameter - Fixed LoadConfigFile function will not warn on wrong config file - Without --verbose parameter, last sync details are still logged to /tmp/osync_(pid) - Added --no-maxtime parameter for sync big changes without enforcing execution time checks diff --git a/osync.sh b/osync.sh index c032b49..0e59c46 100755 --- a/osync.sh +++ b/osync.sh @@ -1,9 +1,9 @@ -#!/bin/bash +*#!/bin/bash ###### Osync - Rsync based two way sync engine with fault tolerance ###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr) OSYNC_VERSION=0.98 -OSYNC_BUILD=0408201302 +OSYNC_BUILD=0408201305 DEBUG=no SCRIPT_PID=$$ @@ -914,6 +914,11 @@ function delete_on_master function master_tree_after { + if [ $dryrun -eq 1 ] + then + Log "No need to create after run master replica file list, nothing should have changed." + return 0 + fi Log "Creating after run master replica file list." $(which $RSYNC_EXECUTABLE) --rsync-path="$RSYNC_PATH" -rlptgoDE8 $RSYNC_ARGS --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --list-only "$MASTER_SYNC_DIR/" | grep "^-\|^d" | awk '{$1=$2=$3=$4="" ;print}' | awk '{$1=$1 ;print}' | (grep -v "^\.$" || :) | sort > /dev/shm/osync_master-tree-after_$SCRIPT_PID & child_pid=$! @@ -932,6 +937,11 @@ function master_tree_after function slave_tree_after { + if [ $dryrun -eq 1 ] + then + Log "No need to create after frun slave replica file list, nothing should have changed." + return 0 + fi Log "Creating after run slave replica file list." if [ "$REMOTE_SYNC" == "yes" ] then @@ -1134,9 +1144,18 @@ function SoftDelete Log "Removing backups older than $CONFLICT_BACKUP_DAYS days on master replica." if [ $dryrun -eq 1 ] then - find "$MASTER_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS + find "$MASTER_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS & else - find "$MASTER_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS | xargs rm -rf + find "$MASTER_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS | xargs rm -rf & + fi + child_pid=$! + WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0 + retval=$? + if [ $retval -ne 0 ] + then + LogError "Error while executing conflict backup cleanup on master replica." + else + Log "Conflict backup cleanup complete on master replica." fi fi @@ -1149,6 +1168,15 @@ function SoftDelete else eval "$SSH_CMD \"if [ -d \\\"$SLAVE_BACKUP_DIR\\\" ]; then $COMMAND_SUDO find \\\"$SLAVE_BACKUP_DIR/\\\" -ctime +$CONFLICT_BACKUP_DAYS | xargs rm -rf; fi\"" fi + child_pid=$! + WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0 + retval=$? + if [ $retval -ne 0 ] + then + LogError "Error while executing conflict backup cleanup on slave replica." + else + Log "Conflict backup cleanup complete on slave replica." + fi else if [ -d "$SLAVE_BACKUP_DIR" ] then @@ -1159,6 +1187,15 @@ function SoftDelete else find "$SLAVE_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS | xargs rm -rf fi + child_pid=$! + WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0 + retval=$? + if [ $retval -ne 0 ] + then + LogError "Error while executing conflict backup cleanup on slave replica." + else + Log "Conflict backup cleanup complete on slave replica." + fi fi fi fi @@ -1174,6 +1211,15 @@ function SoftDelete else find "$MASTER_DELETE_DIR/" -ctime +$SOFT_DELETE_DAYS | xargs rm -rf fi + child_pid=$! + WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0 + retval=$? + if [ $retval -ne 0 ] + then + LogError "Error while executing soft delete cleanup on master replica." + else + Log "Soft delete cleanup complete on master replica." + fi fi if [ "$REMOTE_SYNC" == "yes" ] @@ -1185,6 +1231,16 @@ function SoftDelete else eval "$SSH_CMD \"if [ -d \\\"$SLAVE_DELETE_DIR\\\" ]; then $COMMAND_SUDO find \\\"$SLAVE_DELETE_DIR/\\\" -ctime +$SOFT_DELETE_DAYS | xargs rm -rf; fi\"" fi + child_pid=$! + WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0 + retval=$? + if [ $retval -ne 0 ] + then + LogError "Error while executing soft delete cleanup on slave replica." + else + Log "Soft delete cleanup complete on slave replica." + fi + else if [ -d "$SLAVE_DELETE_DIR" ] then @@ -1195,6 +1251,15 @@ function SoftDelete else find "$SLAVE_DELETE_DIR/" -ctime +$SOFT_DELETE_DAYS | xargs rm -rf fi + child_pid=$! + WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0 + retval=$? + if [ $retval -ne 0 ] + then + LogError "Error while executing soft delete cleanup on slave replica." + else + Log "Soft delete cleanup complete on slave replica." + fi fi fi fi @@ -1287,6 +1352,11 @@ function Init RSYNC_ARGS="" fi + if [ "$BANDWIDTH" != "0" ] + then + RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH" + fi + ## Conflict options if [ "$CONFLICT_BACKUP" != "no" ] then diff --git a/sync.conf b/sync.conf index 83c2b78..3ad48ed 100755 --- a/sync.conf +++ b/sync.conf @@ -2,17 +2,17 @@ ###### Osync - Rsync based two way sync engine with fault tolerance ###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr) -#### Config file rev 2207201302 +#### Config file rev 0408201301 ## Sync job identification, any string you want, no spaces SYNC_ID="sync_test" ## Directories to synchronize -MASTER_SYNC_DIR="/home/git/osync/test/the separate dir1" -SLAVE_SYNC_DIR="/home/git/osync/test/the separate dir2" +MASTER_SYNC_DIR="/home/git/osync/test/dir1" +SLAVE_SYNC_DIR="/home/git/osync/test/dir2" ## Create sync directories if they do not exist -CREATE_DIRS=yes +CREATE_DIRS=no ## List of directories to exclude in sync on both sides (rsync patterns, wildcards work). Must be relative paths. List is separated by PATH SEPARATOR CHAR defined below (semicolon by default). RSYNC_EXCLUDE_PATTERN="tmp;archives" @@ -22,6 +22,9 @@ PATH_SEPARATOR_CHAR=";" ## Generate an alert if master or slave have lass space than given value in KB. MINIMUM_SPACE=10240 +## Bandwidth limit Kbytes / second. Leave 0 to disable limitation +BANDWIDTH=0 + ## If enabled, synchronization will be processed with sudo command. See documentation SUDO_EXEC=no ## Paranoia option. Don't change this unless you read the documentation and know what you are doing. @@ -48,8 +51,8 @@ PRESERVE_XATTR=no RSYNC_COMPRESS=yes ## Maximum execution time (in seconds) for sync process. Soft exec time only generates warning. Hard exec time will generate warning and stop sync process. -SOFT_MAX_EXEC_TIME=18000 -HARD_MAX_EXEC_TIME=43200 +SOFT_MAX_EXEC_TIME=7200 +HARD_MAX_EXEC_TIME=10600 ## If the same file exists on both sides, newer version will be used. If both files have the same timestamp but differ, CONFILCT_PREVALANCE sets winner CONFLICT_PREVALANCE=master