diff --git a/CHANGELOG.md b/CHANGELOG.md index ca1936c..7b78ca9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,9 +16,21 @@ KNOWN ISSUES - Still need more testing on BSD, MacOSX and Windows MSYS +UNDER WORK +---------- + +!- Better deletion propagation (again). Using rsync for deletion propagation is definitly not working in all cases (especially empty sub directories) +!- Mutlislave asynchronous sync support + RECENT CHANGES -------------- +- Added an easier debug setting i.e DEBUG=yes ./osync.sh (Again, thanks to Ulrich Norbisrath) +- Added hardlink preservation (Thanks to Ulrich Norbisrath) +- Added external exclusion file support (Thanks to Pierre Clement) +- Fixed some typos in doc and program itself (Thanks to Pierre Clement) +- More detailled verbose status messages +- More detailled status messages - Fixed a bug preventing propagation of empty directory deletions - Fixed a nasty bug preventing writing lock files on remote system as superuser - Gzipped logs are now deleted once sent diff --git a/exclude.list.example b/exclude.list.example new file mode 100644 index 0000000..8108833 --- /dev/null +++ b/exclude.list.example @@ -0,0 +1,11 @@ +.bash_history +.ssh +.config +.localized +.AppleDouble/ +._* +.DS_Store +Thumbs.db +System Volume Information +$Recycle.Bin + diff --git a/osync.sh b/osync.sh index a6f784f..6890961 100755 --- a/osync.sh +++ b/osync.sh @@ -4,9 +4,14 @@ PROGRAM="Osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(L) 2013-2014 by Orsiris \"Ozy\" de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=0.99preRC3 -PROGRAM_BUILD=3103201402 +PROGRAM_BUILD=0805201403 + +## allow debugging from command line with preceding ocsync with DEBUG=yes +if [ ! "$DEBUG" == "yes" ] +then + DEBUG=no +fi -DEBUG=no SCRIPT_PID=$$ LOCAL_USER=$(whoami) @@ -735,6 +740,14 @@ function RsyncExcludePattern IFS=$OLD_IFS } +function RsyncExcludeFrom +{ + if [ ! $RSYNC_EXCLUDE_FROM == "" ] && [ -e $RSYNC_EXCLUDE_FROM ] + then + RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude-from=\"$RSYNC_EXCLUDE_FROM\"" + fi +} + function WriteLockFiles { echo $SCRIPT_PID > "$MASTER_LOCK" @@ -909,7 +922,7 @@ function tree_list ESC=$(EscapeSpaces "$1") rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -rlptgoDE8 $RSYNC_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$ESC/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/osync_$2_$SCRIPT_PID\" &" else - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -rlptgoDE8 $RSYNC_ARGS --exclude \"$OSYNC_DIR\" $RSNYC_EXCLUDE --list-only \"$1/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/osync_$2_$SCRIPT_PID\" &" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -rlptgoDE8 $RSYNC_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --list-only \"$1/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/osync_$2_$SCRIPT_PID\" &" fi LogDebug "RSYNC_CMD: $rsync_cmd" ## Redirect commands stderr here to get rsync stderr output in logfile @@ -1051,12 +1064,12 @@ function deletion_propagation if [ "$1" == "master" ] then #rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats -e \"$RSYNC_SSH_CMD\" $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list\" --filter=\"-! */\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &" - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats -e \"$RSYNC_SSH_CMD\" $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list\" --filter=\"- *\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats -e \"$RSYNC_SSH_CMD\" $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include=\"*/\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list\" --filter=\"- *\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &" else - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats -e \"$RSYNC_SSH_CMD\" $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list\" --filter=\"- *\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\"> $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats -e \"$RSYNC_SSH_CMD\" $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include=\"*/\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list\" --filter=\"- *\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\"> $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &" fi else - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list\" --filter=\"- *\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include=\"*/\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list\" --filter=\"- *\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &" fi LogDebug "RSYNC_CMD: $rsync_cmd" eval "$rsync_cmd" @@ -1127,7 +1140,7 @@ function Sync ################################################################################################################################################# Actual sync begins here - ## This replaces the case statement below because ;& operator is not supported in bash 3.2... Code is more messy than case :( + ## This replaces the case statement because ;& operator is not supported in bash 3.2... Code is more messy than case :( if [ "$resume_sync" == "none" ] || [ "$resume_sync" == "noresume" ] || [ "$resume_sync" == "master-replica-tree.fail" ] then #master_tree_current @@ -1218,7 +1231,12 @@ function SoftDelete if [ $dryrun -eq 1 ] then Log "Listing backups older than $CONFLICT_BACKUP_DAYS days on master replica. Won't remove anything." - $FIND_CMD "$MASTER_SYNC_DIR$MASTER_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS & + if [ $verbose -eq 1 ] + then + $FIND_CMD "$MASTER_SYNC_DIR$MASTER_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS & + else + $FIND_CMD "$MASTER_SYNC_DIR$MASTER_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS > /dev/null & + fi else Log "Removing backups older than $CONFLICT_BACKUP_DAYS days on master replica." $FIND_CMD "$MASTER_SYNC_DIR$MASTER_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS -exec rm -rf '{}' \; & @@ -1287,7 +1305,7 @@ function SoftDelete if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ] then - if [ -w "$MASTER_SYNC_DIR$MASTER_DELETE_DIR" ] + if [ -d "$MASTER_SYNC_DIR$MASTER_DELETE_DIR" ] then if [ $dryrun -eq 1 ] then @@ -1517,6 +1535,10 @@ function Init then RSYNC_ARGS=$RSYNC_ARGS"z" fi + if [ "$PRESERVE_HARDLINKS" != "no" ] + then + RSYNC_ARGS$RSYNC_ARGS"H" + fi if [ $dryrun -eq 1 ] then RSYNC_ARGS=$RSYNC_ARGS"n" @@ -1559,6 +1581,8 @@ function Init ## Add Rsync exclude patterns RsyncExcludePattern + ## Add Rsync exclude from file + RsyncExcludeFrom } function Main @@ -1592,6 +1616,7 @@ function Usage echo "--master=\"\" Master replica path. Will contain state and backup directory (is mandatory)." echo "--slave=\"\" Local or remote slave replica path. Can be a ssh uri like ssh://user@host.com:22//path/to/slave/replica (is mandatory)." echo "--rsakey=\"\" Alternative path to rsa private key for ssh connection to slave replica" + echo "--sync-id=\"\" Optional task-id to identify this synchronization task when using multiple slaves." exit 128 } @@ -1693,6 +1718,10 @@ do SSH_RSA_PRIVATE_KEY=${i##*=} opts=$opts" --rsakey=\"$SSH_RSA_PRIVATE_KEY\"" ;; + --sync-id=*) + SYNC_ID=${i##*=} + opts=$opts" --sync-id=\"$SYNC_ID\"" + ;; --on-changes) sync_on_changes=1 ;; @@ -1708,9 +1737,15 @@ opts="${opts# *}" CheckEnvironment if [ $? == 0 ] then + + ## Here we set default options for quicksync tasks when no configuration file is provided. + if [ $quick_sync -eq 2 ] then - SYNC_ID="quicksync task" + if [ "$SYNC_ID" == "" ] + then + SYNC_ID="quicksync task" + fi MINIMUM_SPACE=1024 REMOTE_SYNC=no CONFLICT_BACKUP_DAYS=30 diff --git a/sync.conf b/sync.conf index 0b70a49..8e0fa6d 100755 --- a/sync.conf +++ b/sync.conf @@ -2,7 +2,7 @@ ###### Osync - Rsync based two way sync engine with fault tolerance ###### (L) 2013-2014 by Orsiris "Ozy" de Jong (www.netpower.fr) -###### Config file rev 2303201401 +###### Config file rev 0805201402 ## ---------- GENERAL OPTIONS @@ -12,7 +12,7 @@ SYNC_ID="sync_test" ## Directories to synchronize. Master must be on the system Osync runs on. Slave can be either a local directory, or a remote one. MASTER_SYNC_DIR="/home/git/osync/test/dir1" #SLAVE_SYNC_DIR="/home/git/osync/test/dir2" -SLAVE_SYNC_DIR=""ssh://user@host.com:22//path/to/dir2" +SLAVE_SYNC_DIR="ssh://user@host.com:22//path/to/dir2" ## If slave replica is a remote directory, you must specifiy a RSA key (please use full path). Please see documentation for further information. SSH_RSA_PRIVATE_KEY="/home/backupuser/.ssh/id_rsa" @@ -25,6 +25,9 @@ LOGFILE="" ## List of directories to exclude from sync on both sides (rsync patterns, wildcards work). ## Paths are relative to sync dirs. List elements are separated by a semicolon. RSYNC_EXCLUDE_PATTERN="tmp;archives" +## File that contains the list of directories or files to exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file. +## Paths are relative to sync dirs. One element per line. +RSYNC_EXCLUDE_FROM="exclude.list" ## List elements separator char. You may set an alternative seperator char for your directories lists above. PATH_SEPARATOR_CHAR=";" @@ -61,6 +64,8 @@ RSYNC_REMOTE_PATH="" PRESERVE_ACL=no ## Preserve Xattr. Make sure source and target FS can manage same Xattrs or you'll get loads of errors. PRESERVE_XATTR=no +## Preserve hard links. Make sure source and target FS can manage hard links or you will lose them. +PRESERVE_HARDLINKS=yes ## Let RSYNC compress file transfers. Do not use this if both master and slave replicas are on local system. Also, do not use this if you already enabled SSH compression. RSYNC_COMPRESS=yes