Archives
/
osync
mirror of https://github.com/deajan/osync
2
0
Fork 0
Code Issues Releases Wiki Activity

Patch merges plus bug fixesÃ

Browse Source
pull/5/head
deajan 10 years ago
parent deb0058b6e
commit b65c5a22ff
4 changed files with 75 additions and 12 deletions
Show Stats Download Patch File Download Diff File

12
CHANGELOG.md
Unescape Escape View File

@ -16,9 +16,21 @@ KNOWN ISSUES
- Still need more testing on BSD, MacOSX and Windows MSYS
UNDER WORK
----------
!- Better deletion propagation (again). Using rsync for deletion propagation is definitly not working in all cases (especially empty sub directories)
!- Mutlislave asynchronous sync support
RECENT CHANGES
--------------
- Added an easier debug setting i.e DEBUG=yes ./osync.sh (Again, thanks to Ulrich Norbisrath)
- Added hardlink preservation (Thanks to Ulrich Norbisrath)
- Added external exclusion file support (Thanks to Pierre Clement)
- Fixed some typos in doc and program itself (Thanks to Pierre Clement)
- More detailled verbose status messages
- More detailled status messages
- Fixed a bug preventing propagation of empty directory deletions
- Fixed a nasty bug preventing writing lock files on remote system as superuser
- Gzipped logs are now deleted once sent

11
exclude.list.example
Unescape Escape View File

@ -0,0 +1,11 @@
.bash_history
.ssh
.config
.localized
.AppleDouble/
._*
.DS_Store
Thumbs.db
System Volume Information
$Recycle.Bin

55
osync.sh
Unescape Escape View File

@ -4,9 +4,14 @@ PROGRAM="Osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(L) 2013-2014 by Orsiris \"Ozy\" de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=0.99preRC3
PROGRAM_BUILD=3103201402
PROGRAM_BUILD=0805201403
## allow debugging from command line with preceding ocsync with DEBUG=yes
if [ ! "$DEBUG" == "yes" ]
then
DEBUG=no
fi
DEBUG=no
SCRIPT_PID=$$
LOCAL_USER=$(whoami)
@ -735,6 +740,14 @@ function RsyncExcludePattern
IFS=$OLD_IFS
}
function RsyncExcludeFrom
{
if [ ! $RSYNC_EXCLUDE_FROM == "" ] && [ -e $RSYNC_EXCLUDE_FROM ]
then
RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude-from=\"$RSYNC_EXCLUDE_FROM\""
fi
}
function WriteLockFiles
{
echo $SCRIPT_PID > "$MASTER_LOCK"
@ -909,7 +922,7 @@ function tree_list
ESC=$(EscapeSpaces "$1")
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -rlptgoDE8 $RSYNC_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$ESC/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/osync_$2_$SCRIPT_PID\" &"
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -rlptgoDE8 $RSYNC_ARGS --exclude \"$OSYNC_DIR\" $RSNYC_EXCLUDE --list-only \"$1/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/osync_$2_$SCRIPT_PID\" &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -rlptgoDE8 $RSYNC_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --list-only \"$1/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/osync_$2_$SCRIPT_PID\" &"
fi
LogDebug "RSYNC_CMD: $rsync_cmd"
## Redirect commands stderr here to get rsync stderr output in logfile
@ -1051,12 +1064,12 @@ function deletion_propagation
if [ "$1" == "master" ]
then
#rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats -e \"$RSYNC_SSH_CMD\" $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list\" --filter=\"-! */\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats -e \"$RSYNC_SSH_CMD\" $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list\" --filter=\"- *\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats -e \"$RSYNC_SSH_CMD\" $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include=\"*/\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list\" --filter=\"- *\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &"
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats -e \"$RSYNC_SSH_CMD\" $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list\" --filter=\"- *\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\"> $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats -e \"$RSYNC_SSH_CMD\" $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include=\"*/\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list\" --filter=\"- *\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\"> $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &"
fi
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list\" --filter=\"- *\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -rlptgoDEui --stats $DELETE_DIR --delete --exclude \"$OSYNC_DIR\" --include=\"*/\" --include-from=\"$MASTER_STATE_DIR/$1-deleted-list\" --filter=\"- *\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/osync_deletion_on_$2_$SCRIPT_PID 2>&1 &"
fi
LogDebug "RSYNC_CMD: $rsync_cmd"
eval "$rsync_cmd"
@ -1127,7 +1140,7 @@ function Sync
################################################################################################################################################# Actual sync begins here
## This replaces the case statement below because ;& operator is not supported in bash 3.2... Code is more messy than case :(
## This replaces the case statement because ;& operator is not supported in bash 3.2... Code is more messy than case :(
if [ "$resume_sync" == "none" ] || [ "$resume_sync" == "noresume" ] || [ "$resume_sync" == "master-replica-tree.fail" ]
then
#master_tree_current
@ -1218,7 +1231,12 @@ function SoftDelete
if [ $dryrun -eq 1 ]
then
Log "Listing backups older than $CONFLICT_BACKUP_DAYS days on master replica. Won't remove anything."
$FIND_CMD "$MASTER_SYNC_DIR$MASTER_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS &
if [ $verbose -eq 1 ]
then
$FIND_CMD "$MASTER_SYNC_DIR$MASTER_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS &
else
$FIND_CMD "$MASTER_SYNC_DIR$MASTER_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS > /dev/null &
fi
else
Log "Removing backups older than $CONFLICT_BACKUP_DAYS days on master replica."
$FIND_CMD "$MASTER_SYNC_DIR$MASTER_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS -exec rm -rf '{}' \; &
@ -1287,7 +1305,7 @@ function SoftDelete
if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]
then
if [ -w "$MASTER_SYNC_DIR$MASTER_DELETE_DIR" ]
if [ -d "$MASTER_SYNC_DIR$MASTER_DELETE_DIR" ]
then
if [ $dryrun -eq 1 ]
then
@ -1517,6 +1535,10 @@ function Init
then
RSYNC_ARGS=$RSYNC_ARGS"z"
fi
if [ "$PRESERVE_HARDLINKS" != "no" ]
then
RSYNC_ARGS$RSYNC_ARGS"H"
fi
if [ $dryrun -eq 1 ]
then
RSYNC_ARGS=$RSYNC_ARGS"n"
@ -1559,6 +1581,8 @@ function Init
## Add Rsync exclude patterns
RsyncExcludePattern
## Add Rsync exclude from file
RsyncExcludeFrom
}
function Main
@ -1592,6 +1616,7 @@ function Usage
echo "--master=\"\" Master replica path. Will contain state and backup directory (is mandatory)."
echo "--slave=\"\" Local or remote slave replica path. Can be a ssh uri like ssh://user@host.com:22//path/to/slave/replica (is mandatory)."
echo "--rsakey=\"\" Alternative path to rsa private key for ssh connection to slave replica"
echo "--sync-id=\"\" Optional task-id to identify this synchronization task when using multiple slaves."
exit 128
}
@ -1693,6 +1718,10 @@ do
SSH_RSA_PRIVATE_KEY=${i##*=}
opts=$opts" --rsakey=\"$SSH_RSA_PRIVATE_KEY\""
;;
--sync-id=*)
SYNC_ID=${i##*=}
opts=$opts" --sync-id=\"$SYNC_ID\""
;;
--on-changes)
sync_on_changes=1
;;
@ -1708,9 +1737,15 @@ opts="${opts# *}"
CheckEnvironment
if [ $? == 0 ]
then
## Here we set default options for quicksync tasks when no configuration file is provided.
if [ $quick_sync -eq 2 ]
then
SYNC_ID="quicksync task"
if [ "$SYNC_ID" == "" ]
then
SYNC_ID="quicksync task"
fi
MINIMUM_SPACE=1024
REMOTE_SYNC=no
CONFLICT_BACKUP_DAYS=30

9
sync.conf
Unescape Escape View File

@ -2,7 +2,7 @@
###### Osync - Rsync based two way sync engine with fault tolerance
###### (L) 2013-2014 by Orsiris "Ozy" de Jong (www.netpower.fr)
###### Config file rev 2303201401
###### Config file rev 0805201402
## ---------- GENERAL OPTIONS
@ -12,7 +12,7 @@ SYNC_ID="sync_test"
## Directories to synchronize. Master must be on the system Osync runs on. Slave can be either a local directory, or a remote one.
MASTER_SYNC_DIR="/home/git/osync/test/dir1"
#SLAVE_SYNC_DIR="/home/git/osync/test/dir2"
SLAVE_SYNC_DIR=""ssh://user@host.com:22//path/to/dir2"
SLAVE_SYNC_DIR="ssh://user@host.com:22//path/to/dir2"
## If slave replica is a remote directory, you must specifiy a RSA key (please use full path). Please see documentation for further information.
SSH_RSA_PRIVATE_KEY="/home/backupuser/.ssh/id_rsa"
@ -25,6 +25,9 @@ LOGFILE=""
## List of directories to exclude from sync on both sides (rsync patterns, wildcards work).
## Paths are relative to sync dirs. List elements are separated by a semicolon.
RSYNC_EXCLUDE_PATTERN="tmp;archives"
## File that contains the list of directories or files to exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file.
## Paths are relative to sync dirs. One element per line.
RSYNC_EXCLUDE_FROM="exclude.list"
## List elements separator char. You may set an alternative seperator char for your directories lists above.
PATH_SEPARATOR_CHAR=";"
@ -61,6 +64,8 @@ RSYNC_REMOTE_PATH=""
PRESERVE_ACL=no
## Preserve Xattr. Make sure source and target FS can manage same Xattrs or you'll get loads of errors.
PRESERVE_XATTR=no
## Preserve hard links. Make sure source and target FS can manage hard links or you will lose them.
PRESERVE_HARDLINKS=yes
## Let RSYNC compress file transfers. Do not use this if both master and slave replicas are on local system. Also, do not use this if you already enabled SSH compression.
RSYNC_COMPRESS=yes

Write Preview
Loading…
Cancel
Save