diff --git a/CHANGELOG.md b/CHANGELOG.md index ee90410..a9fc4aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,16 @@ -TODO LIST ---------- +KNOWN BUGS +---------- -! Tag as v1.0 beta -! verify spaces in sync dir names -! exit code line 627 +- Cannot write pidlock on remote slave with SUDO_EXEC=yes but insufficient rights (sudo does not work for command echo) +- If master and remote slave aren't the same distros and rsync binary isn't in the same path, execution may fail (RSYNC_PATH should be configurable) RECENT CHANGES -------------- +24 Jul. 2013: beta milestone +- Fixed some bad error handling in CheckMasterSlaveDirs and LockDirectories +- Added support for spaces in sync dirs and exclude lists +- Fixed false exit code if no remote slave lock present - Added minimum disk space checks - Added osync support in ssh_filter.sh - Added support for sudo exec on remote slave @@ -33,6 +36,5 @@ RECENT CHANGES - Added master/slave conflict prevalance option - Added soft-deleted items - Added backup items in case of conflict - 19 Jun. 2013: Project begin diff --git a/README.md b/README.md index 2166c53..1106879 100644 --- a/README.md +++ b/README.md @@ -6,17 +6,49 @@ A two way sync script based that adds script fault tolerance from obackup projec ## About Having created obackup script in order to make reliable quick backups, i searched for a nice tool to handle two (or more) way sync scenarios in a reliable way. -While unison handles these scenarios, it's pretty messy to configure, slow and won't handle ACLs. -That's where bitpocket came handy, a nice script provided by sickill https://github.com/sickill/bitpocket.git -It's quick and small, but lacks some of the features i searched for like fault tolerance, stop and resume scenarios, and email warnings. - -I then decided to write my own implementation of a two way rsync sync script, which would the features i wanted. +While unison handles these scenarios, it's pretty messy to configure, slow, won't handle ACLs and won't resume if something bad happened. +Then i read about bitpocket, a nice script provided by sickill https://github.com/sickill/bitpocket.git +Bitpocked inspired me to write my own implementation of a two way sync script, implementing features i wanted among: + - Fault tolerance with resume scenarios + - Email alerts + - Logging facility + - Soft deletition and multiple backups handling + +Osync uses a master / slave sync schema. It can sync local or remote directories. By definition, master replica should always be a local directory on the system osync runs on. +Also, osync uses pidlocks to prevent multiple concurrent sync processes on/to the same master / slave replica. Be sure a sync process is finished before launching next one. +You may launch concurrent sync processes on the same system but only for different master replicas. ## Installation -Not even beta ready yet. The whole code is not stable at all. -Hopefully will work (more or less) by the end of July. I'm developping this in my free time. +Osync developpment is still not finished. It's currently at beta stage. Please read CHANGELOG.md for a list of known bugs. +Keep in mind that Osync has been designed to not delete any data, but rather make backups or soft deletes. +Nevertheless, as we're still in beta stage, please make a backup of your data before using Osync. + +First, grab a fresh copy of osync and make it executable: + $ git clone https://github.com/deajan/osync + $ chmod +x ./osync.sh + +Then, edit the sync.conf file according to your needs. + +## Usage + +Once you've setup a personalized sync.conf file, you may run osync with the following test run: + + $ ./osync.sh /path/to/your.conf --dry + +If everything went well, you may run the actual configuration with one of the following: + + $ ./osync.sh /path/to/your.conf + $ ./osync.sh /path/to/your.conf --verbose + +Verbose option will display which files and attrs are actually synchronized. +Once you're confident about your fist runs, you may add osync as cron task with: + + $ ./osync.sh /path/to/your.conf --silent + +You may then find osync output in /var/log/osync-*.log ## Author -Orsiris "Ozy" de Jong | ozy@badministrateur.com +Feel free to mail me for limited support in my free time :) +Orsiris "Ozy" de Jong | ozy@netpower.fr diff --git a/osync.sh b/osync.sh index afc19dc..f2021d0 100755 --- a/osync.sh +++ b/osync.sh @@ -2,10 +2,10 @@ ###### Osync - Rsync based two way sync engine with fault tolerance ###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr) -OSYNC_VERSION=0.92 -OSYNC_BUILD=2407201303 +OSYNC_VERSION=0.95 +OSYNC_BUILD=2407201304 -DEBUG=yes +DEBUG=no SCRIPT_PID=$$ LOCAL_USER=$(whoami) @@ -433,7 +433,8 @@ function CreateOsyncDirs if [ "$REMOTE_SYNC" == "yes" ] then eval "$SSH_CMD \"if ! [ -d \\\"$SLAVE_STATE_DIR\\\" ]; then $COMMAND_SUDO mkdir --parents \\\"$SLAVE_STATE_DIR\\\"; fi\"" & - WaitForTaskCompletion $! 0 1800 + child_pid=$! + WaitForTaskCompletion $child_pid 0 1800 else if ! [ -d "$SLAVE_STATE_DIR" ]; then mkdir --parents "$SLAVE_STATE_DIR"; fi fi @@ -468,19 +469,24 @@ function CheckMasterSlaveDirs if [ "$CREATE_DIRS" == "yes" ] then eval "$SSH_CMD \"if ! [ -d \\\"$SLAVE_SYNC_DIR\\\" ]; then $COMMAND_SUDO mkdir --parents \\\"$SLAVE_SYNC_DIR\\\"; fi"\" & - WaitForTaskCompletion $! 0 1800 + child_pid=$! + WaitForTaskCompletion $child_pid 0 1800 if [ $? != 0 ] then LogError "Cannot create slave directory [$SLAVE_SYNC_DIR]." exit 1 fi else - eval "$SSH_CMD \"if ! [ -d \\\"$SLAVE_SYNC_DIR\\\" ]; then exit 1; fi"\" & - WaitForTaskCompletion $! 0 1800 - if [ $? != 0 ] + eval "$SSH_CMD \"if ! [ -d \\\"$SLAVE_SYNC_DIR\\\" ]; then exit 1; fi"\" & + child_pid=$! + WaitForTaskCompletion $child_pid 0 1800 + res=$? + if [ $res != 0 ] then LogError "Slave directory [$SLAVE_SYNC_DIR] does not exist." exit 1 + else + LogError "dafuck" fi fi else @@ -517,7 +523,8 @@ function CheckMinimumSpace if [ "$REMOTE_SYNC" == "yes" ] then eval "$SSH_CMD \"$COMMAND_SUDO df -P \\\"$SLAVE_SYNC_DIR\\\"\"" > /dev/shm/osync_slave_space_$SCRIPT_PID & - WaitForTaskCompletion $! 0 1800 + child_pid=$! + WaitForTaskCompletion $child_pid 0 1800 SLAVE_SPACE=$(cat /dev/shm/osync_slave_space_$SCRIPT_PID | tail -1 | awk '{print $4}') else SLAVE_SPACE=$(df -P "$SLAVE_SYNC_DIR" | tail -1 | awk '{print $4}') @@ -559,7 +566,8 @@ function WriteLockFiles if [ "$REMOTE_SYNC" == "yes" ] then eval "$SSH_CMD \"$COMMAND_SUDO echo $SCRIPT_PID@$SYNC_ID > \\\"$SLAVE_STATE_DIR/lock\\\"\"" & - WaitForTaskCompletion $! 0 1800 + child_pid=$! + WaitForTaskCompletion $child_pid 0 1800 if [ $? != 0 ] then LogError "Could not set lock on remote slave replica." @@ -609,7 +617,8 @@ function LockDirectories if [ "$REMOTE_SYNC" == "yes" ] then eval "$SSH_CMD \"if [ -f \\\"$SLAVE_STATE_DIR/lock\\\" ]; then cat \\\"$SLAVE_STATE_DIR/lock\\\"; fi\" > /dev/shm/osync_remote_slave_lock_$SCRIPT_PID" & - WaitForTaskCompletion $! 0 1800 + child_pid=$! + WaitForTaskCompletion $child_pid 0 1800 if [ -d /dev/shm/osync_remote_slave_lock_$SCRIPT_PID ] then slave_lock_pid=$(cat /dev/shm/osync_remote_slave_lock_$SCRIPT_PID | cut -d'@' -f1) @@ -657,7 +666,8 @@ function UnlockDirectories if [ "$REMOTE_SYNC" == "yes" ] then eval "$SSH_CMD \"if [ -f \\\"$SLAVE_STATE_DIR/lock\\\" ]; then $COMMAND_SUDO rm \\\"$SLAVE_STATE_DIR/lock\\\"; fi\"" & - WaitForTaskCompletion $! 0 1800 + child_pid=$! + WaitForTaskCompletion $child_pid 0 1800 else if [ -f "$SLAVE_STATE_DIR/lock" ];then rm "$SLAVE_STATE_DIR/lock"; fi fi diff --git a/sync.conf b/sync.conf index 9bccedc..83c2b78 100755 --- a/sync.conf +++ b/sync.conf @@ -8,14 +8,14 @@ SYNC_ID="sync_test" ## Directories to synchronize -MASTER_SYNC_DIR="/home/git/osync/test/dir1" -SLAVE_SYNC_DIR="/home/git/osync/test/dir2" +MASTER_SYNC_DIR="/home/git/osync/test/the separate dir1" +SLAVE_SYNC_DIR="/home/git/osync/test/the separate dir2" ## Create sync directories if they do not exist CREATE_DIRS=yes ## List of directories to exclude in sync on both sides (rsync patterns, wildcards work). Must be relative paths. List is separated by PATH SEPARATOR CHAR defined below (semicolon by default). -RSYNC_EXCLUDE_PATTERN="nosyncdir;otherdir" +RSYNC_EXCLUDE_PATTERN="tmp;archives" ## You might change this separator case in the unholy case that your filename may contain semicolons. Change it then to whatever unholy char you want. PATH_SEPARATOR_CHAR=";" @@ -23,40 +23,40 @@ PATH_SEPARATOR_CHAR=";" MINIMUM_SPACE=10240 ## If enabled, synchronization will be processed with sudo command. See documentation -SUDO_EXEC=yes -## Paranoia option. Don't change this unless you read the documentation and still feel concerned about security issues. +SUDO_EXEC=no +## Paranoia option. Don't change this unless you read the documentation and know what you are doing. RSYNC_EXECUTABLE=rsync ##Remote options (will sync slave through ssh tunnel, needs RSA key. See documentation for remote sync. REMOTE_SYNC=no SSH_RSA_PRIVATE_KEY=~/.ssh/id_rsa -REMOTE_USER=backupmaster -REMOTE_HOST=badministrateur.com -REMOTE_PORT=48884 +REMOTE_USER=backupuser +REMOTE_HOST=your-remote-host.tld +REMOTE_PORT=22 ## ssh compression should be used unless your remote connection is good enough (LAN) SSH_COMPRESSION=yes ## Check for connectivity to remote host before launching remote backup tasks. Be sure the hosts responds to ping. Failing to ping will skip current task. REMOTE_HOST_PING=no ## Check for internet access by pinging one or more 3rd party hosts before remote backup tasks. Leave empty if you don't want this check to be be performed. Failing to ping will skip current task. -REMOTE_3RD_PARTY_HOST="www.kernel.org" +REMOTE_3RD_PARTY_HOST="www.kernel.org www.google.fr" ## Preserve ACLS. Make sure target FS can hold ACLs or you'll get loads of errors. -PRESERVE_ACL=yes +PRESERVE_ACL=no ## Preserve Xattr -PRESERVE_XATTR=yes +PRESERVE_XATTR=no ## Let RSYNC compress file transfers. Do not use if you already enabled SSH compression. RSYNC_COMPRESS=yes -## Maximum execution time for sync process. Soft exec time only generates warning. Hard exec time will generate warning and stop sync process. -SOFT_MAX_EXEC_TIME=30000 -HARD_MAX_EXEC_TIME=36000 +## Maximum execution time (in seconds) for sync process. Soft exec time only generates warning. Hard exec time will generate warning and stop sync process. +SOFT_MAX_EXEC_TIME=18000 +HARD_MAX_EXEC_TIME=43200 ## If the same file exists on both sides, newer version will be used. If both files have the same timestamp but differ, CONFILCT_PREVALANCE sets winner CONFLICT_PREVALANCE=master ## Keep a backup of a file if gets updated from remote side CONFLICT_BACKUP=yes ## Keep multiple backups of a file if it gets updated from remote side. This can be very space consuming -CONFLICT_BACKUP_MULTIPLE=yes +CONFLICT_BACKUP_MULTIPLE=no ## Number of days to keep backups CONFLICT_BACKUP_DAYS=30 @@ -65,25 +65,25 @@ SOFT_DELETE=yes ## Number of days to keep deleted files SOFT_DELETE_DAYS=30 -## Resume an aborted sync task +## Try to resume an aborted sync task RESUME_SYNC=yes -## Number of times to try resuming before initating a new sync +## Number maximum resume tries before initating a new sync RESUME_TRY=2 -## When a dead pidlock exists on slave that does not correspond to master's sync-id, force pidlock removal +## When a pidlock exists on slave that does not correspond to master's sync-id, force pidlock removal. Be carefull with this option if you have multiple masters. FORCE_STRANGER_LOCK_RESUME=no ## List of alert mails separated by spaces -DESTINATION_MAILS="ozy@badministrateur.com" +DESTINATION_MAILS="your@alert.tld" ## Run local commands before and after sync task LOCAL_RUN_BEFORE_CMD="" LOCAL_RUN_AFTER_CMD="" ## Run commands on remote slave befre and after sync task -REMOTE_RUN_BEFORE_CMD="du /var/log" -REMOTE_RUN_AFTER_CMD="du /tmp" +REMOTE_RUN_BEFORE_CMD="" +REMOTE_RUN_AFTER_CMD="" -## Maximum execution time for commands before sync task. Commands get killed if not finished after MAX_EXC_TIME. Set this to 0 to disable killing. +## Maximum execution time (in seconds) for commands before sync task. Commands get killed if not finished after MAX_EXC_TIME. Set this to 0 to disable killing. MAX_EXEC_TIME_PER_CMD_BEFORE=0 -## Maximum execution time for commands after sync task. Commands get killed if not finished after MAX_EXEC_TIME. Set this to 0 to disable killing command. +## Maximum execution time (in seconds) for commands after sync task. Commands get killed if not finished after MAX_EXEC_TIME. Set this to 0 to disable killing command. MAX_EXEC_TIME_PER_CMD_AFTER=0