From d8ff286b32d10c1c9c5eb46e8bb39ee71c87f57c Mon Sep 17 00:00:00 2001 From: deajan Date: Tue, 27 May 2014 00:50:46 +0200 Subject: [PATCH] Code cleanup --- CHANGELOG.md | 8 +- install.sh | 4 +- osync-srv | 3 +- osync.sh | 403 ++++++++++++++++++++++++++++++--------------------- sync.conf | 4 +- 5 files changed, 247 insertions(+), 175 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 773752d..e05180a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ KNOWN ISSUES ------------ - Still need more testing on BSD, MacOSX and Windows MSYS +- Cannot finish sync if one replica contains a directory and the other replica contains a file named the same way (Unix doesn't allow this) UNDER WORK ---------- @@ -23,9 +24,13 @@ UNDER WORK RECENT CHANGES -------------- +- Added some additionnal checks for *BSD and MacOS environments +- Changed /bin/bash to /usr/bin/env bash for sanity on other systems, also check for bash presence before running +- Changed default behavior for quick sync tasks: Will try to resume failed sync tasks once +- Some code cleanup for state filenames and sync action names - Fixed deletion propagation (again). Rsync is definitly not designed to delete a list of files / folders. Rsync replaced by rm function which downloads deletion list to remote system. - Added path detection for exclude list file -- Added a simple init script working for RHEL / CentOS and an install script +- Added a simple init script and an install script - Fixed an issue with MacOSX using rsync -E differently than other *nix (Thanks to Pierre Clement) - Multislave asynchronous task support (Thanks to Ulrich Norbisrath) - This breaks compat with elder osync runs. Add the SYNC_ID suffix to elder state files to keep deleted file information. @@ -39,7 +44,6 @@ RECENT CHANGES - Fixed a nasty bug preventing writing lock files on remote system as superuser - Gzipped logs are now deleted once sent - Fixed some typos (thanks to Pavel Kiryukhin) -- Added a simple RHEL / CentOS compatible init script - Fixed a bug with double trailing slashes in certain sceanrios - Sync execution don't fails anymore if files vanish during execution, also vanished files get logged - Add eventual "comm -23" replacement by "grep -F -x -v -f" to enhance compatibility with other platforms (comm is still much faster than grep, so we keep it) diff --git a/install.sh b/install.sh index 8b38309..4b3a378 100755 --- a/install.sh +++ b/install.sh @@ -1,4 +1,6 @@ -#!/bin/bash +#!/usr/bin/env bash + +SCRIPT_BUILD=2605201401 ## Osync daemon install script ## Tested on RHEL / CentOS 6 diff --git a/osync-srv b/osync-srv index 1afe07c..df59df4 100755 --- a/osync-srv +++ b/osync-srv @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # osync two way directory sync tool # @@ -12,6 +12,7 @@ progpath=/usr/local/bin confdir=/etc/osync pidfile=/var/run/$prog lockfile=/var/lock/subsys/$prog +SCRIPT_BUILD=2605201401 if [ ! -f $progpath/$progexec ] && [ ! -f $progexec ] then diff --git a/osync.sh b/osync.sh index 426cfe9..52f79ee 100755 --- a/osync.sh +++ b/osync.sh @@ -1,10 +1,16 @@ -#!/bin/bash +#!/usr/bin/env bash PROGRAM="Osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(L) 2013-2014 by Orsiris \"Ozy\" de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" -PROGRAM_VERSION=0.99preRC3 -PROGRAM_BUILD=2505201406 +PROGRAM_VERSION=0.99RC3 +PROGRAM_BUILD=2605201403 + +if ! type -p "$BASH" > /dev/null +then + echo "Please run this script only with bash shell. Tested on bash >= 3.2" + exit 127 +fi ## allow debugging from command line with preceding ocsync with DEBUG=yes if [ ! "$DEBUG" == "yes" ] @@ -276,7 +282,7 @@ function CheckEnvironment fi } -function GetOperatingSystem +function GetLocalOS { LOCAL_OS_VAR=$(uname -spio 2>&1) if [ $? != 0 ] @@ -292,8 +298,8 @@ function GetOperatingSystem *"Linux"*) LOCAL_OS="Linux" ;; - *"FreeBSD"*) - LOCAL_OS="FreeBSD" + *"BSD"*) + LOCAL_OS="BSD" ;; *"MINGW32"*) LOCAL_OS="msys" @@ -307,7 +313,10 @@ function GetOperatingSystem ;; esac LogDebug "Local OS: [$LOCAL_OS_VAR]." +} +function GetRemoteOS +{ if [ "$REMOTE_SYNC" == "yes" ] then CheckConnectivity3rdPartyHosts @@ -341,8 +350,8 @@ function GetOperatingSystem *"Linux"*) REMOTE_OS="Linux" ;; - *"FreeBSD"*) - REMOTE_OS="FreeBSD" + *"BSD"*) + REMOTE_OS="BSD" ;; *"MINGW32"*) REMOTE_OS="msys" @@ -371,11 +380,11 @@ function WaitForTaskCompletion SECONDS_BEGIN=$SECONDS if [ "$LOCAL_OS" == "msys" ] then - PROCESS_TEST="ps -a | awk '{\$1=\$1}\$1' | awk '{print \$1}' | grep $1" + PROCESS_TEST_CMD="ps -a | awk '{\$1=\$1}\$1' | awk '{print \$1}' | grep $1" else - PROCESS_TEST="ps -p$1" + PROCESS_TEST_CMD="ps -p$1" fi - while eval $PROCESS_TEST > /dev/null + while eval $PROCESS_TEST_CMD > /dev/null do Spinner EXEC_TIME=$(($SECONDS - $SECONDS_BEGIN)) @@ -557,12 +566,7 @@ function CheckConnectivityRemoteHost { if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_SYNC" != "no" ] then - if [ "$LOCAL_OS" == "msys" ] - then - ping -n 2 $REMOTE_HOST > /dev/null 2>&1 - else - ping -c 2 $REMOTE_HOST > /dev/null 2>&1 - fi + $PING_CMD $REMOTE_HOST > /dev/null 2>&1 if [ $? != 0 ] then LogError "Cannot ping $REMOTE_HOST" @@ -580,12 +584,7 @@ function CheckConnectivity3rdPartyHosts IFS=$' \t\n' for i in $REMOTE_3RD_PARTY_HOSTS do - if [ "$LOCAL_OS" == "msys" ] - then - ping -n 2 $i > /dev/null 2>&1 - else - ping -c 2 $i > /dev/null 2>&1 - fi + $PING_CMD $i > /dev/null 2>&1 if [ $? != 0 ] then Log "Cannot ping 3rd party host $i" @@ -917,11 +916,11 @@ function UnlockDirectories ## The same applies for slave sync dir..............................................T.H.I.S..I.S..A..P.R.O.G.R.A.M.M.I.N.G..N.I.G.H.T.M.A.R.E -## tree_list(replica_path, tree_file, current_action) Creates a list of files in replica_path and stores it's action in $STATE_DIR/last-action +## tree_list(replica_path, replica type, tree_filename) Creates a list of files in replica_path for replica type (master/slave) in filename $3 function tree_list { Log "Creating $2 replica file list [$1]." - if [ "$REMOTE_SYNC" == "yes" ] && [[ "$2" == "slave"* ]] + if [ "$REMOTE_SYNC" == "yes" ] && [ "$2" == "slave" ] then CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost @@ -939,72 +938,41 @@ function tree_list ## Retval 24 = some files vanished while creating list if ([ $retval == 0 ] || [ $retval == 24 ]) && [ -f $RUN_DIR/osync_$2_$SCRIPT_PID ] then - if [ $dryrun -eq 1 ] - then - mv $RUN_DIR/osync_$2_$SCRIPT_PID "$MASTER_STATE_DIR/dry-$2-$SYNC_ID" - else - mv $RUN_DIR/osync_$2_$SCRIPT_PID "$MASTER_STATE_DIR/$2-$SYNC_ID" - fi - echo "$3.success" > "$MASTER_LAST_ACTION" - + mv $RUN_DIR/osync_$2_$SCRIPT_PID "$MASTER_STATE_DIR/$2$3" + return $? else LogError "Cannot create replica file list." - echo "$3.fail" > "$MASTER_LAST_ACTION" - exit 1 + exit $retval fi } -# delete_list(replica): Creates a list of files vanished from last run on replica $1 +# delete_list(replica, tree-file-after, tree-file-current, deleted-list-file): Creates a list of files vanished from last run on replica $1 (master/slave) function delete_list { Log "Creating $1 replica deleted file list." if [ -f "$MASTER_STATE_DIR/$1-tree-after-$SYNC_ID" ] then - if [ $dryrun -eq 1 ] + ## Same functionnality, comm is much faster than grep but is not available on every platform + if type -p comm > /dev/null 2>&1 then - ## Same functionnality, comm is much faster than grep but is not available on every platform - if type -p comm > /dev/null 2>&1 - then - cmd="comm -23 \"$MASTER_STATE_DIR/$1-tree-after-$SYNC_ID\" \"$MASTER_STATE_DIR/dry-$1-tree-current-$SYNC_ID\" > \"$MASTER_STATE_DIR/dry-$1-deleted-list-$SYNC_ID\"" - else - ## The || : forces the command to have a good result - cmd="grep -F -x -v -f \"$MASTER_STATE_DIR/dry-$1-tree-current-$SYNC_ID\" \"$MASTER_STATE_DIR/$1-tree-after-$SYNC_ID\" || : > \"$MASTER_STATE_DIR/dry-$1-deleted-list-$SYNC_ID\"" - fi + cmd="comm -23 \"$MASTER_STATE_DIR/$1$2\" \"$MASTER_STATE_DIR/$1$3\" > \"$MASTER_STATE_DIR/$1$4\"" else - if type -p comm > /dev/null 2>&1 - then - cmd="comm -23 \"$MASTER_STATE_DIR/$1-tree-after-$SYNC_ID\" \"$MASTER_STATE_DIR/$1-tree-current-$SYNC_ID\" > \"$MASTER_STATE_DIR/$1-deleted-list-$SYNC_ID\"" - else - cmd="grep -F -x -v -f \"$MASTER_STATE_DIR/$1-tree-current-$SYNC_ID\" \"$MASTER_STATE_DIR/$1-tree-after-$SYNC_ID\" || : > \"$MASTER_STATE_DIR/$1-deleted-list-$SYNC_ID\"" - fi + ## The || : forces the command to have a good result + cmd="grep -F -x -v -f \"$MASTER_STATE_DIR/$1$3\" \"$MASTER_STATE_DIR/$1$2\" || : > \"$MASTER_STATE_DIR/$1$4\"" fi LogDebug "CMD: $cmd" eval $cmd - echo "$1-replica-deleted-list.success" > "$MASTER_LAST_ACTION" + return $? else - if [ $dryrun -eq 1 ] - then - touch "$MASTER_STATE_DIR/dry-$1-deleted-list-$SYNC_ID" - else - touch "$MASTER_STATE_DIR/$1-deleted-list-$SYNC_ID" - fi - echo "$1-replica-deleted-list.empty" > "$MASTER_LAST_ACTION" + touch "$MASTER_STATE_DIR/$1$4" + return $? fi } -# sync_update(source replica, destination replica) +# sync_update(source replica, destination replica, delete_list_filename) function sync_update { - if [ $dryrun -eq 1 ] - then - delfname1="dry-$1-deleted-list-$SYNC_ID" - delfname2="dry-$2-deleted-list-$SYNC_ID" - else - delfname1="$1-deleted-list-$SYNC_ID" - delfname2="$2-deleted-list-$SYNC_ID" - fi - Log "Updating $2 replica." if [ "$1" == "master" ] then @@ -1027,12 +995,12 @@ function sync_update CheckConnectivityRemoteHost if [ "$1" == "master" ] then - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$MASTER_STATE_DIR/$delfname1\" --exclude-from=\"$MASTER_STATE_DIR/$delfname2\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID 2>&1 &" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$MASTER_STATE_DIR/$1$3\" --exclude-from=\"$MASTER_STATE_DIR/$2$3\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID 2>&1 &" else - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$MASTER_STATE_DIR/$delfname1\" --exclude-from=\"$MASTER_STATE_DIR/$delfname2\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID 2>&1 &" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$MASTER_STATE_DIR/$2$3\" --exclude-from=\"$MASTER_STATE_DIR/$1$3\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID 2>&1 &" fi else - rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$MASTER_STATE_DIR/$delfname1\" --exclude-from=\"$MASTER_STATE_DIR/$delfname2\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID 2>&1 &" + rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$MASTER_STATE_DIR/$1$3\" --exclude-from=\"$MASTER_STATE_DIR/$2$3\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID 2>&1 &" fi LogDebug "RSYNC_CMD: $rsync_cmd" eval "$rsync_cmd" @@ -1051,11 +1019,10 @@ function sync_update then LogError "Rsync output:\n$(cat $RUN_DIR/osync_update_$2_replica_$SCRIPT_PID)" fi - echo "update-$2-replica.fail" > "$MASTER_LAST_ACTION" - exit 1 + exit $retval else Log "Updating $2 replica succeded." - echo "update-$2-replica.success" > "$MASTER_LAST_ACTION" + return 0 fi } @@ -1214,7 +1181,7 @@ ENDSSH } -# delete_propagation(replica name) +# delete_propagation(replica name, deleted_list_filename) # replica name = "master" / "slave" function deletion_propagation { @@ -1224,40 +1191,25 @@ function deletion_propagation then REPLICA_DIR="$MASTER_SYNC_DIR" DELETE_DIR="$MASTER_DELETE_DIR" - if [ $dryrun -eq 1 ] - then - DELETION_FILE_LIST="dry-slave-deleted-list-$SYNC_ID" - else - DELETION_FILE_LIST="slave-deleted-list-$SYNC_ID" - fi - _delete_local "$REPLICA_DIR" "$DELETION_FILE_LIST" "$DELETE_DIR" & + _delete_local "$REPLICA_DIR" "slave$2" "$DELETE_DIR" & child_pid=$! WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME retval=$? - if [ $retval == 0 ] + if [ $retval != 0 ] then - echo "delete-propagation-$1.success" > "$MASTER_LAST_ACTION" - else LogError "Deletion on replica $1 failed." - echo "delete-propagation-$1.fail" > "$MASTER_LAST_ACTION" exit 1 fi else REPLICA_DIR="$SLAVE_SYNC_DIR" DELETE_DIR="$SLAVE_DELETE_DIR" - if [ $dryrun -eq 1 ] - then - DELETION_FILE_LIST="dry-master-deleted-list-$SYNC_ID" - else - DELETION_FILE_LIST="master-deleted-list-$SYNC_ID" - fi if [ "$REMOTE_SYNC" == "yes" ] then - _delete_remote "$REPLICA_DIR" "$DELETION_FILE_LIST" "$DELETE_DIR" & + _delete_remote "$REPLICA_DIR" "master$2" "$DELETE_DIR" & else - _delete_local "$REPLICA_DIR" "$DELETION_FILE_LIST" "$DELETE_DIR" & + _delete_local "$REPLICA_DIR" "master$2" "$DELETE_DIR" & fi child_pid=$! WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME @@ -1268,27 +1220,33 @@ function deletion_propagation then Log "Remote:\n$(cat $RUN_DIR/osync_remote_deletion_$SCRIPT_PID)" fi - echo "delete-propagation-$1.success" > "$MASTER_LAST_ACTION" + return $retval else LogError "Deletion on remote system failed." if [ -f $RUN_DIR/osync_remote_deletion_$SCRIPT_PID ] then LogError "Remote:\n$(cat $RUN_DIR/osync_remote_deletion_$SCRIPT_PID)" fi - echo "delete-propagation-$1.fail" > "$MASTER_LAST_ACTION" - exit 1 + exit 1 fi fi } -###### Sync function in 10 steps (functions above) +###### Sync function in 5 steps of each 2 runs (functions above) +###### +###### Step 1: Create current tree list for master and slave replicas (Steps 1M and 1S) +###### Step 2: Create deleted file list for master and slave replicas (Steps 2M and 2S) +###### Step 3: Update master and slave replicas (Steps 3M and 3S, order depending on conflict prevalence) +###### Step 4: Deleted file propagation to master and slave replicas (Steps 4M and 4S) +###### Step 5: Create after run tree list for master and slave replicas (Steps 5M and 5S) + function Sync { Log "Starting synchronization task." CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - if [ -f "$MASTER_LAST_ACTION" ] && [ "$RESUME_SYNC" == "yes" ] + if [ -f "$MASTER_LAST_ACTION" ] && [ "$RESUME_SYNC" != "no" ] then resume_sync=$(cat "$MASTER_LAST_ACTION") if [ -f "$MASTER_RESUME_COUNT" ] @@ -1302,21 +1260,15 @@ function Sync then if [ "$resume_sync" != "sync.success" ] then - Log "WARNING: Trying to resume aborted osync execution on $(stat --format %y "$MASTER_LAST_ACTION") at task [$resume_sync]. [$resume_count] previous tries." - if [ $dryrun -ne 1 ] - then - echo $(($resume_count+1)) > "$MASTER_RESUME_COUNT" - fi + Log "WARNING: Trying to resume aborted osync execution on $($STAT_CMD "$MASTER_LAST_ACTION") at task [$resume_sync]. [$resume_count] previous tries." + echo $(($resume_count+1)) > "$MASTER_RESUME_COUNT" else resume_sync=none fi else Log "Will not resume aborted osync execution. Too much resume tries [$resume_count]." - if [ $dryrun -ne 1 ] - then - echo "noresume" > "$MASTER_LAST_ACTION" - echo "0" > "$MASTER_RESUME_COUNT" - fi + echo "noresume" > "$MASTER_LAST_ACTION" + echo "0" > "$MASTER_RESUME_COUNT" resume_sync=none fi else @@ -1330,82 +1282,151 @@ function Sync if [ "$resume_sync" == "none" ] || [ "$resume_sync" == "noresume" ] || [ "$resume_sync" == "master-replica-tree.fail" ] then #master_tree_current - tree_list "$MASTER_SYNC_DIR" master-tree-current master-replica-tree + tree_list "$MASTER_SYNC_DIR" master "$TREE_CURRENT_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[0]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNC_ACTION[0]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "master-replica-tree.success" ] || [ "$resume_sync" == "slave-replica-tree.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[0]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[1]}.fail" ] then #slave_tree_current - tree_list "$SLAVE_SYNC_DIR" slave-tree-current slave-replica-tree + tree_list "$SLAVE_SYNC_DIR" slave "$TREE_CURRENT_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[1]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNC_ACTION[1]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "slave-replica-tree.success" ] || [ "$resume_sync" == "master-replica-deleted-list.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[1]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[2]}.fail" ] then - delete_list master + delete_list master "$TREE_AFTER_FILENAME" "$TREE_CURRENT_FILENAME" "$DELETED_LIST_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[2]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNc_ACTION[2]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "master-replica-deleted-list.success" ] || [ "$resume_sync" == "slave-replica-deleted-list.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[2]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.fail" ] then - delete_list slave + delete_list slave "$TREE_AFTER_FILENAME" "$TREE_CURRENT_FILENAME" "$DELETED_LIST_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[3]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNC_ACTION[3]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "slave-replica-deleted-list.success" ] || [ "$resume_sync" == "update-master-replica.fail" ] || [ "$resume_sync" == "update-slave-replica.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ] then if [ "$CONFLICT_PREVALANCE" != "master" ] then - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "slave-replica-deleted-list.success" ] || [ "$resume_sync" == "update-master-replica.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ] then - sync_update slave master + sync_update slave master "$DELETED_LIST_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[4]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNC_ACTION[4]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "update-master-replica.success" ] || [ "$resume_sync" == "update-slave-replica.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ] then - sync_update master slave + sync_update master slave "$DELETED_LIST_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[5]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNC_ACTION[5]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi else - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "slave-replica-deleted-list.success" ] || [ "$resume_sync" == "update-slave-replica.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ] then - sync_update master slave + sync_update master slave "$DELETED_LIST_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[5]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNC_ACTION[5]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "update-slave-replica.success" ] || [ "$resume_sync" == "update-master-replica.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ] then - sync_update slave master + sync_update slave master "$DELETED_LIST_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[4]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNC_ACTION[4]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi fi fi - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "update-slave-replica.success" ] || [ "$resume_sync" == "update-master-replica.success" ] || [ "$resume_sync" == "delete-propagation-slave.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.fail" ] then - deletion_propagation slave + deletion_propagation slave "$DELETED_LIST_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[6]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNC_ACTION[6]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "delete-propagation-slave.success" ] || [ "$resume_sync" == "delete-propagation-master.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[6]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[7]}.fail" ] then - deletion_propagation master + deletion_propagation master "$DELETED_LIST_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[7]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNC_ACTION[7]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "delete-propagation-master.success" ] || [ "$resume_sync" == "master-replica-tree-after.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[7]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[8]}.fail" ] then #master_tree_after - tree_list "$MASTER_SYNC_DIR" master-tree-after master-replica-tree-after + tree_list "$MASTER_SYNC_DIR" master "$TREE_AFTER_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[8]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNC_ACTION[8]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi - if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "master-replica-tree-after.success" ] || [ "$resume_sync" == "slave-replica-tree-after.fail" ] + if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[8]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[9]}.fail" ] then #slave_tree_after - tree_list "$SLAVE_SYNC_DIR" slave-tree-after slave-replica-tree-after + tree_list "$SLAVE_SYNC_DIR" slave "$TREE_AFTER_FILENAME" + if [ $? == 0 ] + then + echo "${SYNC_ACTION[9]}.success" > "$MASTER_LAST_ACTION" + else + echo "${SYNC_ACTION[9]}.fail" > "$MASTER_LAST_ACTION" + fi resume_sync="resumed" fi Log "Finished synchronization task." - echo "sync.success" > "$MASTER_LAST_ACTION" + echo "${SYNC_ACTION[10]}" > "$MASTER_LAST_ACTION" - if [ $dryrun -ne 1 ] - then - echo "0" > "$MASTER_RESUME_COUNT" - fi + echo "0" > "$MASTER_RESUME_COUNT" } function SoftDelete @@ -1599,21 +1620,6 @@ function Init MAIL_ALERT_MSG="Warning: Execution of osync instance $OSYNC_ID (pid $SCRIPT_PID) as $LOCAL_USER@$LOCAL_HOST produced errors on $(date)." - ## If running Msys, find command of windows is used instead of msys one - if [ "$LOCAL_OS" == "msys" ] - then - FIND_CMD=$(dirname $BASH)/find - else - FIND_CMD=find - fi - - if [ "$REMOTE_OS" == "msys" ] - then - REMOTE_FIND_CMD=$(dirname $BASH)/find - else - REMOTE_FIND_CMD=find - fi - ## Test if slave dir is a ssh uri, and if yes, break it down it its values if [ "${SLAVE_SYNC_DIR:0:6}" == "ssh://" ] then @@ -1653,15 +1659,6 @@ function Init MASTER_LOCK="$MASTER_STATE_DIR/lock" SLAVE_LOCK="$SLAVE_STATE_DIR/lock" - if [ $dryrun -eq 1 ] - then - MASTER_LAST_ACTION="$MASTER_STATE_DIR/dry-last-action-$SYNC_ID" - MASTER_RESUME_COUNT="$MASTER_STATE_DIR/dry-resume-count-$SYNC_ID" - else - MASTER_LAST_ACTION="$MASTER_STATE_DIR/last-action-$SYNC_ID" - MASTER_RESUME_COUNT="$MASTER_STATE_DIR/resume-count-$SYNC_ID" - fi - ## Working directories to keep backups of updated / deleted files MASTER_BACKUP_DIR="$OSYNC_DIR/backups" MASTER_DELETE_DIR="$OSYNC_DIR/deleted" @@ -1713,11 +1710,6 @@ function Init ## Set rsync default arguments RSYNC_ARGS="-rlptgoD" - ## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability) - if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ] - then - RSYNC_ARGS=$RSYNC_ARGS"E" - fi if [ "$PRESERVE_ACL" == "yes" ] then RSYNC_ARGS=$RSYNC_ARGS"A" @@ -1777,6 +1769,76 @@ function Init RsyncExcludePattern ## Add Rsync exclude from file RsyncExcludeFrom + + ## Filenames for state files + if [ $dryrun -eq 1 ] + then + dry_suffix="-dry" + fi + + TREE_CURRENT_FILENAME="-tree-current-$SYNC_ID$dry_suffix" + TREE_AFTER_FILENAME="-tree-after-$SYNC_ID$dry_suffix" + DELETED_LIST_FILENAME="-deleted-list$SYNC_ID$dry_suffix" + MASTER_LAST_ACTION="$MASTER_STATE_DIR/last-action-$SYNC_ID$dry_suffix" + MASTER_RESUME_COUNT="$MASTER_STATE_DIR/resume-count-$SYNC_ID$dry_suffix" + + ## Sync function actions (0-9) + SYNC_ACTION=( + 'master-replica-tree' + 'slave-replica-tree' + 'master-deleted-list' + 'slave-deleted-list' + 'update-master-replica' + 'update-slave-replica' + 'delete-propagation-slave' + 'delete-propagation-master' + 'master-replica-tree-after' + 'slave-replica-tree-after' + 'sync.success' + ) +} + +function InitLocalOSSettings +{ + ## If running Msys, find command of windows is used instead of msys one + if [ "$LOCAL_OS" == "msys" ] + then + FIND_CMD=$(dirname $BASH)/find + else + FIND_CMD=find + fi + + ## Stat command has different syntax on Linux and FreeBSD/MacOSX + if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ] + then + STAT_CMD="stat -f \"%Sm\"" + else + STAT_CMD="stat --format %y" + fi + + ## Ping command has different syntax on Msys and others + if [ "$LOCAL_OS" == "msys" ] + then + PING_CMD="ping -n 2" + else + PING_CMD="ping -c 2 -i .2" + fi +} + +function InitRemoteOSSettings +{ + ## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability) + if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ] + then + RSYNC_ARGS=$RSYNC_ARGS"E" + fi + + if [ "$REMOTE_OS" == "msys" ] + then + REMOTE_FIND_CMD=$(dirname $BASH)/find + else + REMOTE_FIND_CMD=find + fi } function Main @@ -1933,6 +1995,8 @@ done # Remove leading space if there is one opts="${opts# *}" +GetLocalOS +InitLocalOSSettings CheckEnvironment if [ $? == 0 ] then @@ -1956,6 +2020,8 @@ then LoadConfigFile "$ConfigFile" fi Init + GetRemoteOS + InitRemoteOSSettings if [ $sync_on_changes -eq 1 ] then SyncOnChanges @@ -1965,7 +2031,6 @@ then Log "$DRY_WARNING $DATE - $PROGRAM $PROGRAM_VERSION script begin." Log "-------------------------------------------------------------" Log "Sync task [$SYNC_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" - GetOperatingSystem if [ $no_maxtime -eq 1 ] then SOFT_MAX_EXEC_TIME=0 diff --git a/sync.conf b/sync.conf index 79d0247..5b97ada 100755 --- a/sync.conf +++ b/sync.conf @@ -1,8 +1,8 @@ -#!/bin/bash +#!/usr/bin/env bash ###### Osync - Rsync based two way sync engine with fault tolerance ###### (L) 2013-2014 by Orsiris "Ozy" de Jong (www.netpower.fr) -###### Config file rev 2505201402 +###### Config file rev 2605201401 ## ---------- GENERAL OPTIONS