Archives
/
osync
mirror of https://github.com/deajan/osync
2
0
Fork 0
Code Issues Releases Wiki Activity

Added rsync include patterns

Browse Source
pull/48/merge
deajan 9 years ago
parent 9c2df9990f
commit 4398c02f9d
5 changed files with 145 additions and 79 deletions
Show Stats Download Patch File Download Diff File

73
dev/debug_osync.sh
Unescape Escape View File

@ -4,7 +4,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(L) 2013-2015 by Orsiris \"Ozy\" de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.1-pre
PROGRAM_BUILD=2015111901
PROGRAM_BUILD=2015112702
IS_STABLE=no
FUNC_BUILD=2015111901
@ -83,6 +83,7 @@ function Dummy {
sleep .1
}
#__FUNC:Logger
function _Logger {
local svalue="${1}" # What to log to screen
local lvalue="${2:-$svalue}" # What to log to logfile, defaults to screen value
@ -135,6 +136,7 @@ function Logger {
_Logger "$prefix$value"
fi
}
#__ENDFUNC
# Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X
function KillChilds {
@ -1132,12 +1134,14 @@ function CheckDiskSpace {
fi
}
function RsyncExcludePattern {
__CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
function RsyncPatternsAdd {
local pattern="${1}"
__CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
# Disable globbing so wildcards from exclusions do not get expanded
set -f
rest="$RSYNC_EXCLUDE_PATTERN"
rest="$pattern"
while [ -n "$rest" ]
do
# Take the string until first occurence until $PATH_SEPARATOR_CHAR
@ -1149,31 +1153,50 @@ function RsyncExcludePattern {
# Cut everything before the first occurence of $PATH_SEPARATOR_CHAR
rest=${rest#*$PATH_SEPARATOR_CHAR}
fi
if [ "$RSYNC_EXCLUDE" == "" ]; then
RSYNC_EXCLUDE="--exclude=\"$str\""
if [ "$RSYNC_PATTERNS" == "" ]; then
RSYNC_PATTERNS="--exclude=\"$str\""
else
RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$str\""
RSYNC_PATTERNS="$RSYNC_PATTERNS --exclude=\"$str\""
fi
done
set +f
}
function RsyncExcludeFrom {
__CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
function RsyncPatternsFromAdd {
local pattern_from="${1}"
__CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
if [ ! "$RSYNC_EXCLUDE_FROM" == "" ]; then
if [ ! "$pattern_from" == "" ]; then
## Check if the exclude list has a full path, and if not, add the config file path if there is one
if [ "$(basename $RSYNC_EXCLUDE_FROM)" == "$RSYNC_EXCLUDE_FROM" ]; then
RSYNC_EXCLUDE_FROM=$(dirname $ConfigFile)/$RSYNC_EXCLUDE_FROM
if [ "$(basename $pattern_from)" == "$pattern_from" ]; then
pattern_from=$(dirname $ConfigFile)/$pattern_ffrom
fi
if [ -e "$RSYNC_EXCLUDE_FROM" ]; then
RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude-from=\"$RSYNC_EXCLUDE_FROM\""
if [ -e "$pattern_from" ]; then
RSYNC_PATTERNS="$RSYNC_PATTERNS --exclude-from=\"$pattern_from\""
fi
fi
}
function RsyncPatterns {
__CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
if [ "$RSYNC_PATTERN_ORDER" == "exclude" ]; then
RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN"
RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM"
RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN"
RsyncPatternsFromAdd "$RSYNC_INCLUDE_FROM"
elif [ "$RSYNC_PATTERN_ORDER" == "include" ]; then
RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN"
RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM"
RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN"
RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM"
else
Logger "Bogus RSYNC_PATTERN_ORDER in config file" "WARN"
fi
}
function _WriteLockFilesLocal {
local lockfile="${1}"
__CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
@ -1372,9 +1395,9 @@ function tree_list {
if [ "$REMOTE_OPERATION" == "yes" ] && [ "$replica_type" == "target" ]; then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escaped_replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escaped_replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --list-only \"$replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS --list-only \"$replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
## Redirect commands stderr here to get rsync stderr output in logfile
@ -1455,12 +1478,12 @@ function sync_update {
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
if [ "$source_replica" == "initiator" ]; then
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
fi
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd"
@ -2087,7 +2110,7 @@ function Init {
if [ "$PARTIAL" == "yes" ]; then
SYNC_OPTS=$SYNC_OPTS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$PARTIAL_DIR\""
RSYNC_PATTERNS="$RSYNC_PATTERNS --exclude=\"$PARTIAL_DIR\""
fi
## Conflict options
@ -2103,10 +2126,8 @@ function Init {
TARGET_BACKUP=
fi
## Add Rsync exclude patterns
RsyncExcludePattern
## Add Rsync exclude from file
RsyncExcludeFrom
## Add Rsync include / exclude patterns
RsyncPatterns
## Filenames for state files
if [ $_DRYRUN -eq 1 ]; then
@ -2209,7 +2230,7 @@ function SyncOnChanges {
fi
Logger "#### Monitoring now." "NOTICE"
inotifywait --exclude $OSYNC_DIR $RSYNC_EXCLUDE -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" &
inotifywait --exclude $OSYNC_DIR $RSYNC_PATTERNS -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" &
OSYNC_SUB_PID=$!
wait $OSYNC_SUB_PID
retval=$?

71
dev/n_osync.sh
Unescape Escape View File

@ -4,7 +4,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(L) 2013-2015 by Orsiris \"Ozy\" de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.1-pre
PROGRAM_BUILD=2015111901
PROGRAM_BUILD=2015112702
IS_STABLE=no
source "./ofunctions.sh"
@ -246,12 +246,14 @@ function CheckDiskSpace {
fi
}
function RsyncExcludePattern {
__CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
function RsyncPatternsAdd {
local pattern="${1}"
__CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
# Disable globbing so wildcards from exclusions do not get expanded
set -f
rest="$RSYNC_EXCLUDE_PATTERN"
rest="$pattern"
while [ -n "$rest" ]
do
# Take the string until first occurence until $PATH_SEPARATOR_CHAR
@ -263,31 +265,50 @@ function RsyncExcludePattern {
# Cut everything before the first occurence of $PATH_SEPARATOR_CHAR
rest=${rest#*$PATH_SEPARATOR_CHAR}
fi
if [ "$RSYNC_EXCLUDE" == "" ]; then
RSYNC_EXCLUDE="--exclude=\"$str\""
if [ "$RSYNC_PATTERNS" == "" ]; then
RSYNC_PATTERNS="--exclude=\"$str\""
else
RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$str\""
RSYNC_PATTERNS="$RSYNC_PATTERNS --exclude=\"$str\""
fi
done
set +f
}
function RsyncExcludeFrom {
__CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
function RsyncPatternsFromAdd {
local pattern_from="${1}"
__CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
if [ ! "$RSYNC_EXCLUDE_FROM" == "" ]; then
if [ ! "$pattern_from" == "" ]; then
## Check if the exclude list has a full path, and if not, add the config file path if there is one
if [ "$(basename $RSYNC_EXCLUDE_FROM)" == "$RSYNC_EXCLUDE_FROM" ]; then
RSYNC_EXCLUDE_FROM=$(dirname $ConfigFile)/$RSYNC_EXCLUDE_FROM
if [ "$(basename $pattern_from)" == "$pattern_from" ]; then
pattern_from=$(dirname $ConfigFile)/$pattern_ffrom
fi
if [ -e "$RSYNC_EXCLUDE_FROM" ]; then
RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude-from=\"$RSYNC_EXCLUDE_FROM\""
if [ -e "$pattern_from" ]; then
RSYNC_PATTERNS="$RSYNC_PATTERNS --exclude-from=\"$pattern_from\""
fi
fi
}
function RsyncPatterns {
__CheckArguments 0 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
if [ "$RSYNC_PATTERN_ORDER" == "exclude" ]; then
RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN"
RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM"
RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN"
RsyncPatternsFromAdd "$RSYNC_INCLUDE_FROM"
elif [ "$RSYNC_PATTERN_ORDER" == "include" ]; then
RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN"
RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM"
RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN"
RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM"
else
Logger "Bogus RSYNC_PATTERN_ORDER in config file" "WARN"
fi
}
function _WriteLockFilesLocal {
local lockfile="${1}"
__CheckArguments 1 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
@ -486,9 +507,9 @@ function tree_list {
if [ "$REMOTE_OPERATION" == "yes" ] && [ "$replica_type" == "target" ]; then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escaped_replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escaped_replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --list-only \"$replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS --list-only \"$replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
## Redirect commands stderr here to get rsync stderr output in logfile
@ -569,12 +590,12 @@ function sync_update {
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
if [ "$source_replica" == "initiator" ]; then
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
fi
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd"
@ -1201,7 +1222,7 @@ function Init {
if [ "$PARTIAL" == "yes" ]; then
SYNC_OPTS=$SYNC_OPTS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$PARTIAL_DIR\""
RSYNC_PATTERNS="$RSYNC_PATTERNS --exclude=\"$PARTIAL_DIR\""
fi
## Conflict options
@ -1217,10 +1238,8 @@ function Init {
TARGET_BACKUP=
fi
## Add Rsync exclude patterns
RsyncExcludePattern
## Add Rsync exclude from file
RsyncExcludeFrom
## Add Rsync include / exclude patterns
RsyncPatterns
## Filenames for state files
if [ $_DRYRUN -eq 1 ]; then
@ -1323,7 +1342,7 @@ function SyncOnChanges {
fi
Logger "#### Monitoring now." "NOTICE"
inotifywait --exclude $OSYNC_DIR $RSYNC_EXCLUDE -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" &
inotifywait --exclude $OSYNC_DIR $RSYNC_PATTERNS -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" &
OSYNC_SUB_PID=$!
wait $OSYNC_SUB_PID
retval=$?

2
dev/ofunctions.sh
Unescape Escape View File

@ -74,6 +74,7 @@ function Dummy {
sleep .1
}
#__FUNC:Logger
function _Logger {
local svalue="${1}" # What to log to screen
local lvalue="${2:-$svalue}" # What to log to logfile, defaults to screen value
@ -126,6 +127,7 @@ function Logger {
_Logger "$prefix$value"
fi
}
#__ENDFUNC
# Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X
function KillChilds {

68
osync.sh
Unescape Escape View File

@ -4,7 +4,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(L) 2013-2015 by Orsiris \"Ozy\" de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.1-pre
PROGRAM_BUILD=2015111901
PROGRAM_BUILD=2015112702
IS_STABLE=no
FUNC_BUILD=2015111901
@ -78,6 +78,7 @@ function Dummy {
sleep .1
}
#__FUNC:Logger
function _Logger {
local svalue="${1}" # What to log to screen
local lvalue="${2:-$svalue}" # What to log to logfile, defaults to screen value
@ -125,6 +126,7 @@ function Logger {
_Logger "$prefix$value"
fi
}
#__ENDFUNC
# Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X
function KillChilds {
@ -1021,11 +1023,13 @@ function CheckDiskSpace {
fi
}
function RsyncExcludePattern {
function RsyncPatternsAdd {
local pattern="${1}"
# Disable globbing so wildcards from exclusions do not get expanded
set -f
rest="$RSYNC_EXCLUDE_PATTERN"
rest="$pattern"
while [ -n "$rest" ]
do
# Take the string until first occurence until $PATH_SEPARATOR_CHAR
@ -1037,30 +1041,48 @@ function RsyncExcludePattern {
# Cut everything before the first occurence of $PATH_SEPARATOR_CHAR
rest=${rest#*$PATH_SEPARATOR_CHAR}
fi
if [ "$RSYNC_EXCLUDE" == "" ]; then
RSYNC_EXCLUDE="--exclude=\"$str\""
if [ "$RSYNC_PATTERNS" == "" ]; then
RSYNC_PATTERNS="--exclude=\"$str\""
else
RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$str\""
RSYNC_PATTERNS="$RSYNC_PATTERNS --exclude=\"$str\""
fi
done
set +f
}
function RsyncExcludeFrom {
function RsyncPatternsFromAdd {
local pattern_from="${1}"
if [ ! "$RSYNC_EXCLUDE_FROM" == "" ]; then
if [ ! "$pattern_from" == "" ]; then
## Check if the exclude list has a full path, and if not, add the config file path if there is one
if [ "$(basename $RSYNC_EXCLUDE_FROM)" == "$RSYNC_EXCLUDE_FROM" ]; then
RSYNC_EXCLUDE_FROM=$(dirname $ConfigFile)/$RSYNC_EXCLUDE_FROM
if [ "$(basename $pattern_from)" == "$pattern_from" ]; then
pattern_from=$(dirname $ConfigFile)/$pattern_ffrom
fi
if [ -e "$RSYNC_EXCLUDE_FROM" ]; then
RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude-from=\"$RSYNC_EXCLUDE_FROM\""
if [ -e "$pattern_from" ]; then
RSYNC_PATTERNS="$RSYNC_PATTERNS --exclude-from=\"$pattern_from\""
fi
fi
}
function RsyncPatterns {
if [ "$RSYNC_PATTERN_ORDER" == "exclude" ]; then
RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN"
RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM"
RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN"
RsyncPatternsFromAdd "$RSYNC_INCLUDE_FROM"
elif [ "$RSYNC_PATTERN_ORDER" == "include" ]; then
RsyncPatternsAdd "$RSYNC_INCLUDE_PATTERN"
RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM"
RsyncPatternsAdd "$RSYNC_EXCLUDE_PATTERN"
RsyncPatternsFromAdd "$RSYNC_EXCLUDE_FROM"
else
Logger "Bogus RSYNC_PATTERN_ORDER in config file" "WARN"
fi
}
function _WriteLockFilesLocal {
local lockfile="${1}"
@ -1249,9 +1271,9 @@ function tree_list {
if [ "$REMOTE_OPERATION" == "yes" ] && [ "$replica_type" == "target" ]; then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escaped_replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escaped_replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --list-only \"$replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS --list-only \"$replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
## Redirect commands stderr here to get rsync stderr output in logfile
@ -1330,12 +1352,12 @@ function sync_update {
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
if [ "$source_replica" == "initiator" ]; then
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
fi
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd"
@ -1954,7 +1976,7 @@ function Init {
if [ "$PARTIAL" == "yes" ]; then
SYNC_OPTS=$SYNC_OPTS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_EXCLUDE="$RSYNC_EXCLUDE --exclude=\"$PARTIAL_DIR\""
RSYNC_PATTERNS="$RSYNC_PATTERNS --exclude=\"$PARTIAL_DIR\""
fi
## Conflict options
@ -1970,10 +1992,8 @@ function Init {
TARGET_BACKUP=
fi
## Add Rsync exclude patterns
RsyncExcludePattern
## Add Rsync exclude from file
RsyncExcludeFrom
## Add Rsync include / exclude patterns
RsyncPatterns
## Filenames for state files
if [ $_DRYRUN -eq 1 ]; then
@ -2073,7 +2093,7 @@ function SyncOnChanges {
fi
Logger "#### Monitoring now." "NOTICE"
inotifywait --exclude $OSYNC_DIR $RSYNC_EXCLUDE -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" &
inotifywait --exclude $OSYNC_DIR $RSYNC_PATTERNS -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" &
OSYNC_SUB_PID=$!
wait $OSYNC_SUB_PID
retval=$?

10
sync.conf
Unescape Escape View File

@ -2,12 +2,12 @@
###### Osync - Rsync based two way sync engine with fault tolerance
###### (L) 2013-2015 by Orsiris "Ozy" de Jong (www.netpower.fr)
###### Config file rev 2015091201
###### Config file rev 2015103001
## ---------- GENERAL OPTIONS
## Sync job identification
SYNC_ID="sync_test"
INSTANCE_ID="sync_test"
## Directories to synchronize.
## Initiator is the system osync runs on. The initiator directory must be a local path.
@ -26,17 +26,21 @@ CREATE_DIRS=no
## Log file location. Leaving this empty will create a logfile at /var/log/osync_version_SYNC_ID.log (or current directory if /var/log doesn't exist)
LOGFILE=""
## Rsync exclude / include order (the option here will be set first)
RSYNC_PATTERN_ORDER=exclude
## List of directories to exclude from sync on both sides (rsync patterns, wildcards work).
## Paths are relative to sync dirs. List elements are separated by a semicolon.
RSYNC_EXCLUDE_PATTERN=""
#RSYNC_EXCLUDE_PATTERN="tmp;archives"
RSYNC_INCLUDE_PATTERN=""
## File that contains the list of directories or files to exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file.
## This file has to be in the same directory as the config file
## Paths are relative to sync dirs. One element per line.
RSYNC_EXCLUDE_FROM=""
#RSYNC_EXCLUDE_FROM="exclude.list"
RSYNC_INCLUDE_FROM=""
## List elements separator char. You may set an alternative separator char for your directories lists above.
PATH_SEPARATOR_CHAR=";"
@ -137,7 +141,7 @@ SMTP_PASSWORD=
## ---------- EXECUTION HOOKS
## Commands can will be run before and / or after sync process (remote execution will only happen if REMOTE_SYNC is set).
## Commands can will be run before and / or after sync process (remote execution will only happen if REMOTE_OPERATION is set).
LOCAL_RUN_BEFORE_CMD=""
LOCAL_RUN_AFTER_CMD=""

Write Preview
Loading…
Cancel
Save