Archives
/
osync
mirror of https://github.com/deajan/osync
2
0
Fork 0
Code Issues Releases Wiki Activity

WIP on attr sync function

Browse Source
pull/48/merge
deajan 8 years ago
parent ccb108e7e2
commit 47fbcfd817
4 changed files with 250 additions and 79 deletions
Show Stats Download Patch File Download Diff File

110
dev/debug_osync.sh
Unescape Escape View File

@ -3,11 +3,11 @@
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(L) 2013-2015 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.1-pre
PROGRAM_BUILD=2016020804
PROGRAM_VERSION=1.1-dev
PROGRAM_BUILD=2016021602
IS_STABLE=no
FUNC_BUILD=2015122101
FUNC_BUILD=2016021603
## BEGIN Generic functions for osync & obackup written in 2013-2015 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
@ -791,13 +791,14 @@ function PreInit {
fi
## Set rsync default arguments
RSYNC_ARGS="-rlptgoD"
RSYNC_ARGS="-rltD"
RSYNC_ATTR_ARGS="-pgo"
if [ "$PRESERVE_ACL" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -A"
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A"
fi
if [ "$PRESERVE_XATTR" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -X"
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X"
fi
if [ "$RSYNC_COMPRESS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -z"
@ -812,7 +813,7 @@ function PreInit {
RSYNC_ARGS=$RSYNC_ARGS" -H"
fi
if [ "$CHECKSUM" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --checksum"
RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum"
fi
if [ $_DRYRUN -eq 1 ]; then
RSYNC_ARGS=$RSYNC_ARGS" -n"
@ -822,6 +823,17 @@ function PreInit {
RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH"
fi
if [ "$PARTIAL" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\""
fi
if [ "$DELTA_COPIES" != "no" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file"
else
RSYNC_ARGS=$RSYNC_ARGS" --whole-file"
fi
## Set compression executable and extension
COMPRESSION_LEVEL=3
if type xz > /dev/null 2>&1
@ -891,7 +903,7 @@ function InitRemoteOSSettings {
## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability)
if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -E"
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -E"
fi
if [ "$REMOTE_OS" == "msys" ]; then
@ -1450,15 +1462,15 @@ function tree_list {
local tree_filename="${3}" # filename to output tree (will be prefixed with $replica_type)
__CheckArguments 3 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
local escaped_replica_path=$(EscapeSpaces "$replica_path") #TODO: See if escpaed still needed when using singlequotes instead of doublequotes for command eval
local escaped_replica_path=$(EscapeSpaces "$replica_path")
Logger "Creating $replica_type replica file list [$replica_path]." "NOTICE"
if [ "$REMOTE_OPERATION" == "yes" ] && [ "$replica_type" == "target" ]; then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escaped_replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escaped_replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
## Redirect commands stderr here to get rsync stderr output in logfile
@ -1513,6 +1525,58 @@ function delete_list {
fi
}
function _get_file_attrs_local {
__CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
}
function _get_file_attrs_remote {
__CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
}
function sync_attrs {
local initiator_replica="${1}" # Contains #TODO: Write ACL update function, check other ctime related attributes (xattr ?)
local target_replica="${2}"
__CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
if [ "$REMOTE_OPERATION" == "yes" ]; then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$initiator_replica/\" $REMOTE_USER@$REMOTE_HOST:\"$target_replica/\" | grep -Ev \"^[^ ]*(c|s|t)[^ ]* \" | grep -E \"^[^ ]*(p|o|g|a)[^ ]* \" | sed -e 's/^[^ ]* //' > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1 &"
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$initiator_replica/\" \"$target_replica/\" | grep -Ev \"^[^ ]*(c|s|t)[^ ]* \" | grep -E \"^[^ ]*(p|o|g|a)[^ ]* \" | sed -e 's/^[^ ]* //' > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1 &"
fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd"
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $FUNCNAME
retval=$?
if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "NOTICE"
fi
if [ $retval != 0 ] && [ $retval != 24 ]; then
Logger "Getting file attributes failed. Stopping execution." "CRITICAL"
if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then
Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "NOTICE"
fi
exit $retval
else
Logger "Getting file attributes on replicas succeded." "NOTICE"
return 0
fi
_get_file_attrs_local "$INITIATOR_SYNC_DIR"
if [ "$REMOTE_OPERATION" != "yes" ]; then
_get_file_attrs_remote "$TARGET_SYNC_DIR"
else
_get_file_attrs_local "$TARGET_SYNC_DIR"
fi
#WIP
# Join both files into tmp file
# Rsync file from tmp file
}
# sync_update(source replica, destination replica, delete_list_filename)
function sync_update {
local source_replica="${1}" # Contains replica type of source: initiator, target
@ -1539,12 +1603,12 @@ function sync_update {
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
if [ "$source_replica" == "initiator" ]; then
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
fi
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS $SYNC_OPTS $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd"
@ -1825,7 +1889,8 @@ function deletion_propagation {
######
###### Step 1: Create current tree list for initiator and target replicas (Steps 1M and 1S)
###### Step 2: Create deleted file list for initiator and target replicas (Steps 2M and 2S)
###### Step 3: Update initiator and target replicas (Steps 3M and 3S, order depending on conflict prevalence)
###### Step 3a: Update initiator and target file attributes only
###### Step 3b: Update initiator and target replicas (Steps 3M and 3S, order depending on conflict prevalence)
###### Step 4: Deleted file propagation to initiator and target replicas (Steps 4M and 4S)
###### Step 5: Create after run tree list for initiator and target replicas (Steps 5M and 5S)
@ -1903,6 +1968,8 @@ function Sync {
fi
resume_sync="resumed"
fi
#TODO write resume step
#sync_attrs "$INITIATOR_SYNC_DIR" "$TARGET_SYNC_DIR" #TODO: escapespaces if remote... need to refactor with sync_update
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ]; then
if [ "$CONFLICT_PREVALANCE" != "initiator" ]; then
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ]; then
@ -2160,6 +2227,8 @@ function Init {
## Partial downloads dirs
PARTIAL_DIR=$OSYNC_DIR"_partial"
## TODO: checksum=no / yes not implemented, be careful with sync_attrs which must itemize based on mtime
## Set sync only function arguments for rsync
SYNC_OPTS="-u"
@ -2171,17 +2240,6 @@ function Init {
SYNC_OPTS=$SYNC_OPTS" --stats"
fi
if [ "$PARTIAL" == "yes" ]; then
SYNC_OPTS=$SYNC_OPTS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\""
fi
if [ "$DELTA_COPIES" != "no" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file"
else
RSYNC_ARGS=$RSYNC_ARGS" --whole-file"
fi
## Conflict options
if [ "$CONFLICT_BACKUP" != "no" ]; then
INITIATOR_BACKUP="--backup --backup-dir=\"$INITIATOR_BACKUP_DIR\""

86
dev/n_osync.sh
Unescape Escape View File

@ -3,8 +3,8 @@
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(L) 2013-2015 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.1-pre
PROGRAM_BUILD=2016020804
PROGRAM_VERSION=1.1-dev
PROGRAM_BUILD=2016021602
IS_STABLE=no
source "./ofunctions.sh"
@ -556,15 +556,15 @@ function tree_list {
local tree_filename="${3}" # filename to output tree (will be prefixed with $replica_type)
__CheckArguments 3 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
local escaped_replica_path=$(EscapeSpaces "$replica_path") #TODO: See if escpaed still needed when using singlequotes instead of doublequotes for command eval
local escaped_replica_path=$(EscapeSpaces "$replica_path")
Logger "Creating $replica_type replica file list [$replica_path]." "NOTICE"
if [ "$REMOTE_OPERATION" == "yes" ] && [ "$replica_type" == "target" ]; then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escaped_replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escaped_replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
## Redirect commands stderr here to get rsync stderr output in logfile
@ -619,6 +619,58 @@ function delete_list {
fi
}
function _get_file_attrs_local {
__CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
}
function _get_file_attrs_remote {
__CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
}
function sync_attrs {
local initiator_replica="${1}" # Contains #TODO: Write ACL update function, check other ctime related attributes (xattr ?)
local target_replica="${2}"
__CheckArguments 2 $# $FUNCNAME "$@" #__WITH_PARANOIA_DEBUG
if [ "$REMOTE_OPERATION" == "yes" ]; then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$initiator_replica/\" $REMOTE_USER@$REMOTE_HOST:\"$target_replica/\" | grep -Ev \"^[^ ]*(c|s|t)[^ ]* \" | grep -E \"^[^ ]*(p|o|g|a)[^ ]* \" | sed -e 's/^[^ ]* //' > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1 &"
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$initiator_replica/\" \"$target_replica/\" | grep -Ev \"^[^ ]*(c|s|t)[^ ]* \" | grep -E \"^[^ ]*(p|o|g|a)[^ ]* \" | sed -e 's/^[^ ]* //' > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1 &"
fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd"
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $FUNCNAME
retval=$?
if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "NOTICE"
fi
if [ $retval != 0 ] && [ $retval != 24 ]; then
Logger "Getting file attributes failed. Stopping execution." "CRITICAL"
if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then
Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "NOTICE"
fi
exit $retval
else
Logger "Getting file attributes on replicas succeded." "NOTICE"
return 0
fi
_get_file_attrs_local "$INITIATOR_SYNC_DIR"
if [ "$REMOTE_OPERATION" != "yes" ]; then
_get_file_attrs_remote "$TARGET_SYNC_DIR"
else
_get_file_attrs_local "$TARGET_SYNC_DIR"
fi
#WIP
# Join both files into tmp file
# Rsync file from tmp file
}
# sync_update(source replica, destination replica, delete_list_filename)
function sync_update {
local source_replica="${1}" # Contains replica type of source: initiator, target
@ -645,12 +697,12 @@ function sync_update {
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
if [ "$source_replica" == "initiator" ]; then
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
fi
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS $SYNC_OPTS $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd"
@ -931,7 +983,8 @@ function deletion_propagation {
######
###### Step 1: Create current tree list for initiator and target replicas (Steps 1M and 1S)
###### Step 2: Create deleted file list for initiator and target replicas (Steps 2M and 2S)
###### Step 3: Update initiator and target replicas (Steps 3M and 3S, order depending on conflict prevalence)
###### Step 3a: Update initiator and target file attributes only
###### Step 3b: Update initiator and target replicas (Steps 3M and 3S, order depending on conflict prevalence)
###### Step 4: Deleted file propagation to initiator and target replicas (Steps 4M and 4S)
###### Step 5: Create after run tree list for initiator and target replicas (Steps 5M and 5S)
@ -1009,6 +1062,8 @@ function Sync {
fi
resume_sync="resumed"
fi
#TODO write resume step
#sync_attrs "$INITIATOR_SYNC_DIR" "$TARGET_SYNC_DIR" #TODO: escapespaces if remote... need to refactor with sync_update
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ]; then
if [ "$CONFLICT_PREVALANCE" != "initiator" ]; then
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ]; then
@ -1266,6 +1321,8 @@ function Init {
## Partial downloads dirs
PARTIAL_DIR=$OSYNC_DIR"_partial"
## TODO: checksum=no / yes not implemented, be careful with sync_attrs which must itemize based on mtime
## Set sync only function arguments for rsync
SYNC_OPTS="-u"
@ -1277,17 +1334,6 @@ function Init {
SYNC_OPTS=$SYNC_OPTS" --stats"
fi
if [ "$PARTIAL" == "yes" ]; then
SYNC_OPTS=$SYNC_OPTS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\""
fi
if [ "$DELTA_COPIES" != "no" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file"
else
RSYNC_ARGS=$RSYNC_ARGS" --whole-file"
fi
## Conflict options
if [ "$CONFLICT_BACKUP" != "no" ]; then
INITIATOR_BACKUP="--backup --backup-dir=\"$INITIATOR_BACKUP_DIR\""

26
dev/ofunctions.sh
Unescape Escape View File

@ -1,4 +1,4 @@
FUNC_BUILD=2015121503
FUNC_BUILD=2016021603
## BEGIN Generic functions for osync & obackup written in 2013-2015 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
@ -318,7 +318,7 @@ function SendAlert {
fi
# If function has not returned 0 yet, assume it's critical that no alert can be sent
Logger "/!\ CRITICAL: Cannot send alert" "ERROR" # Is not marked critical because execution must continue
Logger "/!\ CRITICAL: Cannot send alert (neither mutt, mail, sendmail nor sendemail found)." "ERROR" # Is not marked critical because execution must continue
# Delete tmp log file
if [ -f "$ALERT_LOG_FILE" ]; then
@ -782,13 +782,14 @@ function PreInit {
fi
## Set rsync default arguments
RSYNC_ARGS="-rlptgoD"
RSYNC_ARGS="-rltD"
RSYNC_ATTR_ARGS="-pgo"
if [ "$PRESERVE_ACL" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -A"
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A"
fi
if [ "$PRESERVE_XATTR" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -X"
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X"
fi
if [ "$RSYNC_COMPRESS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -z"
@ -803,7 +804,7 @@ function PreInit {
RSYNC_ARGS=$RSYNC_ARGS" -H"
fi
if [ "$CHECKSUM" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --checksum"
RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum"
fi
if [ $_DRYRUN -eq 1 ]; then
RSYNC_ARGS=$RSYNC_ARGS" -n"
@ -813,6 +814,17 @@ function PreInit {
RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH"
fi
if [ "$PARTIAL" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\""
fi
if [ "$DELTA_COPIES" != "no" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file"
else
RSYNC_ARGS=$RSYNC_ARGS" --whole-file"
fi
## Set compression executable and extension
COMPRESSION_LEVEL=3
if type xz > /dev/null 2>&1
@ -882,7 +894,7 @@ function InitRemoteOSSettings {
## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability)
if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -E"
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -E"
fi
if [ "$REMOTE_OS" == "msys" ]; then

107
osync.sh
Unescape Escape View File

@ -3,11 +3,11 @@
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(L) 2013-2015 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.1-pre
PROGRAM_BUILD=2016020804
PROGRAM_VERSION=1.1-dev
PROGRAM_BUILD=2016021602
IS_STABLE=no
FUNC_BUILD=2015122101
FUNC_BUILD=2016021603
## BEGIN Generic functions for osync & obackup written in 2013-2015 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
@ -693,13 +693,14 @@ function PreInit {
fi
## Set rsync default arguments
RSYNC_ARGS="-rlptgoD"
RSYNC_ARGS="-rltD"
RSYNC_ATTR_ARGS="-pgo"
if [ "$PRESERVE_ACL" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -A"
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A"
fi
if [ "$PRESERVE_XATTR" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -X"
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X"
fi
if [ "$RSYNC_COMPRESS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -z"
@ -714,7 +715,7 @@ function PreInit {
RSYNC_ARGS=$RSYNC_ARGS" -H"
fi
if [ "$CHECKSUM" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --checksum"
RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum"
fi
if [ $_DRYRUN -eq 1 ]; then
RSYNC_ARGS=$RSYNC_ARGS" -n"
@ -724,6 +725,17 @@ function PreInit {
RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH"
fi
if [ "$PARTIAL" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\""
fi
if [ "$DELTA_COPIES" != "no" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file"
else
RSYNC_ARGS=$RSYNC_ARGS" --whole-file"
fi
## Set compression executable and extension
COMPRESSION_LEVEL=3
if type xz > /dev/null 2>&1
@ -790,7 +802,7 @@ function InitRemoteOSSettings {
## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability)
if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -E"
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -E"
fi
if [ "$REMOTE_OS" == "msys" ]; then
@ -1325,15 +1337,15 @@ function tree_list {
local replica_type="${2}" # replica type: initiator, target
local tree_filename="${3}" # filename to output tree (will be prefixed with $replica_type)
local escaped_replica_path=$(EscapeSpaces "$replica_path") #TODO: See if escpaed still needed when using singlequotes instead of doublequotes for command eval
local escaped_replica_path=$(EscapeSpaces "$replica_path")
Logger "Creating $replica_type replica file list [$replica_path]." "NOTICE"
if [ "$REMOTE_OPERATION" == "yes" ] && [ "$replica_type" == "target" ]; then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escaped_replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escaped_replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
## Redirect commands stderr here to get rsync stderr output in logfile
@ -1387,6 +1399,55 @@ function delete_list {
fi
}
function _get_file_attrs_local {
}
function _get_file_attrs_remote {
}
function sync_attrs {
local initiator_replica="${1}" # Contains #TODO: Write ACL update function, check other ctime related attributes (xattr ?)
local target_replica="${2}"
if [ "$REMOTE_OPERATION" == "yes" ]; then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$initiator_replica/\" $REMOTE_USER@$REMOTE_HOST:\"$target_replica/\" | grep -Ev \"^[^ ]*(c|s|t)[^ ]* \" | grep -E \"^[^ ]*(p|o|g|a)[^ ]* \" | sed -e 's/^[^ ]* //' > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1 &"
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$initiator_replica/\" \"$target_replica/\" | grep -Ev \"^[^ ]*(c|s|t)[^ ]* \" | grep -E \"^[^ ]*(p|o|g|a)[^ ]* \" | sed -e 's/^[^ ]* //' > $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID 2>&1 &"
fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd"
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $FUNCNAME
retval=$?
if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "NOTICE"
fi
if [ $retval != 0 ] && [ $retval != 24 ]; then
Logger "Getting file attributes failed. Stopping execution." "CRITICAL"
if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID" ]; then
Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.$FUNCNAME.$SCRIPT_PID)" "NOTICE"
fi
exit $retval
else
Logger "Getting file attributes on replicas succeded." "NOTICE"
return 0
fi
_get_file_attrs_local "$INITIATOR_SYNC_DIR"
if [ "$REMOTE_OPERATION" != "yes" ]; then
_get_file_attrs_remote "$TARGET_SYNC_DIR"
else
_get_file_attrs_local "$TARGET_SYNC_DIR"
fi
#WIP
# Join both files into tmp file
# Rsync file from tmp file
}
# sync_update(source replica, destination replica, delete_list_filename)
function sync_update {
local source_replica="${1}" # Contains replica type of source: initiator, target
@ -1412,12 +1473,12 @@ function sync_update {
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
if [ "$source_replica" == "initiator" ]; then
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" $REMOTE_USER@$REMOTE_HOST:\"$ESC_SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
fi
else
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $SYNC_OPTS $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSNYC_TYPE_ARGS $SYNC_OPTS $BACKUP_DIR --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"$INITIATOR_STATE_DIR/$source_replica$delete_list_filename\" --exclude-from=\"$INITIATOR_STATE_DIR/$destination_replica$delete_list_filename\" \"$SOURCE_DIR/\" \"$DEST_DIR/\" > $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID 2>&1 &"
fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd"
@ -1695,7 +1756,8 @@ function deletion_propagation {
######
###### Step 1: Create current tree list for initiator and target replicas (Steps 1M and 1S)
###### Step 2: Create deleted file list for initiator and target replicas (Steps 2M and 2S)
###### Step 3: Update initiator and target replicas (Steps 3M and 3S, order depending on conflict prevalence)
###### Step 3a: Update initiator and target file attributes only
###### Step 3b: Update initiator and target replicas (Steps 3M and 3S, order depending on conflict prevalence)
###### Step 4: Deleted file propagation to initiator and target replicas (Steps 4M and 4S)
###### Step 5: Create after run tree list for initiator and target replicas (Steps 5M and 5S)
@ -1772,6 +1834,8 @@ function Sync {
fi
resume_sync="resumed"
fi
#TODO write resume step
#sync_attrs "$INITIATOR_SYNC_DIR" "$TARGET_SYNC_DIR" #TODO: escapespaces if remote... need to refactor with sync_update
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.fail" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[5]}.success" ]; then
if [ "$CONFLICT_PREVALANCE" != "initiator" ]; then
if [ "$resume_sync" == "resumed" ] || [ "$resume_sync" == "${SYNC_ACTION[3]}.success" ] || [ "$resume_sync" == "${SYNC_ACTION[4]}.fail" ]; then
@ -2025,6 +2089,8 @@ function Init {
## Partial downloads dirs
PARTIAL_DIR=$OSYNC_DIR"_partial"
## TODO: checksum=no / yes not implemented, be careful with sync_attrs which must itemize based on mtime
## Set sync only function arguments for rsync
SYNC_OPTS="-u"
@ -2036,17 +2102,6 @@ function Init {
SYNC_OPTS=$SYNC_OPTS" --stats"
fi
if [ "$PARTIAL" == "yes" ]; then
SYNC_OPTS=$SYNC_OPTS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\""
fi
if [ "$DELTA_COPIES" != "no" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file"
else
RSYNC_ARGS=$RSYNC_ARGS" --whole-file"
fi
## Conflict options
if [ "$CONFLICT_BACKUP" != "no" ]; then
INITIATOR_BACKUP="--backup --backup-dir=\"$INITIATOR_BACKUP_DIR\""

Write Preview
Loading…
Cancel
Save