|
|
|
@ -1,10 +1,12 @@
|
|
|
|
|
#!/usr/bin/env bash
|
|
|
|
|
|
|
|
|
|
# Remove Waitfor function in already sent to background functions to reduce code overhead
|
|
|
|
|
|
|
|
|
|
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
|
|
|
|
|
AUTHOR="(C) 2013-2016 by Orsiris de Jong"
|
|
|
|
|
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
|
|
|
|
|
PROGRAM_VERSION=1.2-dev-parallel-unstable
|
|
|
|
|
PROGRAM_BUILD=2016080401
|
|
|
|
|
PROGRAM_BUILD=2016080701
|
|
|
|
|
IS_STABLE=no
|
|
|
|
|
|
|
|
|
|
# Function Name Is parallel #__WITH_PARANOIA_DEBUG
|
|
|
|
@ -200,7 +202,7 @@ function _CheckReplicaPathsRemote {
|
|
|
|
|
cmd=$SSH_CMD' "if [ ! -w \"'$replica_path'\" ];then exit 1; fi" 2>&1'
|
|
|
|
|
Logger "cmd: $cmd" "DEBUG"
|
|
|
|
|
eval "$cmd" &
|
|
|
|
|
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]}
|
|
|
|
|
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} false true
|
|
|
|
|
if [ $? != 0 ]; then
|
|
|
|
|
Logger "Remote replica path [$replica_path] is not writable." "CRITICAL"
|
|
|
|
|
exit 1
|
|
|
|
@ -209,7 +211,7 @@ function _CheckReplicaPathsRemote {
|
|
|
|
|
cmd=$SSH_CMD' "if ! [ -d \"'$replica_path'\" ]; then if [ \"'$CREATE_DIRS'\" == \"yes\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_path'\"; fi; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
|
|
|
|
|
Logger "cmd: $cmd" "DEBUG"
|
|
|
|
|
eval "$cmd" &
|
|
|
|
|
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]}
|
|
|
|
|
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} false true
|
|
|
|
|
if [ $? != 0 ]; then
|
|
|
|
|
Logger "Cannot create remote replica path [$replica_path]." "CRITICAL"
|
|
|
|
|
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR"
|
|
|
|
@ -241,7 +243,7 @@ function CheckReplicaPaths {
|
|
|
|
|
_CheckReplicaPathsRemote "${TARGET[1]}" &
|
|
|
|
|
pids="$pids;$!"
|
|
|
|
|
fi
|
|
|
|
|
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true
|
|
|
|
|
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true true
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function _CheckDiskSpaceLocal {
|
|
|
|
@ -273,7 +275,7 @@ function _CheckDiskSpaceRemote {
|
|
|
|
|
cmd=$SSH_CMD' "'$COMMAND_SUDO' df -P \"'$replica_path'\"" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
|
|
|
|
|
Logger "cmd: $cmd" "DEBUG"
|
|
|
|
|
eval "$cmd" &
|
|
|
|
|
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]}
|
|
|
|
|
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} false true
|
|
|
|
|
if [ $? != 0 ]; then
|
|
|
|
|
Logger "Cannot get free space on target [$replica_path]." "ERROR"
|
|
|
|
|
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
|
|
|
|
@ -304,7 +306,7 @@ function CheckDiskSpace {
|
|
|
|
|
_CheckDiskSpaceRemote "${TARGET[1]}" &
|
|
|
|
|
pids="$pids;$!"
|
|
|
|
|
fi
|
|
|
|
|
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false
|
|
|
|
|
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false true
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -334,7 +336,7 @@ function _CreateStateDirsRemote {
|
|
|
|
|
cmd=$SSH_CMD' "if ! [ -d \"'$replica_state_dir'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$replica_state_dir'\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
|
|
|
|
|
Logger "cmd: $cmd" "DEBUG"
|
|
|
|
|
eval "$cmd" &
|
|
|
|
|
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]}
|
|
|
|
|
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} false true
|
|
|
|
|
if [ $? != 0 ]; then
|
|
|
|
|
Logger "Cannot create remote state dir [$replica_state_dir]." "CRITICAL"
|
|
|
|
|
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR"
|
|
|
|
@ -356,7 +358,7 @@ function CreateStateDirs {
|
|
|
|
|
_CreateStateDirsRemote "${TARGET[1]}${TARGET[3]}" &
|
|
|
|
|
pids="$pids;$!"
|
|
|
|
|
fi
|
|
|
|
|
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true
|
|
|
|
|
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true true
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function _CheckLocksLocal {
|
|
|
|
@ -397,7 +399,7 @@ function _CheckLocksRemote {
|
|
|
|
|
cmd=$SSH_CMD' "if [ -f \"'$lockfile'\" ]; then cat \"'$lockfile'\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'"'
|
|
|
|
|
Logger "cmd: $cmd" "DEBUG"
|
|
|
|
|
eval "$cmd" &
|
|
|
|
|
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]}
|
|
|
|
|
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} false true
|
|
|
|
|
if [ $? != 0 ]; then
|
|
|
|
|
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then
|
|
|
|
|
lockfile_content=$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)
|
|
|
|
@ -458,7 +460,7 @@ function CheckLocks {
|
|
|
|
|
_CheckLocksRemote "${TARGET[2]}" &
|
|
|
|
|
pids="$pids;$!"
|
|
|
|
|
fi
|
|
|
|
|
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true
|
|
|
|
|
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true true
|
|
|
|
|
WriteLockFiles
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@ -487,7 +489,7 @@ function _WriteLockFilesRemote {
|
|
|
|
|
cmd=$SSH_CMD' "echo '$SCRIPT_PID@$INSTANCE_ID' | '$COMMAND_SUDO' tee \"'$lockfile'\"" > /dev/null 2>&1'
|
|
|
|
|
Logger "cmd: $cmd" "DEBUG"
|
|
|
|
|
eval "$cmd" &
|
|
|
|
|
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]}
|
|
|
|
|
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} false true
|
|
|
|
|
if [ $? != 0 ]; then
|
|
|
|
|
Logger "Could not set lock on remote $replica_type replica." "CRITICAL"
|
|
|
|
|
exit 1
|
|
|
|
@ -510,7 +512,7 @@ function WriteLockFiles {
|
|
|
|
|
_WriteLockFilesRemote "${TARGET[2]}" &
|
|
|
|
|
pids="$pids;$!"
|
|
|
|
|
fi
|
|
|
|
|
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true
|
|
|
|
|
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true true
|
|
|
|
|
LOCK_FILES_EXIST=1
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@ -540,7 +542,7 @@ function _UnlockReplicasRemote {
|
|
|
|
|
cmd=$SSH_CMD' "if [ -f \"'$lockfile'\" ]; then '$COMMAND_SUDO' rm -f \"'$lockfile'\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
|
|
|
|
|
Logger "cmd: $cmd" "DEBUG"
|
|
|
|
|
eval "$cmd" &
|
|
|
|
|
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]}
|
|
|
|
|
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} false true
|
|
|
|
|
if [ $? != 0 ]; then
|
|
|
|
|
Logger "Could not unlock remote replica." "ERROR"
|
|
|
|
|
Logger "Command Output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
|
|
|
|
@ -568,7 +570,7 @@ function UnlockReplicas {
|
|
|
|
|
_UnlockReplicasRemote "${TARGET[2]}" &
|
|
|
|
|
pids="$pids;$!"
|
|
|
|
|
fi
|
|
|
|
|
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false
|
|
|
|
|
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false true
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
###### Sync core functions
|
|
|
|
@ -600,9 +602,9 @@ function tree_list {
|
|
|
|
|
fi
|
|
|
|
|
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
|
|
|
|
|
## Redirect commands stderr here to get rsync stderr output in logfile with eval "$rsync_cmd" 2>> "$LOG_FILE"
|
|
|
|
|
## When log writing fails, $! is empty and WaitForCompletion fails. Removing the 2>> log
|
|
|
|
|
## When log writing fails, $! is empty and WaitForTaskCompletion fails. Removing the 2>> log
|
|
|
|
|
eval "$rsync_cmd"
|
|
|
|
|
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
|
|
|
|
|
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false
|
|
|
|
|
retval=$?
|
|
|
|
|
## Retval 24 = some files vanished while creating list
|
|
|
|
|
if ([ $retval == 0 ] || [ $retval == 24 ]) && [ -f "$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID" ]; then
|
|
|
|
@ -673,7 +675,7 @@ function _get_file_ctime_mtime_remote {
|
|
|
|
|
cmd='cat "'$file_list'" | '$SSH_CMD' "while read file; do '$REMOTE_STAT_CTIME_MTIME_CMD' \"'$replica_path'\$file\"; done | sort" > "'$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID'"'
|
|
|
|
|
Logger "CMD: $cmd" "DEBUG"
|
|
|
|
|
eval $cmd
|
|
|
|
|
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
|
|
|
|
|
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false
|
|
|
|
|
if [ $? != 0 ]; then
|
|
|
|
|
Logger "Getting file attributes failed [$retval] on $replica_type. Stopping execution." "CRITICAL"
|
|
|
|
|
if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID" ]; then
|
|
|
|
@ -705,7 +707,7 @@ function sync_attrs {
|
|
|
|
|
fi
|
|
|
|
|
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
|
|
|
|
|
eval "$rsync_cmd"
|
|
|
|
|
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
|
|
|
|
|
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false
|
|
|
|
|
retval=$?
|
|
|
|
|
if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then
|
|
|
|
|
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
|
|
|
|
@ -781,7 +783,7 @@ function sync_attrs {
|
|
|
|
|
|
|
|
|
|
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
|
|
|
|
|
eval "$rsync_cmd"
|
|
|
|
|
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
|
|
|
|
|
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false
|
|
|
|
|
retval=$?
|
|
|
|
|
if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID" ]; then
|
|
|
|
|
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID)" "NOTICE"
|
|
|
|
@ -836,7 +838,7 @@ function sync_update {
|
|
|
|
|
fi
|
|
|
|
|
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
|
|
|
|
|
eval "$rsync_cmd"
|
|
|
|
|
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
|
|
|
|
|
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false
|
|
|
|
|
retval=$?
|
|
|
|
|
if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID" ]; then
|
|
|
|
|
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID)" "NOTICE"
|
|
|
|
@ -1072,7 +1074,7 @@ function deletion_propagation {
|
|
|
|
|
delete_dir="${INITIATOR[5]}"
|
|
|
|
|
|
|
|
|
|
_delete_local "$replica_dir" "${TARGET[0]}$deleted_list_file" "$delete_dir" "${TARGET[0]}$deleted_failed_list_file" &
|
|
|
|
|
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
|
|
|
|
|
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false
|
|
|
|
|
retval=$?
|
|
|
|
|
if [ $retval != 0 ]; then
|
|
|
|
|
Logger "Deletion on replica $replica_type failed." "CRITICAL"
|
|
|
|
@ -1087,7 +1089,7 @@ function deletion_propagation {
|
|
|
|
|
else
|
|
|
|
|
_delete_local "$replica_dir" "${INITIATOR[0]}$deleted_list_file" "$delete_dir" "${INITIATOR[0]}$deleted_failed_list_file" &
|
|
|
|
|
fi
|
|
|
|
|
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
|
|
|
|
|
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false
|
|
|
|
|
retval=$?
|
|
|
|
|
if [ $retval == 0 ]; then
|
|
|
|
|
if [ -f "$RUN_DIR/$PROGRAM._delete_remote.$SCRIPT_PID" ] && [ $_VERBOSE -eq 1 ]; then
|
|
|
|
@ -1310,7 +1312,7 @@ function _SoftDeleteLocal {
|
|
|
|
|
else
|
|
|
|
|
Dummy &
|
|
|
|
|
fi
|
|
|
|
|
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
|
|
|
|
|
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false
|
|
|
|
|
retval=$?
|
|
|
|
|
if [ $retval -ne 0 ]; then
|
|
|
|
|
Logger "Error while executing cleanup on $replica_type replica." "ERROR"
|
|
|
|
@ -1345,7 +1347,7 @@ function _SoftDeleteRemote {
|
|
|
|
|
cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} echo Will delete file {} && '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
|
|
|
|
|
Logger "cmd: $cmd" "DEBUG"
|
|
|
|
|
eval "$cmd" &
|
|
|
|
|
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
|
|
|
|
|
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false
|
|
|
|
|
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
@ -1357,7 +1359,7 @@ function _SoftDeleteRemote {
|
|
|
|
|
else
|
|
|
|
|
Dummy &
|
|
|
|
|
fi
|
|
|
|
|
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
|
|
|
|
|
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false false
|
|
|
|
|
retval=$?
|
|
|
|
|
if [ $retval -ne 0 ]; then
|
|
|
|
|
Logger "Error while executing cleanup on remote $replica_type replica." "ERROR"
|
|
|
|
@ -1384,7 +1386,7 @@ function SoftDelete {
|
|
|
|
|
_SoftDeleteRemote "${TARGET[0]}" "${TARGET[1]}${TARGET[4]}" $CONFLICT_BACKUP_DAYS &
|
|
|
|
|
pids="$pids;$!"
|
|
|
|
|
fi
|
|
|
|
|
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false
|
|
|
|
|
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false true
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]; then
|
|
|
|
@ -1399,7 +1401,7 @@ function SoftDelete {
|
|
|
|
|
_SoftDeleteRemote "${TARGET[0]}" "${TARGET[1]}${TARGET[5]}" $SOFT_DELETE_DAYS &
|
|
|
|
|
pids="$pids;$!"
|
|
|
|
|
fi
|
|
|
|
|
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false
|
|
|
|
|
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false true
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|