Archives
/
osync
mirror of https://github.com/deajan/osync
2
0
Fork 0
Code Issues Releases Wiki Activity

Cosmetic fix

Browse Source
pull/84/head
deajan 8 years ago
parent ae89b7506e
commit fe2a3037f7
3 changed files with 87 additions and 44 deletions
Show Stats Download Patch File Download Diff File

120
dev/debug_osync.sh
Unescape Escape View File

@ -4,7 +4,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.2-beta2
PROGRAM_BUILD=2016102102
PROGRAM_BUILD=2016102203
IS_STABLE=no
# Execution order #__WITH_PARANOIA_DEBUG
@ -45,7 +45,7 @@ IS_STABLE=no
#### MINIMAL-FUNCTION-SET BEGIN ####
## FUNC_BUILD=2016102102
## FUNC_BUILD=2016102203
## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## To use in a program, define the following variables:
@ -729,8 +729,10 @@ function WaitForTaskCompletion {
if [ $(IsInteger $pid) -eq 1 ]; then
if kill -0 $pid > /dev/null 2>&1; then
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
#TODO(high): have this tested on *BSD, Mac & Win
pidState=$(ps -p$pid -o state= 2 > /dev/null)
#TODO(high): have this tested on *BSD, Mac, Win & busybox.
#TODO(high): propagate changes to ParallelExec
#pidState=$(ps -p$pid -o state= 2 > /dev/null)
pidState="$(eval $PROCESS_STATE_CMD)"
if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
newPidsArray+=($pid)
fi
@ -836,7 +838,8 @@ function ParallelExec {
if [ $(IsInteger $pid) -eq 1 ]; then
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
if kill -0 $pid > /dev/null 2>&1; then
pidState=$(ps -p$pid -o state= 2 > /dev/null)
#pidState=$(ps -p$pid -o state= 2 > /dev/null)
pidState="$(eval $PROCESS_STATE_CMD)"
if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
newPidsArray+=($pid)
fi
@ -988,11 +991,15 @@ function GetLocalOS {
local localOsVar
localOsVar="$(uname -spio 2>&1)"
if [ $? != 0 ]; then
localOsVar="$(uname -v 2>&1)"
if type -p busybox > /dev/null; then
localOsVar="BusyBox"
else
localOsVar="$(uname -spio 2>&1)"
if [ $? != 0 ]; then
localOsVar="$(uname)"
localOsVar="$(uname -v 2>&1)"
if [ $? != 0 ]; then
localOsVar="$(uname)"
fi
fi
fi
@ -1009,8 +1016,11 @@ function GetLocalOS {
*"Darwin"*)
LOCAL_OS="MacOSX"
;;
*"BusyBox"*)
LOCAL_OS="BUSYBOX"
;;
*)
if [ "$IGNORE_OS_TYPE" == "yes" ]; then #DOC: Undocumented option
if [ "$IGNORE_OS_TYPE" == "yes" ]; then #TODO(doc): Undocumented option
Logger "Running on unknown local OS [$localOsVar]." "WARN"
return
fi
@ -1029,6 +1039,7 @@ function GetRemoteOS {
local cmd
local remoteOsVar
#TODO: Add busybox detection here
if [ "$REMOTE_OPERATION" == "yes" ]; then
CheckConnectivity3rdPartyHosts
@ -1532,12 +1543,20 @@ function InitLocalOSSettings {
PING_CMD="ping -c 2 -i .2"
fi
if [ "$LOCAL_OS" == "BUSYBOX" ]; then
PROCESS_STATE_CMD="echo none"
else
PROCESS_STATE_CMD='ps -p$pid -o state= 2 > /dev/null'
fi
## Stat command has different syntax on Linux and FreeBSD/MacOSX
if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then
# Tested on BSD and Mac
STAT_CMD="stat -f \"%Sm\""
STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m"
else
STAT_CMD="stat --format %y"
# Tested on GNU stat and busybox
STAT_CMD="stat -c %y"
STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y"
fi
}
@ -1679,14 +1698,14 @@ function CheckCurrentConfig {
# Check all variables that should contain "yes" or "no"
declare -a yes_no_vars=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_PERMISSIONS PRESERVE_OWNER PRESERVE_GROUP PRESERVE_EXECUTABILITY PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR)
for i in "${yes_no_vars[@]}"; do
test="if [ \"\$$i\" != \"yes\" ] && [ \"\$$i\" != \"no\" ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
test="if [ \"\$$i\" != \"yes\" ] && [ \"\$$i\" != \"no\" ]; then Logger \"Bogus $i value [$$i] defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
eval "$test"
done
# Check all variables that should contain a numerical value >= 0
declare -a num_vars=(MINIMUM_SPACE BANDWIDTH SOFT_MAX_EXEC_TIME HARD_MAX_EXEC_TIME KEEP_LOGGING MIN_WAIT MAX_WAIT CONFLICT_BACKUP_DAYS SOFT_DELETE_DAYS RESUME_TRY MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER)
for i in "${num_vars[@]}"; do
test="if [ $(IsNumeric \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
test="if [ $(IsNumericExpand \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value [$$i] defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
eval "$test"
done
}
@ -2205,9 +2224,6 @@ function treeList {
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS -L $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replicaPath\" 2> \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID\" | grep \"^-\|^d\|^l\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID\""
fi
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
#TODO: check the following statement
## Redirect commands stderr here to get rsync stderr output in logfile with eval "$rsyncCmd" 2>> "$LOG_FILE"
## When log writing fails, $! is empty and WaitForTaskCompletion fails. Removing the 2>> log
eval "$rsyncCmd"
retval=$?
@ -2319,6 +2335,11 @@ function syncAttrs {
local escDestDir
local destReplica
if [ "$LOCAL_OS" == "BUSYBOX" ]; then
Logger "Skipping acl synchronization. Busybox doesn't have join command." "NOTICE"
return 0
fi
Logger "Getting list of files that need updates." "NOTICE"
if [ "$REMOTE_OPERATION" == "yes" ]; then
@ -2343,8 +2364,7 @@ function syncAttrs {
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "VERBOSE"
fi
#TODO: Apply SC2002: unnecessary cat
cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" | ( grep -Ev "^[^ ]*(c|s|t)[^ ]* " || :) | ( grep -E "^[^ ]*(p|o|g|a)[^ ]* " || :) | sed -e 's/^[^ ]* //' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-cleaned.$SCRIPT_PID"
( grep -Ev "^[^ ]*(c|s|t)[^ ]* " "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" || :) | ( grep -E "^[^ ]*(p|o|g|a)[^ ]* " || :) | sed -e 's/^[^ ]* //' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-cleaned.$SCRIPT_PID"
if [ $? != 0 ]; then
Logger "Cannot prepare file list for attribute sync." "CRITICAL"
exit 1
@ -2850,10 +2870,10 @@ function Sync {
targetFail=false
for pid in "${pidArray[@]}"; do
pid=${pid%:*}
if [ $pid == $initiatorPid ]; then
if [ "$pid" == "$initiatorPid" ]; then
echo "${SYNC_ACTION[0]}" > "${INITIATOR[$__initiatorLastActionFile]}"
initiatorFail=true
elif [ $pid == $targetPid ]; then
elif [ "$pid" == "$targetPid" ]; then
echo "${SYNC_ACTION[0]}" > "${INITIATOR[$__targetLastActionFile]}"
targetFail=true
fi
@ -2895,10 +2915,10 @@ function Sync {
targetFail=false
for pid in "${pidArray[@]}"; do
pid=${pid%:*}
if [ $pid == $initiatorPid ]; then
if [ "$pid" == "$initiatorPid" ]; then
echo "${SYNC_ACTION[1]}" > "${INITIATOR[$__initiatorLastActionFile]}"
initiatorFail=true
elif [ $pid == $targetPid ]; then
elif [ "$pid" == "$targetPid" ]; then
echo "${SYNC_ACTION[1]}" > "${INITIATOR[$__targetLastActionFile]}"
targetFail=true
fi
@ -2999,10 +3019,10 @@ function Sync {
targetFail=false
for pid in "${pidArray[@]}"; do
pid=${pid%:*}
if [ $pid == $initiatorPid ]; then
if [ "$pid" == "$initiatorPid" ]; then
echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__initiatorLastActionFile]}"
initiatorFail=true
elif [ $pid == $targetPid ]; then
elif [ "$pid" == "$targetPid" ]; then
echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__targetLastActionFile]}"
targetFail=true
fi
@ -3045,10 +3065,10 @@ function Sync {
targetFail=false
for pid in "${pidArray[@]}"; do
pid=${pid%:*}
if [ $pid == $initiatorPid ]; then
if [ "$pid" == "$initiatorPid" ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}"
initiatorFail=true
elif [ $pid == $targetPid ]; then
elif [ "$pid" == "$targetPid" ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}"
targetFail=true
fi
@ -3078,17 +3098,23 @@ function Sync {
function _SoftDeleteLocal {
local replicaType="${1}" # replica type (initiator, target)
local replicaDeletionPath="${2}" # Contains the full path to softdelete / backup directory without ending slash
local changeTime="${3}"
local changeTime="${3}" # Delete files older than changeTime days
local deletionType="${4}" # Trivial deletion type string
__CheckArguments 3 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG
__CheckArguments 4 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG
local retval
if [ "$LOCAL_OS" == "BUSYBOX" ]; then
Logger "Skipping $deletionType deletion on $replicaType. Busybox find -ctime not supported." "NOTICE"
return 0
fi
if [ -d "$replicaDeletionPath" ]; then
if [ $_DRYRUN == true ]; then
Logger "Listing files older than $changeTime days on $replicaType replica. Does not remove anything." "NOTICE"
Logger "Listing files older than $changeTime days on $replicaType replica for $deletionType deletion. Does not remove anything." "NOTICE"
else
Logger "Removing files older than $changeTime days on $replicaType replica." "NOTICE"
Logger "Removing files older than $changeTime days on $replicaType replica for $deletionType deletion." "NOTICE"
fi
if [ $_VERBOSE == true ]; then
@ -3121,18 +3147,25 @@ function _SoftDeleteLocal {
function _SoftDeleteRemote {
local replicaType="${1}"
local replicaDeletionPath="${2}" # Contains the full path to softdelete / backup directory without ending slash
local changeTime="${3}"
__CheckArguments 3 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG
local changeTime="${3}" # Delete files older than changeTime days
local deletionType="${4}" # Trivial deletion type string
__CheckArguments 4 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG
local retval
if [ "$REMOTE_OS" == "BUSYBOX" ]; then
Logger "Skipping $deletionType deletion on $replicaType. Busybox find -ctime not supported." "NOTICE"
return 0
fi
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
if [ $_DRYRUN == true ]; then
Logger "Listing files older than $changeTime days on $replicaType replica. Does not remove anything." "NOTICE"
Logger "Listing files older than $changeTime days on $replicaType replica for $deletionType deletion. Does not remove anything." "NOTICE"
else
Logger "Removing files older than $changeTime days on $replicaType replica." "NOTICE"
Logger "Removing files older than $changeTime days on $replicaType replica for $deletionType deletion." "NOTICE"
fi
if [ $_VERBOSE == true ]; then
@ -3168,13 +3201,13 @@ function SoftDelete {
if [ "$CONFLICT_BACKUP" != "no" ] && [ $CONFLICT_BACKUP_DAYS -ne 0 ]; then
Logger "Running conflict backup cleanup." "NOTICE"
_SoftDeleteLocal "${INITIATOR[$__type]}" "${INITIATOR[$__replicaDir]}${INITIATOR[$__backupDir]}" $CONFLICT_BACKUP_DAYS &
_SoftDeleteLocal "${INITIATOR[$__type]}" "${INITIATOR[$__replicaDir]}${INITIATOR[$__backupDir]}" $CONFLICT_BACKUP_DAYS "conflict backup" &
pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then
_SoftDeleteLocal "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__backupDir]}" $CONFLICT_BACKUP_DAYS &
_SoftDeleteLocal "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__backupDir]}" $CONFLICT_BACKUP_DAYS "conflict backup" &
pids="$pids;$!"
else
_SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__backupDir]}" $CONFLICT_BACKUP_DAYS &
_SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__backupDir]}" $CONFLICT_BACKUP_DAYS "conflict backup" &
pids="$pids;$!"
fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING
@ -3183,13 +3216,13 @@ function SoftDelete {
if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]; then
Logger "Running soft deletion cleanup." "NOTICE"
_SoftDeleteLocal "${INITIATOR[$__type]}" "${INITIATOR[$__replicaDir]}${INITIATOR[$__deleteDir]}" $SOFT_DELETE_DAYS &
_SoftDeleteLocal "${INITIATOR[$__type]}" "${INITIATOR[$__replicaDir]}${INITIATOR[$__deleteDir]}" $SOFT_DELETE_DAYS "softdelete" &
pids="$!"
if [ "$REMOTE_OPERATION" != "yes" ]; then
_SoftDeleteLocal "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__deleteDir]}" $SOFT_DELETE_DAYS &
_SoftDeleteLocal "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__deleteDir]}" $SOFT_DELETE_DAYS "softdelete" &
pids="$pids;$!"
else
_SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__deleteDir]}" $SOFT_DELETE_DAYS &
_SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__deleteDir]}" $SOFT_DELETE_DAYS "softdelete" &
pids="$pids;$!"
fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING
@ -3619,7 +3652,12 @@ opts="${opts# *}"
else
LOG_FILE="$LOGFILE"
fi
Logger "Script begin, logging to [$LOG_FILE]." "DEBUG"
if [ ! -w "$LOG_FILE" ]; then
echo "Cannot write to log $[LOG_FILE]."
exit 1
else
Logger "Script begin, logging to [$LOG_FILE]." "DEBUG"
fi
if [ "$IS_STABLE" != "yes" ]; then
Logger "This is an unstable dev build. Please use with caution." "WARN"

2
dev/ofunctions.sh
Unescape Escape View File

@ -946,7 +946,7 @@ function GetLocalOS {
local localOsVar
if type -p busybox > /dev/null; then
if type busybox > /dev/null 2>&1; then
localOsVar="BusyBox"
else
localOsVar="$(uname -spio 2>&1)"

9
install.sh
Unescape Escape View File

@ -4,7 +4,7 @@ PROGRAM=osync
PROGRAM_VERSION=1.2-beta2
PROGRAM_BINARY=$PROGRAM".sh"
PROGRAM_BATCH=$PROGRAM"-batch.sh"
SCRIPT_BUILD=2016090605
SCRIPT_BUILD=2016102201
## osync / obackup / pmocr / zsnap install script
## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8 & 10
@ -70,7 +70,7 @@ function urlencode() {
local c="${1:i:1}"
case $c in
[a-zA-Z0-9.~_-]) printf "$c" ;;
*) printf '%%%02X' "'$c" ;;
*) printf '%%%02X' "'$c" ;;
esac
done
}
@ -78,6 +78,11 @@ function urlencode() {
function SetOSSettings {
USER=root
if type busybox > /dev/null 2>&1; then
QuickLogger "$0 won't work in busybox. Please use $PROGRAM_BINARY.sh directly."
exit 1
fi
local local_os_var
local_os_var="$(uname -spio 2>&1)"

Write Preview
Loading…
Cancel
Save