Archives
/
osync
mirror of https://github.com/deajan/osync
2
0
Fork 0
Code Issues Releases Wiki Activity

Rebuilt targets

Browse Source
pull/100/head
deajan 7 years ago
parent bbff9d55d6
commit 56b9efd686
3 changed files with 232 additions and 127 deletions
Show Stats Download Patch File Download Diff File

234
dev/debug_osync.sh
Unescape Escape View File

@ -40,8 +40,8 @@ IS_STABLE=yes
# CleanUp no #__WITH_PARANOIA_DEBUG
_OFUNCTIONS_VERSION=2.1
_OFUNCTIONS_BUILD=2017032301
_OFUNCTIONS_VERSION=2.1.2
_OFUNCTIONS_BUILD=2017052601
_OFUNCTIONS_BOOTSTRAP=true
## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
@ -66,6 +66,9 @@ fi
## Correct output of sort command (language agnostic sorting)
export LC_ALL=C
## Default umask for file creation
umask 0077
# Standard alert mail body
MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
@ -293,7 +296,7 @@ function Logger {
return
elif [ "$level" == "VERBOSE" ]; then
if [ $_LOGGER_VERBOSE == true ]; then
_Logger "$prefix:$value" "$prefix$value"
_Logger "$prefix($level):$value" "$prefix$value"
fi
return
elif [ "$level" == "ALWAYS" ]; then
@ -662,6 +665,18 @@ function Spinner {
fi
}
function _PerfProfiler { #__WITH_PARANOIA_DEBUG
local perfString #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
perfString=$(ps -p $$ -o %cpu,%mem,cmd,time) #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
for i in $(pgrep -P $$); do #__WITH_PARANOIA_DEBUG
perfString="$perfString\n"$(ps -p $i -o %cpu,%mem,cmd,time | tail -1) #__WITH_PARANOIA_DEBUG
done #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
Logger "PerfProfiler: $perfString" "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
} #__WITH_PARANOIA_DEBUG
# Time control function for background processes, suitable for multiple synchronous processes
# Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2
@ -794,6 +809,11 @@ function WaitForTaskCompletion {
pidsArray=("${newPidsArray[@]}")
# Trivial wait time for bash to not eat up all CPU
sleep $sleepTime
if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG
_PerfProfiler ##__WITH_PARANOIA_DEBUG
fi ##__WITH_PARANOIA_DEBUG
done
Logger "${FUNCNAME[0]} ended for [$callerName] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
@ -959,6 +979,10 @@ function ParallelExec {
# Trivial wait time for bash to not eat up all CPU
sleep $sleepTime
if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG
_PerfProfiler ##__WITH_PARANOIA_DEBUG
fi ##__WITH_PARANOIA_DEBUG
done
return $errorCount
@ -1184,6 +1208,62 @@ function GetLocalOS {
LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)"
}
#__BEGIN_WITH_PARANOIA_DEBUG
function __CheckArguments {
# Checks the number of arguments of a function and raises an error if some are missing
if [ "$_DEBUG" == "yes" ]; then
local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments
local numberOfGivenArguments="${2}" # Number of arguments that have been passed
local minArgs
local maxArgs
# All arguments of the function to check are passed as array in ${3} (the function call waits for $@)
# If any of the arguments contains spaces, bash things there are two aguments
# In order to avoid this, we need to iterate over ${3} and count
callerName="${FUNCNAME[1]}"
local iterate=3
local fetchArguments=true
local argList=""
local countedArguments
while [ $fetchArguments == true ]; do
cmd='argument=${'$iterate'}'
eval $cmd
if [ "$argument" == "" ]; then
fetchArguments=false
else
argList="$argList[Argument $((iterate-2)): $argument] "
iterate=$((iterate+1))
fi
done
countedArguments=$((iterate-3))
if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then
minArgs=$numberOfArguments
maxArgs=$numberOfArguments
else
IFS='-' read minArgs maxArgs <<< "$numberOfArguments"
fi
Logger "Entering function [$callerName]." "PARANOIA_DEBUG"
if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then
Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR"
Logger "$callerName arguments: $argList" "ERROR"
else
if [ ! -z "$argList" ]; then
Logger "$callerName arguments: $argList" "PARANOIA_DEBUG"
fi
fi
fi
}
#__END_WITH_PARANOIA_DEBUG
function GetRemoteOS {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
@ -1439,62 +1519,6 @@ function CheckConnectivity3rdPartyHosts {
fi #__WITH_PARANOIA_DEBUG
}
#__BEGIN_WITH_PARANOIA_DEBUG
function __CheckArguments {
# Checks the number of arguments of a function and raises an error if some are missing
if [ "$_DEBUG" == "yes" ]; then
local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments
local numberOfGivenArguments="${2}" # Number of arguments that have been passed
local minArgs
local maxArgs
# All arguments of the function to check are passed as array in ${3} (the function call waits for $@)
# If any of the arguments contains spaces, bash things there are two aguments
# In order to avoid this, we need to iterate over ${3} and count
callerName="${FUNCNAME[1]}"
local iterate=3
local fetchArguments=true
local argList=""
local countedArguments
while [ $fetchArguments == true ]; do
cmd='argument=${'$iterate'}'
eval $cmd
if [ "$argument" == "" ]; then
fetchArguments=false
else
argList="$argList[Argument $((iterate-2)): $argument] "
iterate=$((iterate+1))
fi
done
countedArguments=$((iterate-3))
if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then
minArgs=$numberOfArguments
maxArgs=$numberOfArguments
else
IFS='-' read minArgs maxArgs <<< "$numberOfArguments"
fi
Logger "Entering function [$callerName]." "PARANOIA_DEBUG"
if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then
Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR"
Logger "$callerName arguments: $argList" "ERROR"
else
if [ ! -z "$argList" ]; then
Logger "$callerName arguments: $argList" "PARANOIA_DEBUG"
fi
fi
fi
}
#__END_WITH_PARANOIA_DEBUG
function RsyncPatternsAdd {
local patternType="${1}" # exclude or include
local pattern="${2}"
@ -1845,35 +1869,46 @@ function ParentPid {
# Neat version compare function found at http://stackoverflow.com/a/4025065/2635443
# Returns 0 if equal, 1 if $1 > $2 and 2 if $1 < $2
vercomp () {
if [[ $1 == $2 ]]
then
return 0
fi
local IFS=.
local i ver1=($1) ver2=($2)
# fill empty fields in ver1 with zeros
for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
do
ver1[i]=0
done
for ((i=0; i<${#ver1[@]}; i++))
do
if [[ -z ${ver2[i]} ]]
then
# fill empty fields in ver2 with zeros
ver2[i]=0
fi
if ((10#${ver1[i]} > 10#${ver2[i]}))
then
return 1
fi
if ((10#${ver1[i]} < 10#${ver2[i]}))
then
return 2
fi
done
return 0
function VerComp () {
if [ "$1" == "" ] || [ "$2" == "" ]; then
Logger "Bogus Vercomp values [$1] and [$2]." "WARN"
return 1
fi
if [[ $1 == $2 ]]
then
echo 0
return
fi
local IFS=.
local i ver1=($1) ver2=($2)
# fill empty fields in ver1 with zeros
for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
do
ver1[i]=0
done
for ((i=0; i<${#ver1[@]}; i++))
do
if [[ -z ${ver2[i]} ]]
then
# fill empty fields in ver2 with zeros
ver2[i]=0
fi
if ((10#${ver1[i]} > 10#${ver2[i]}))
then
echo 1
return
fi
if ((10#${ver1[i]} < 10#${ver2[i]}))
then
echo 2
return
fi
done
echo 0
return
}
function GetConfFileValue () {
@ -1906,6 +1941,29 @@ function SetConfFileValue () {
fi
}
# Function can replace [ -f /some/file* ] tests
# Modified version of http://stackoverflow.com/a/6364244/2635443
function WildcardFileExists () {
local file="${1}"
local exists=0
for f in $file; do
## Check if the glob gets expanded to existing files.
## If not, f here will be exactly the pattern above
## and the exists test will evaluate to false.
if [ -e "$f" ]; then
exists=1
break
fi
done
if [ $exists -eq 1 ]; then
echo 1
else
echo 0
fi
}
# If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap
_OFUNCTIONS_BOOTSTRAP=true
[ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1

18
install.sh
Unescape Escape View File

@ -12,7 +12,7 @@ PROGRAM_BINARY=$PROGRAM".sh"
PROGRAM_BATCH=$PROGRAM"-batch.sh"
SSH_FILTER="ssh_filter.sh"
SCRIPT_BUILD=2017032101
SCRIPT_BUILD=2017041701
## osync / obackup / pmocr / zsnap install script
## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11
@ -342,9 +342,9 @@ function CopyServiceFiles {
elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_INIT" ] && [ -d "$SERVICE_DIR_INIT" ]); then
CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$SERVICE_FILE_INIT" "755" "" "" true
QuickLogger "Created osync-srv service in [$SERVICE_DIR_INIT]."
QuickLogger "Can be activated with [service $OSYNC_SERVICE_FILE_INIT start]."
QuickLogger "Can be enabled on boot with [chkconfig $OSYNC_SERVICE_FILE_INIT on]."
QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_INIT]."
QuickLogger "Can be activated with [service $SERVICE_FILE_INIT start]."
QuickLogger "Can be enabled on boot with [chkconfig $SERVICE_FILE_INIT on]."
else
QuickLogger "Cannot define what init style is in use on this system. Skipping service file installation."
fi
@ -386,7 +386,11 @@ function RemoveFile {
function RemoveAll {
RemoveFile "$BIN_DIR/$PROGRAM_BINARY"
RemoveFile "$BIN_DIR/$PROGRAM_BATCH"
if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
RemoveFile "$BIN_DIR/$PROGRAM_BATCH"
fi
if [ ! -f "$BIN_DIR/osync.sh" ] && [ ! -f "$BIN_DIR/obackup.sh" ]; then # Check if any other program requiring ssh filter is present before removal
RemoveFile "$BIN_DIR/$SSH_FILTER"
else
@ -447,7 +451,9 @@ else
CreateDir "$BIN_DIR"
CopyExampleFiles
CopyProgram
CopyServiceFiles
if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "pmocr" ]; then
CopyServiceFiles
fi
QuickLogger "$PROGRAM installed. Use with $BIN_DIR/$PROGRAM"
if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
QuickLogger ""

107
osync.sh
Unescape Escape View File

@ -9,8 +9,8 @@ IS_STABLE=yes
_OFUNCTIONS_VERSION=2.1
_OFUNCTIONS_BUILD=2017032301
_OFUNCTIONS_VERSION=2.1.2
_OFUNCTIONS_BUILD=2017052601
_OFUNCTIONS_BOOTSTRAP=true
## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
@ -35,6 +35,9 @@ fi
## Correct output of sort command (language agnostic sorting)
export LC_ALL=C
## Default umask for file creation
umask 0077
# Standard alert mail body
MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
@ -252,7 +255,7 @@ function Logger {
return
elif [ "$level" == "VERBOSE" ]; then
if [ $_LOGGER_VERBOSE == true ]; then
_Logger "$prefix:$value" "$prefix$value"
_Logger "$prefix($level):$value" "$prefix$value"
fi
return
elif [ "$level" == "ALWAYS" ]; then
@ -612,6 +615,7 @@ function Spinner {
}
# Time control function for background processes, suitable for multiple synchronous processes
# Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2
# Also sets a global variable called HARD_MAX_EXEC_TIME_REACHED_$callerName to true if hardMaxTime is reached
@ -736,6 +740,8 @@ function WaitForTaskCompletion {
pidsArray=("${newPidsArray[@]}")
# Trivial wait time for bash to not eat up all CPU
sleep $sleepTime
done
@ -894,6 +900,7 @@ function ParallelExec {
# Trivial wait time for bash to not eat up all CPU
sleep $sleepTime
done
return $errorCount
@ -1119,6 +1126,7 @@ function GetLocalOS {
}
function GetRemoteOS {
if [ "$REMOTE_OPERATION" != "yes" ]; then
@ -1362,7 +1370,6 @@ function CheckConnectivity3rdPartyHosts {
fi
}
function RsyncPatternsAdd {
local patternType="${1}" # exclude or include
local pattern="${2}"
@ -1706,35 +1713,46 @@ function ParentPid {
# Neat version compare function found at http://stackoverflow.com/a/4025065/2635443
# Returns 0 if equal, 1 if $1 > $2 and 2 if $1 < $2
vercomp () {
if [[ $1 == $2 ]]
then
return 0
fi
local IFS=.
local i ver1=($1) ver2=($2)
# fill empty fields in ver1 with zeros
for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
do
ver1[i]=0
done
for ((i=0; i<${#ver1[@]}; i++))
do
if [[ -z ${ver2[i]} ]]
then
# fill empty fields in ver2 with zeros
ver2[i]=0
fi
if ((10#${ver1[i]} > 10#${ver2[i]}))
then
return 1
fi
if ((10#${ver1[i]} < 10#${ver2[i]}))
then
return 2
fi
done
return 0
function VerComp () {
if [ "$1" == "" ] || [ "$2" == "" ]; then
Logger "Bogus Vercomp values [$1] and [$2]." "WARN"
return 1
fi
if [[ $1 == $2 ]]
then
echo 0
return
fi
local IFS=.
local i ver1=($1) ver2=($2)
# fill empty fields in ver1 with zeros
for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
do
ver1[i]=0
done
for ((i=0; i<${#ver1[@]}; i++))
do
if [[ -z ${ver2[i]} ]]
then
# fill empty fields in ver2 with zeros
ver2[i]=0
fi
if ((10#${ver1[i]} > 10#${ver2[i]}))
then
echo 1
return
fi
if ((10#${ver1[i]} < 10#${ver2[i]}))
then
echo 2
return
fi
done
echo 0
return
}
function GetConfFileValue () {
@ -1767,6 +1785,29 @@ function SetConfFileValue () {
fi
}
# Function can replace [ -f /some/file* ] tests
# Modified version of http://stackoverflow.com/a/6364244/2635443
function WildcardFileExists () {
local file="${1}"
local exists=0
for f in $file; do
## Check if the glob gets expanded to existing files.
## If not, f here will be exactly the pattern above
## and the exists test will evaluate to false.
if [ -e "$f" ]; then
exists=1
break
fi
done
if [ $exists -eq 1 ]; then
echo 1
else
echo 0
fi
}
# If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap
_OFUNCTIONS_BOOTSTRAP=true
[ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1

Write Preview
Loading…
Cancel
Save