Archives
/
osync
mirror of https://github.com/deajan/osync
2
0
Fork 0
Code Issues Releases Wiki Activity

Rebuilt targets

Browse Source
pull/128/head
deajan 7 years ago
parent 9b8fe3774e
commit 9a0dba00e4
5 changed files with 1495 additions and 1029 deletions
Show Stats Download Patch File Download Diff File

628
dev/debug_osync.sh
Unescape Escape View File

@ -7,8 +7,8 @@
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2017 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.2.2-dev
PROGRAM_BUILD=2017112302
PROGRAM_VERSION=1.2.3-dev
PROGRAM_BUILD=2018010201
IS_STABLE=no
@ -43,8 +43,8 @@ IS_STABLE=no
# CleanUp no #__WITH_PARANOIA_DEBUG
_OFUNCTIONS_VERSION=2.1.4-rc1+
_OFUNCTIONS_BUILD=2017123002
_OFUNCTIONS_VERSION=2.2.0-dev
_OFUNCTIONS_BUILD=2018010201
_OFUNCTIONS_BOOTSTRAP=true
## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
@ -61,7 +61,7 @@ _OFUNCTIONS_BOOTSTRAP=true
#TODO: global WAIT_FOR_TASK_COMPLETION_id instead of callerName has to be backported to ParallelExec and osync / obackup / pmocr ocde
## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel
## When called from subprocesses, variable of main process can't be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID.$TSTAMP
## When called from subprocesses, variable of main process cannot be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID.$TSTAMP
if ! type "$BASH" > /dev/null; then
echo "Please run this script only with bash shell. Tested on bash >= 3.2"
@ -283,7 +283,7 @@ function Logger {
if [ "$level" == "CRITICAL" ]; then
_Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true
ERROR_ALERT=true
# ERROR_ALERT / WARN_ALERT isn't set in main when Logger is called from a subprocess. Need to keep this flag.
# ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. Need to keep this flag.
echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
return
elif [ "$level" == "ERROR" ]; then
@ -354,7 +354,7 @@ function KillChilds {
local pid="${1}" # Parent pid to kill childs
local self="${2:-false}" # Should parent be killed too ?
# Paranoid checks, we can safely assume that $pid shouldn't be 0 nor 1
# Paranoid checks, we can safely assume that $pid should not be 0 nor 1
if [ $(IsInteger "$pid") -eq 0 ] || [ "$pid" == "" ] || [ "$pid" == "0" ] || [ "$pid" == "1" ]; then
Logger "Bogus pid given [$pid]." "CRITICAL"
return 1
@ -532,13 +532,13 @@ function SendEmail {
if [ $? != 0 ]; then
Logger "Cannot send alert mail via $(type -p sendmail) !!!" "WARN"
# Don't bother try other mail systems with busybox
# Do not bother try other mail systems with busybox
return 1
else
return 0
fi
else
Logger "Sendmail not present. Won't send any mail" "WARN"
Logger "Sendmail not present. Will not send any mail" "WARN"
return 1
fi
fi
@ -673,20 +673,10 @@ function LoadConfigFile {
CONFIG_FILE="$configFile"
}
_OFUNCTIONS_SPINNER="|/-\\"
function Spinner {
if [ $_LOGGER_SILENT == true ] || [ "$_LOGGER_ERR_ONLY" == true ]; then
return 0
else
printf " [%c] \b\b\b\b\b\b" "$_OFUNCTIONS_SPINNER"
#printf "\b\b\b\b\b\b"
_OFUNCTIONS_SPINNER=${_OFUNCTIONS_SPINNER#?}${_OFUNCTIONS_SPINNER%%???}
return 0
fi
}
# Quick and dirty performance logger only used for debugging
function _PerfProfiler { #__WITH_PARANOIA_DEBUG
local perfString #__WITH_PARANOIA_DEBUG
local i #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
perfString=$(ps -p $$ -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan) #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
@ -701,28 +691,142 @@ function _PerfProfiler { #__WITH_PARANOIA_DEBUG
Logger "PerfProfiler:\n$perfString" "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
} #__WITH_PARANOIA_DEBUG
_OFUNCTIONS_SPINNER="|/-\\"
function Spinner {
if [ $_LOGGER_SILENT == true ] || [ "$_LOGGER_ERR_ONLY" == true ]; then
return 0
else
printf " [%c] \b\b\b\b\b\b" "$_OFUNCTIONS_SPINNER"
#printf "\b\b\b\b\b\b"
_OFUNCTIONS_SPINNER=${_OFUNCTIONS_SPINNER#?}${_OFUNCTIONS_SPINNER%%???}
return 0
fi
}
# Time control function for background processes, suitable for multiple synchronous processes
# Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2
# Also sets a global variable called HARD_MAX_EXEC_TIME_REACHED_$callerName to true if hardMaxTime is reached
# WaitForTaskCompletion function emulation, now uses ExecTasks
function WaitForTaskCompletion {
local pids="${1}"
local softMaxTime="${2:-0}"
local hardMaxTime="${3:-0}"
local sleepTime="${4:-.05}"
local keepLogging="${5:-0}"
local counting="${6:-true}"
local spinner="${7:-true}"
local noErrorLog="${8:-false}"
local id="${9-base}"
ExecTasks "$id" 0 0 "$softMaxTime" "$hardMaxTime" "$sleepTime" "$keepLogging" "$counting" "$spinner" "$noErrorlog" false 1 "$pids" "" ""
}
# Standard wait $! emulation would be WaitForTaskCompletion $! 0 0 1 0 true false true false
# ParallelExec function emulation, now uses ExecTasks
function ParallelExec {
local numberOfProcesses="${1}"
local commandsArg="${2}"
local readFromFile="${3:-false}"
local softMaxTime="${4:-0}"
local hardMaxTime="${5:-0}"
local sleepTime="${6:-.05}"
local keepLogging="${7:-0}"
local counting="${8:-true}"
local spinner="${9:-false}"
local noErrorLog="${10:-false}"
ExecTasks "base" 0 0 "$softMaxTime" "$hardMaxTime" "$sleepTime" "$keepLogging" "$counting" "$spinner" "$noErrorLog" false 3 "$commandsArg" "" "$numberOfProcesses"
}
function WaitForTaskCompletion {
local pids="${1}" # pids to wait for, separated by semi-colon
local softMaxTime="${2:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${3:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local sleepTime="${4:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local keepLogging="${5:-0}" # Every keepLogging seconds, an alive log message is send. Setting this value to zero disables any alive logging.
local counting="${6:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${7:-true}" # Show spinner (true), don't show anything (false)
local noErrorLog="${8:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true)
local id="${9-base}" # Optional id in order to get $WAIT_FOR_TASK_COMPLETION_id global variable
## Main asynchronous execution function
## This function can monitor given pids as background processes, and stop them if max execution time is reached. Suitable for multiple synchronous processes.
## It can also take a list of commands to execute in parallel, and stop them if max execution time is reached.
## Function has 8 execution modes
# WaitForTaskCompletion mode: Monitor given pids as background processes, stop them if max execution time is reached. Suitaable for multiple synhronous processes.
# 1 : WaitForTaskCompletion, semi-colon separated list of pids to monitor
# 2 : WaitForTaskCompletion, list of pids to monior, from file, one per line
# Example of improved wait $! emulation
# ExecTasks "some_identifier" 0 0 0 0 1 1800 true false true true 1 $!
# Example: monitor two sleep processes, warn if execution time is higher than 10 seconds, stop after 20 seconds
# sleep 15 &
# pid=$!
# sleep 20 &
# pid2=$!
# ExecTasks "some_identifier" 0 0 10 20 1 1800 true true false false 1 "$pid;$pid2"
# ParallelExecMode: Take list of commands to execute in parallel, stop them if max execution time is reached.
# Also take optional conditional arguments to verifiy before execution main commands. Conditional command exit code 0 means ready to execute. Other exit codes will ignore/postpone main command.
# 3 : ParallelExec, semi-colon separated list of commands to execute in parallel, no conditions
# 4 : ParallelExec, semi-colon separated list of commands to execute in parallel , semi-colon separated list of conditional commands, ignoring main commands on condition failures
# 5 : ParallelExec, semi-colon separated list of commands, semi-colon separated list of conditional commands, postponing main commands on condition failures
# 6 : ParallelExec, list of commands from file, one per line, no conditions
# 7 : ParallelExec, list of commands from file, one per line, list of conditional commands from file, one per line, ignoring main commands on condition failures
# 8 : ParallelExec, list of commands from file, one per line, list of conditional commands from file, one per line, postponing main commands on condition failures
# Exmaple: execute four du commands, only if directories exist, warn if execution takes more than 300 seconds, stop if takes longer than 900 seconds. Execute max 3 commands in parallel.
# commands="du -csh /var;du -csh /etc;du -csh /home;du -csh /usr"
# conditions="[ -d /var ];[ -d /etc ];[ -d /home];[ -d /usr]"
# ExecTasks "some_identifier" 0 0 300 900 1 1800 true true false false 4 "$commands" "$conditions" 3
## ofunctions.sh subfunction requirements:
## Spinner
## Logger
## JoinString
## KillChilds
function ExecTasks {
local id="${1:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id
#TODO: not implemented yet
local softPerProcessTime="${2:-0}"
local hardPerProcessTime="${3:-0}"
local softMaxTime="${4:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${5:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local sleepTime="${6:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local keepLogging="${7:-0}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging.
local counting="${8:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${9:-true}" # Show spinner (true), do not show anything (false)
local noTimeErrorLog="${10:-false}" # Log errors when reaching soft / hard max time (false), do not log errors on those triggers (true)
local noErrorLogAtAll="${11:-false}" # Do not log errros at all (false)
local execTasksMode="${12:-1}" # In which mode the function should work, see above
local mainInput="${13}" # Contains list of pids / commands or filepath to list of pids / commands
local auxInput="${14}" # Contains list of conditional commands or filepath to list of conditional commands
local numberOfProcesses="${15:-2}" # Number of simultaneous commands to run in ParallExec mode
local i
local callerName="${FUNCNAME[1]}"
Logger "${FUNCNAME[0]} called by [${FUNCNAME[0]} < ${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} ...]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
__CheckArguments 8 $# "$@" #__WITH_PARANOIA_DEBUG
Logger "${FUNCNAME[0]} called in $execTasksMode mode by [${FUNCNAME[0]} < ${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} ...]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
__CheckArguments 13-15 $# "$@" #__WITH_PARANOIA_DEBUG
# Since ExecTasks takes up to 15 arguments, do a quick preflight check in DEBUG mode
if [ "$_DEBUG" == "yes" ]; then
declare -a booleans=(counting spinner noTimeErrorLog noErrorLogAtAll)
for i in "${num_vars[@]}"; do
test="if [ $i != false ] && [ $i != true ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
eval "$test"
done
declare -a integers=(softPerProcessTime hardPerProcessTime softMaxTime hardMaxTime keepLogging execTasksMode numberOfProcesses)
for i in "${integers[@]}"; do
test="if [ $(IsNumericExpand \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
eval "$test"
done
fi
# ParallelExec specific variables
local auxCount # Number of conditional commands
local commandsArray=() # Array containing commands
local commandsConditionArray=() # Array containing conditional commands
local currentCommand # Variable containing currently processed command
local currentCommandCondition # Variable containing currently processed conditional command
local commandsArrayPid # Array containing pids of commands currently run
local postPoneIfConditionFails # Boolean to check if command needs to be postponed if condition command fails
# Common variables
local pid # Current pid working on
local pidState # State of the process
local mainItemCount # number of given items (pids or commands)
local readFromFile # Should we read pids / commands from a file (true)
local counter=0
local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
@ -730,28 +834,131 @@ function WaitForTaskCompletion {
local retval=0 # return value of monitored pid process
local errorcount=0 # Number of pids that finished with errors
local pid # Current pid working on
local pidCount # number of given pids
local pidState # State of the process
local pidsArray # Array of currently running pids
local newPidsArray # New array of currently running pids
local newPidsArray # New array of currently running pids for next iteration
local pidsTimeArray # Array containing execution begin time of pids
local executeCommand # Boolean to check if currentCommand can be executed given a condition
local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG
if [ $counting == true ]; then # If counting == false _SOFT_ALERT should be a global value so no more than one soft alert is shown
local _SOFT_ALERT=false # Does a soft alert need to be triggered, if yes, send an alert once
fi
local functionMode
IFS=';' read -a pidsArray <<< "$pids"
pidCount=${#pidsArray[@]}
if [ $counting == true ]; then
local softAlert=false # Does a soft alert need to be triggered, if yes, send an alert once
else
local softAlert=false
fi
# Set global var default
# Initialise global variable
eval "WAIT_FOR_TASK_COMPLETION_$id=\"\""
eval "HARD_MAX_EXEC_TIME_REACHED_$id=false"
while [ ${#pidsArray[@]} -gt 0 ]; do
case $execTasksMode in
1)
IFS=';' read -r -a pidsArray <<< "$mainInput"
mainItemCount=${#pidsArray[@]}
readFromFile=false
functionMode=WaitForTaskCompletion
# Force while condition to be true
counter=$mainItemCount
Logger "Running ${FUNCNAME[0]} mode 1 for [$mainItemCount] pids." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
2)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else
mainItemCount=0
Logger "Cannot read file [$mainInput]." "WARN"
fi
functionMode=WaitForTaskCompletion
# Force while condition to be true
counter=$mainItemCount
Logger "Running ${FUNCNAME[0]} mode 2 for [$mainItemCount] pids." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
3)
IFS=';' read -r -a commandsArray <<< "$mainInput"
mainItemCount=${#commandsArray[@]}
readFromFile=false
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 3 for [$mainItemCount] commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
4)
IFS=';' read -r -a commandsArray <<< "$mainInput"
mainItemCount=${#commandsArray[@]}
IFS=';' read -r -a commandsConditionArray <<< "$auxInput"
auxItemCount=${#commandsConditionArray[@]}
readFromFile=false
postPoneIfConditionFails=false
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 4 for [$mainItemCount] commands and [$auxItemCount] conditional commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
5)
IFS=';' read -r -a commandsArray <<< "$mainInput"
mainItemCount=${#commandsArray[@]}
IFS=';' read -r -a commandsConditionArray <<< "$auxInput"
auxItemCount=${#commandsConditionArray[@]}
readFromFile=false
postPoneIfConditionFails=true
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 5 for [$mainItemCount] commands and [$auxItemCount] conditional commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
6)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else
mainItemCount=0
Logger "Cannot read file [$mainInput]." "WARN"
fi
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 6 for [$mainItemCount] commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
7)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else
mainItemCount=0
Logger "Cannot read file [$mainInput]." "WARN"
fi
if [ -f "$auxInput" ]; then
auxCount=$(wc -l < "$auxInput")
else
auxCount=0
Logger "Cannot read file [$auxInput]." "WARN"
fi
postPoneIfConditionFails=false
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 7 for [$mainItemCount] commands and [$auxItemCount] conditional commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
8)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else
mainItemCount=0
Logger "Cannot read file [$mainInput]." "WARN"
fi
if [ -f "$auxInput" ]; then
auxCount=$(wc -l < "$auxInput")
else
auxCount=0
Logger "Cannot read file [$auxInput]." "WARN"
fi
postPoneIfConditionFails=true
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 8 for [$mainItemCount] commands and [$auxItemCount] conditional commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
*)
Logger "Unknown exec mode for ${FUNCNAME}." "CRITICAL"
exit 1
esac
#while [ ${#pidsArray[@]} -gt 0 ]; do
# The counter -lt mainItemCount has to be false for WaitForTaskCompletion modes
while [ ${#pidsArray[@]} -gt 0 ] || [ $counter -lt $mainItemCount ]; do
newPidsArray=()
if [ $spinner == true ]; then
@ -765,23 +972,27 @@ function WaitForTaskCompletion {
if [ $keepLogging -ne 0 ]; then
if [ $((($exec_time + 1) % $keepLogging)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1s
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1 second
log_ttime=$exec_time
if [ $functionMode == "Wait" ]; then
Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
elif [ $functionMode == "ParallelExec" ]; then
Logger "There are $((mainItemCount-counter)) / $mainItemCount tasks in the queue. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
fi
fi
fi
fi
if [ $exec_time -gt $softMaxTime ]; then
if [ "$_SOFT_ALERT" != true ] && [ $softMaxTime -ne 0 ] && [ $noErrorLog != true ]; then
if [ "$softAlert" != true ] && [ $softMaxTime -ne 0 ] && [ $noTimeErrorLog != true ]; then
Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
_SOFT_ALERT=true
softAlert=true
SendAlert true
fi
fi
if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
if [ $noErrorLog != true ]; then
if [ $noTimeErrorLog != true ]; then
Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
fi
for pid in "${pidsArray[@]}"; do
@ -793,205 +1004,95 @@ function WaitForTaskCompletion {
fi
errorcount=$((errorcount+1))
done
if [ $noErrorLog != true ]; then
if [ $noTimeErrorLog != true ]; then
SendAlert true
fi
eval "HARD_MAX_EXEC_TIME_REACHED_$id=true"
if [ $functionMode == "WaitForTaskCompletion" ]; then
return $errorcount
elif [ $functionMode == "ParallelExec" ]; then
return $((mainItemCount - counter + ${#pidsArray[@]}))
fi
for pid in "${pidsArray[@]}"; do
if [ $(IsInteger $pid) -eq 1 ]; then
if kill -0 $pid > /dev/null 2>&1; then
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
pidState="$(eval $PROCESS_STATE_CMD)"
if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
newPidsArray+=($pid)
fi
else
# pid is dead, get it's exit code from wait command
wait $pid
retval=$?
if [ $retval -ne 0 ]; then
Logger "${FUNCNAME[0]} called by [$callerName] finished monitoring [$pid] with exitcode [$retval]." "DEBUG"
errorcount=$((errorcount+1))
# Welcome to variable variable bash hell
if [ "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_$id\")" == "" ]; then
eval "WAIT_FOR_TASK_COMPLETION_$id=\"$pid:$retval\""
else
eval "WAIT_FOR_TASK_COMPLETION_$id=\";$pid:$retval\""
fi
fi
fi
hasPids=true ##__WITH_PARANOIA_DEBUG
fi
done
if [ $hasPids == false ]; then ##__WITH_PARANOIA_DEBUG
Logger "No valable pids given." "ERROR" ##__WITH_PARANOIA_DEBUG
fi ##__WITH_PARANOIA_DEBUG
pidsArray=("${newPidsArray[@]}")
# Trivial wait time for bash to not eat up all CPU
sleep $sleepTime
if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG
_PerfProfiler ##__WITH_PARANOIA_DEBUG
fi ##__WITH_PARANOIA_DEBUG
done
Logger "${FUNCNAME[0]} ended for [$callerName] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
# Return exit code if only one process was monitored, else return number of errors
# As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value
if [ $pidCount -eq 1 ]; then
return $retval
# The following execution bloc is only needed in ParallelExec mode since WaitForTaskCompletion does not execute commands, but only monitors them
if [ $functionMode == "ParallelExec" ]; then
while [ $counter -lt "$mainItemCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do
if [ $readFromFile == true ]; then
currentCommand=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$mainInput")
currentCommandCondition=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$auxInput")
else
return $errorcount
fi
}
# Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs
# Returns the number of non zero exit codes from commands
# Use cmd1;cmd2;cmd3 syntax for small sets, use file for large command sets
# Only 2 first arguments are mandatory
# Sets a global variable called HARD_MAX_EXEC_TIME_REACHED to true if hardMaxTime is reached
function ParallelExec {
local numberOfProcesses="${1}" # Number of simultaneous commands to run
local commandsArg="${2}" # Semi-colon separated list of commands, or path to file containing one command per line
local readFromFile="${3:-false}" # commandsArg is a file (true), or a string (false)
local softMaxTime="${4:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${5:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local sleepTime="${6:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local keepLogging="${7:-0}" # Every keepLogging seconds, an alive log message is send. Setting this value to zero disables any alive logging.
local counting="${8:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${9:-false}" # Show spinner (true), don't show spinner (false)
local noErrorLog="${10:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true)
local callerName="${FUNCNAME[1]}"
Logger "${FUNCNAME[0]} called by [${FUNCNAME[0]} < ${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} ...]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
__CheckArguments 2-10 $# "$@" #__WITH_PARANOIA_DEBUG
local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function
local commandCount
local command
local pid
local counter=0
local commandsArray
local pidsArray
local newPidsArray
local retval
local errorCount=0
local pidState
local commandsArrayPid
local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG
# Set global var default
eval "HARD_MAX_EXEC_TIME_REACHED_$callerName=false"
if [ $counting == true ]; then # If counting == false _SOFT_ALERT should be a global value so no more than one soft alert is shown
local _SOFT_ALERT=false # Does a soft alert need to be triggered, if yes, send an alert once
currentCommand="${commandArray[$counter]}"
currentCommandCondition="${commandConditionArray[$counter]}"
fi
executeCommand=false
if [ $auxCount -ne 0 ]; then
Logger "Checking condition [$currentCommandCondition] for command [$currentCommand]." "DEBUG"
eval "$currentCommandCondition" &
ExecTasks "subConditionCheck" 0 0 1800 3600 1 $KEEP_LOGGING true true true true 1 $!
if [ $? -ne 0 ]; then
if [ $postPoneIfConditionFails == true ]; then
Logger "Condition not met for command [$currentCommand]. Postponing command." "NOTICE"
if [ $readFromFile == true ]; then
if [ -f "$commandsArg" ]; then
commandCount=$(wc -l < "$commandsArg")
else
commandCount=0
fi
# TODO: we should not write to the original file, but create a copy instead we can write postponed commands to
echo "$currentCommand" >> "$mainInput"
echo "$currentCommandCondition" >> "$auxInput"
else
IFS=';' read -r -a commandsArray <<< "$commandsArg"
commandCount=${#commandsArray[@]}
fi
Logger "Runnning $commandCount commands in $numberOfProcesses simultaneous processes." "DEBUG"
while [ $counter -lt "$commandCount" ] || [ ${#pidsArray[@]} -gt 0 ]; do
if [ $spinner == true ]; then
Spinner
commansdArray+=($currentCommand)
commandsConditionArray+=($currentCommandCondition)
fi
mainItemCount=$((mainItemCount+1))
if [ $counting == true ]; then
exec_time=$((SECONDS - seconds_begin))
# Trivial sleeptime so postponed commands will not stack too fast
sleep $sleepTime
else
exec_time=$SECONDS
fi
if [ $keepLogging -ne 0 ]; then
if [ $((($exec_time + 1) % $keepLogging)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1s
log_ttime=$exec_time
Logger "There are $((commandCount-counter)) / $commandCount tasks in the queue. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
Logger "Condition not met for command [$currentCommand]. Ignoring command." "NOTICE"
fi
fi
fi
if [ $exec_time -gt $softMaxTime ]; then
if [ "$_SOFT_ALERT" != true ] && [ $softMaxTime -ne 0 ] && [ $noErrorLog != true ]; then
Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
_SOFT_ALERT=true
SendAlert true
fi
fi
if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
if [ $noErrorLog != true ]; then
Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
fi
for pid in "${pidsArray[@]}"; do
KillChilds $pid true
if [ $? == 0 ]; then
Logger "Task with pid [$pid] stopped successfully." "NOTICE"
else
Logger "Could not stop task with pid [$pid]." "ERROR"
fi
done
if [ $noErrorLog != true ]; then
SendAlert true
executeCommand=true
fi
eval "HARD_MAX_EXEC_TIME_REACHED_$callerName=true"
# Return the number of commands that haven't run / finished run
return $((commandCount - counter + ${#pidsArray[@]}))
fi
while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do
if [ $readFromFile == true ]; then
command=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$commandsArg")
else
command="${commandsArray[$counter]}"
executeCommand=true
fi
Logger "Running command [$command]." "DEBUG"
eval "$command" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$callerName.$SCRIPT_PID.$TSTAMP" 2>&1 &
if [ $executeCommand == true ]; then
Logger "Running command [$currentCommand]." "DEBUG"
eval "$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$callerName.$SCRIPT_PID.$TSTAMP" 2>&1 &
pid=$!
pidsArray+=($pid)
commandsArrayPid[$pid]="$command"
commandsArrayPid[$pid]="$currentCommand"
#TODO not implemented
pidsTimeArray[$pid]=$((SECONDS - seconds_begin))
fi
counter=$((counter+1))
done
fi
newPidsArray=()
for pid in "${pidsArray[@]}"; do
if [ $(IsInteger $pid) -eq 1 ]; then
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
if kill -0 $pid > /dev/null 2>&1; then
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
pidState="$(eval $PROCESS_STATE_CMD)"
if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
#TODO: implement pidsTimeArray[$pid] check here
newPidsArray+=($pid)
fi
else
# pid is dead, get it's exit code from wait command
# pid is dead, get its exit code from wait command
wait $pid
retval=$?
if [ $retval -ne 0 ]; then
Logger "Command [${commandsArrayPid[$pid]}] failed with exit code [$retval]." "ERROR"
errorCount=$((errorCount+1))
Logger "${FUNCNAME[0]} called by [$callerName] finished monitoring [$pid] [$currentCommad] with exitcode [$retval]." "DEBUG"
errorcount=$((errorcount+1))
# Welcome to variable variable bash hell
if [ "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_$id\")" == "" ]; then
eval "WAIT_FOR_TASK_COMPLETION_$id=\"$pid:$retval\""
else
eval "WAIT_FOR_TASK_COMPLETION_$id=\";$pid:$retval\""
fi
else
Logger "${FUNCNAME[0]} called by [$callerName] finished monitoring [$pid] [$currentCommand] with exitcode [$retval]." "DEBUG"
fi
fi
hasPids=true ##__WITH_PARANOIA_DEBUG
@ -1011,7 +1112,19 @@ function ParallelExec {
fi ##__WITH_PARANOIA_DEBUG
done
return $errorCount
Logger "${FUNCNAME[0]} ended for [$callerName] using [$mainItemCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
# Return exit code if only one process was monitored, else return number of errors
# As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value
if [ $noErrorLogAtAll == true ]; then
return 0
fi
if [ $mainItemCount -eq 1 ]; then
return $retval
else
return $errorcount
fi
}
function CleanUp {
@ -1165,7 +1278,7 @@ function GetLocalOS {
local localOsName
local localOsVer
# There's no good way to tell if currently running in BusyBox shell. Using sluggish way.
# There is no good way to tell if currently running in BusyBox shell. Using sluggish way.
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox"
else
@ -1311,7 +1424,7 @@ function GetOs {
local osInfo="/etc/os-release"
# There's no good way to tell if currently running in BusyBox shell. Using sluggish way.
# There is no good way to tell if currently running in BusyBox shell. Using sluggish way.
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox"
else
@ -1402,7 +1515,7 @@ function RunLocalCommand {
Logger "Running command [$command] on local host." "NOTICE"
eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 &
WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE"
@ -1443,7 +1556,7 @@ function RunRemoteCommand {
cmd=$SSH_CMD' "env LC_ALL=C env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1'
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE"
@ -1477,7 +1590,7 @@ function RunBeforeHook {
pids="$pids;$!"
fi
if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 0 0 true true false false 1 $pids
fi
}
@ -1496,7 +1609,7 @@ function RunAfterHook {
pids="$pids;$!"
fi
if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 0 0 true true false false 1 $pids
fi
}
@ -1509,7 +1622,7 @@ function CheckConnectivityRemoteHost {
if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then
eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" &
WaitForTaskCompletion $! 60 180 $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 60 180 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval != 0 ]; then
Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN"
@ -1524,6 +1637,7 @@ function CheckConnectivity3rdPartyHosts {
local remote3rdPartySuccess
local retval
local i
if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug #__WITH_PARANOIA_DEBUG
@ -1532,7 +1646,7 @@ function CheckConnectivity3rdPartyHosts {
for i in $REMOTE_3RD_PARTY_HOSTS
do
eval "$PING_CMD $i > /dev/null 2>&1" &
WaitForTaskCompletion $! 180 360 $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 180 360 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval != 0 ]; then
Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE"
@ -1697,7 +1811,7 @@ function PostInit {
}
function SetCompression {
## Busybox fix (Termux xz command doesn't support compression at all)
## Busybox fix (Termux xz command does not support compression at all)
if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then
compressionString=""
if type gzip > /dev/null 2>&1
@ -1787,7 +1901,7 @@ function InitLocalOSDependingSettings {
SetCompression
}
# Gets executed regardless of the need of remote connections. It's just that this code needs to get executed after we know if there is a remote os, and if yes, which one
# Gets executed regardless of the need of remote connections. It is just that this code needs to get executed after we know if there is a remote os, and if yes, which one
function InitRemoteOSDependingSettings {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
@ -1880,7 +1994,7 @@ function InitRemoteOSDependingSettings {
RSYNC_ARGS=$RSYNC_ARGS" --whole-file"
fi
# Set compression options again after we know what remote OS we're dealing with
# Set compression options again after we know what remote OS we are dealing with
SetCompression
}
@ -2471,7 +2585,7 @@ function CheckReplicas {
_CheckReplicasRemote "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" &
pids="$pids;$!"
fi
WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $pids
retval=$?
if [ $retval -ne 0 ]; then
Logger "Cancelling task." "CRITICAL" $retval
@ -2821,7 +2935,8 @@ function HandleLocks {
# Do not bother checking for locks when FORCE_UNLOCK is set
if [ $FORCE_UNLOCK == true ]; then
overwrite=true
else
fi
_HandleLocksLocal "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}" "${INITIATOR[$__lockFile]}" "${INITIATOR[$__type]}" $overwrite &
initiatorPid=$!
pids="$initiatorPid"
@ -2834,14 +2949,12 @@ function HandleLocks {
targetPid=$!
pids="$pids;$targetPid"
fi
WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
ExecTasks "bulldog" 0 0 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $pids
retval=$?
if [ $retval -ne 0 ]; then
echo $WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
for pid in "${pidArray[@]}"; do
pid=${pid%:*}
echo "$pid - $initiatorPid - $targetPid"
if [ "$pid" == "$initiatorPid" ]; then
initiatorLockSuccess=false
elif [ "$pid" == "$targetPid" ]; then
@ -2849,17 +2962,22 @@ function HandleLocks {
fi
done
#WIP refactor the following
if [ $initiatorLockSuccess == true ]; then
INITIATOR_LOCK_FILE_EXISTS=true
fi
if [ $targetLockSuccess == true ]; then
TARGET_LOCK_FILE_EXISTS=true
fi
Logger "Cancelling task." "CRITICAL" $retval
exit 1
fi
if [ $initiatorLockSuccess == true ]; then
INITIATOR_LOCK_FILE_EXISTS=true
fi
if [ $targetLockSuccess == true ]; then
TARGET_LOCK_FILE_EXISTS=true
fi
}
@ -2936,7 +3054,7 @@ function UnlockReplicas {
fi
if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $pids
fi
}
@ -3269,7 +3387,7 @@ function syncAttrs {
fi
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
eval "$rsyncCmd"
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
retval=$?
if [ $retval -ne 0 ] && [ $retval -ne 24 ]; then
@ -3303,7 +3421,7 @@ function syncAttrs {
_getFileCtimeMtimeRemote "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-cleaned.$SCRIPT_PID.$TSTAMP" "$RUN_DIR/$PROGRAM.ctime_mtime___.${TARGET[$__type]}.$SCRIPT_PID.$TSTAMP" &
pids="$pids;$!"
fi
WaitForTaskCompletion $pids $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $pids
retval=$?
if [ $retval -ne 0 ]; then
Logger "Getting ctime attributes failed." "CRITICAL" $retval
@ -3356,7 +3474,7 @@ function syncAttrs {
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
eval "$rsyncCmd"
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
retval=$?
if [ $retval -ne 0 ] && [ $retval -ne 24 ]; then
@ -3918,7 +4036,7 @@ function Sync {
targetPid="$!"
fi
WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false
@ -3963,7 +4081,7 @@ function Sync {
targetPid="$!"
fi
WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false
@ -4012,7 +4130,7 @@ function Sync {
targetPid="$!"
fi
WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false
@ -4055,7 +4173,7 @@ function Sync {
if [ "$resumeInitiator" == "${SYNC_ACTION[3]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[3]}" ]; then
if [ "$LOG_CONFLICTS" == "yes" ]; then
conflictList "${INITIATOR[$__timestampCurrentFile]}" "${INITIATOR[$__timestampAfterFileNoSuffix]}" "${INITIATOR[$__conflictListFile]}" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__targetLastActionFile]}"
@ -4079,7 +4197,7 @@ function Sync {
if [ "$resumeInitiator" == "${SYNC_ACTION[4]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[4]}" ]; then
if [[ "$RSYNC_ATTR_ARGS" == *"-X"* ]] || [[ "$RSYNC_ATTR_ARGS" == *"-A"* ]]; then
syncAttrs "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__targetLastActionFile]}"
@ -4104,7 +4222,7 @@ function Sync {
if [ "$CONFLICT_PREVALANCE" == "${TARGET[$__type]}" ]; then
if [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}"
resumeTarget="${SYNC_ACTION[5]}"
@ -4116,7 +4234,7 @@ function Sync {
fi
if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}"
resumeInitiator="${SYNC_ACTION[5]}"
@ -4129,7 +4247,7 @@ function Sync {
else
if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}"
resumeInitiator="${SYNC_ACTION[5]}"
@ -4141,7 +4259,7 @@ function Sync {
fi
if [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}"
resumeTarget="${SYNC_ACTION[5]}"
@ -4166,7 +4284,7 @@ function Sync {
targetPid="$!"
fi
WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false
@ -4212,7 +4330,7 @@ function Sync {
targetPid="$!"
fi
WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false
@ -4261,7 +4379,7 @@ function Sync {
targetPid="$!"
fi
WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false
@ -4422,7 +4540,7 @@ function SoftDelete {
_SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__backupDir]}" $CONFLICT_BACKUP_DAYS "conflict backup" &
pids="$pids;$!"
fi
WaitForTaskCompletion $pids $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$pids"
if [ $? -ne 0 ] && [ "$(eval echo \"\$HARD_MAX_EXEC_TIME_REACHED_${FUNCNAME[0]}\")" == true ]; then
exit 1
fi
@ -4440,7 +4558,7 @@ function SoftDelete {
_SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__deleteDir]}" $SOFT_DELETE_DAYS "softdelete" &
pids="$pids;$!"
fi
WaitForTaskCompletion $pids $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$pids"
if [ $? -ne 0 ] && [ "$(eval echo \"\$HARD_MAX_EXEC_TIME_REACHED_${FUNCNAME[0]}\")" == true ]; then
exit 1
fi
@ -4815,7 +4933,7 @@ function SyncOnChanges {
if [ "$LOCAL_OS" == "MacOSX" ]; then
fswatch $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude "$OSYNC_DIR" -1 "$INITIATOR_SYNC_DIR" > /dev/null &
# Mac fswatch doesn't have timeout switch, replacing wait $! with WaitForTaskCompletion without warning nor spinner and increased SLEEP_TIME to avoid cpu hogging. This sims wait $! with timeout
WaitForTaskCompletion $! 0 $MAX_WAIT 1 0 true false true
ExecTasks "MonitorMacOSXWait" 0 0 0 $MAX_WAIT 1 0 true false true false $!
else
inotifywait $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude "$OSYNC_DIR" -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" &
wait $!

568
dev/debug_osync_target_helper.sh
Unescape Escape View File

@ -8,8 +8,8 @@ PROGRAM_BUILD=2017061901
IS_STABLE=no
_OFUNCTIONS_VERSION=2.1.4-rc1+
_OFUNCTIONS_BUILD=2017123002
_OFUNCTIONS_VERSION=2.2.0-dev
_OFUNCTIONS_BUILD=2018010201
_OFUNCTIONS_BOOTSTRAP=true
## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
@ -26,7 +26,7 @@ _OFUNCTIONS_BOOTSTRAP=true
#TODO: global WAIT_FOR_TASK_COMPLETION_id instead of callerName has to be backported to ParallelExec and osync / obackup / pmocr ocde
## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel
## When called from subprocesses, variable of main process can't be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID.$TSTAMP
## When called from subprocesses, variable of main process cannot be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID.$TSTAMP
if ! type "$BASH" > /dev/null; then
echo "Please run this script only with bash shell. Tested on bash >= 3.2"
@ -248,7 +248,7 @@ function Logger {
if [ "$level" == "CRITICAL" ]; then
_Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true
ERROR_ALERT=true
# ERROR_ALERT / WARN_ALERT isn't set in main when Logger is called from a subprocess. Need to keep this flag.
# ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. Need to keep this flag.
echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
return
elif [ "$level" == "ERROR" ]; then
@ -319,7 +319,7 @@ function KillChilds {
local pid="${1}" # Parent pid to kill childs
local self="${2:-false}" # Should parent be killed too ?
# Paranoid checks, we can safely assume that $pid shouldn't be 0 nor 1
# Paranoid checks, we can safely assume that $pid should not be 0 nor 1
if [ $(IsInteger "$pid") -eq 0 ] || [ "$pid" == "" ] || [ "$pid" == "0" ] || [ "$pid" == "1" ]; then
Logger "Bogus pid given [$pid]." "CRITICAL"
return 1
@ -497,13 +497,13 @@ function SendEmail {
if [ $? != 0 ]; then
Logger "Cannot send alert mail via $(type -p sendmail) !!!" "WARN"
# Don't bother try other mail systems with busybox
# Do not bother try other mail systems with busybox
return 1
else
return 0
fi
else
Logger "Sendmail not present. Won't send any mail" "WARN"
Logger "Sendmail not present. Will not send any mail" "WARN"
return 1
fi
fi
@ -638,20 +638,10 @@ function LoadConfigFile {
CONFIG_FILE="$configFile"
}
_OFUNCTIONS_SPINNER="|/-\\"
function Spinner {
if [ $_LOGGER_SILENT == true ] || [ "$_LOGGER_ERR_ONLY" == true ]; then
return 0
else
printf " [%c] \b\b\b\b\b\b" "$_OFUNCTIONS_SPINNER"
#printf "\b\b\b\b\b\b"
_OFUNCTIONS_SPINNER=${_OFUNCTIONS_SPINNER#?}${_OFUNCTIONS_SPINNER%%???}
return 0
fi
}
# Quick and dirty performance logger only used for debugging
function _PerfProfiler { #__WITH_PARANOIA_DEBUG
local perfString #__WITH_PARANOIA_DEBUG
local i #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
perfString=$(ps -p $$ -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan) #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
@ -666,28 +656,142 @@ function _PerfProfiler { #__WITH_PARANOIA_DEBUG
Logger "PerfProfiler:\n$perfString" "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
} #__WITH_PARANOIA_DEBUG
_OFUNCTIONS_SPINNER="|/-\\"
function Spinner {
if [ $_LOGGER_SILENT == true ] || [ "$_LOGGER_ERR_ONLY" == true ]; then
return 0
else
printf " [%c] \b\b\b\b\b\b" "$_OFUNCTIONS_SPINNER"
#printf "\b\b\b\b\b\b"
_OFUNCTIONS_SPINNER=${_OFUNCTIONS_SPINNER#?}${_OFUNCTIONS_SPINNER%%???}
return 0
fi
}
# Time control function for background processes, suitable for multiple synchronous processes
# Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2
# Also sets a global variable called HARD_MAX_EXEC_TIME_REACHED_$callerName to true if hardMaxTime is reached
# WaitForTaskCompletion function emulation, now uses ExecTasks
function WaitForTaskCompletion {
local pids="${1}"
local softMaxTime="${2:-0}"
local hardMaxTime="${3:-0}"
local sleepTime="${4:-.05}"
local keepLogging="${5:-0}"
local counting="${6:-true}"
local spinner="${7:-true}"
local noErrorLog="${8:-false}"
local id="${9-base}"
# Standard wait $! emulation would be WaitForTaskCompletion $! 0 0 1 0 true false true false
ExecTasks "$id" 0 0 "$softMaxTime" "$hardMaxTime" "$sleepTime" "$keepLogging" "$counting" "$spinner" "$noErrorlog" false 1 "$pids" "" ""
}
function WaitForTaskCompletion {
local pids="${1}" # pids to wait for, separated by semi-colon
local softMaxTime="${2:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${3:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local sleepTime="${4:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local keepLogging="${5:-0}" # Every keepLogging seconds, an alive log message is send. Setting this value to zero disables any alive logging.
local counting="${6:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${7:-true}" # Show spinner (true), don't show anything (false)
local noErrorLog="${8:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true)
local id="${9-base}" # Optional id in order to get $WAIT_FOR_TASK_COMPLETION_id global variable
# ParallelExec function emulation, now uses ExecTasks
function ParallelExec {
local numberOfProcesses="${1}"
local commandsArg="${2}"
local readFromFile="${3:-false}"
local softMaxTime="${4:-0}"
local hardMaxTime="${5:-0}"
local sleepTime="${6:-.05}"
local keepLogging="${7:-0}"
local counting="${8:-true}"
local spinner="${9:-false}"
local noErrorLog="${10:-false}"
ExecTasks "base" 0 0 "$softMaxTime" "$hardMaxTime" "$sleepTime" "$keepLogging" "$counting" "$spinner" "$noErrorLog" false 3 "$commandsArg" "" "$numberOfProcesses"
}
## Main asynchronous execution function
## This function can monitor given pids as background processes, and stop them if max execution time is reached. Suitable for multiple synchronous processes.
## It can also take a list of commands to execute in parallel, and stop them if max execution time is reached.
## Function has 8 execution modes
# WaitForTaskCompletion mode: Monitor given pids as background processes, stop them if max execution time is reached. Suitaable for multiple synhronous processes.
# 1 : WaitForTaskCompletion, semi-colon separated list of pids to monitor
# 2 : WaitForTaskCompletion, list of pids to monior, from file, one per line
# Example of improved wait $! emulation
# ExecTasks "some_identifier" 0 0 0 0 1 1800 true false true true 1 $!
# Example: monitor two sleep processes, warn if execution time is higher than 10 seconds, stop after 20 seconds
# sleep 15 &
# pid=$!
# sleep 20 &
# pid2=$!
# ExecTasks "some_identifier" 0 0 10 20 1 1800 true true false false 1 "$pid;$pid2"
# ParallelExecMode: Take list of commands to execute in parallel, stop them if max execution time is reached.
# Also take optional conditional arguments to verifiy before execution main commands. Conditional command exit code 0 means ready to execute. Other exit codes will ignore/postpone main command.
# 3 : ParallelExec, semi-colon separated list of commands to execute in parallel, no conditions
# 4 : ParallelExec, semi-colon separated list of commands to execute in parallel , semi-colon separated list of conditional commands, ignoring main commands on condition failures
# 5 : ParallelExec, semi-colon separated list of commands, semi-colon separated list of conditional commands, postponing main commands on condition failures
# 6 : ParallelExec, list of commands from file, one per line, no conditions
# 7 : ParallelExec, list of commands from file, one per line, list of conditional commands from file, one per line, ignoring main commands on condition failures
# 8 : ParallelExec, list of commands from file, one per line, list of conditional commands from file, one per line, postponing main commands on condition failures
# Exmaple: execute four du commands, only if directories exist, warn if execution takes more than 300 seconds, stop if takes longer than 900 seconds. Execute max 3 commands in parallel.
# commands="du -csh /var;du -csh /etc;du -csh /home;du -csh /usr"
# conditions="[ -d /var ];[ -d /etc ];[ -d /home];[ -d /usr]"
# ExecTasks "some_identifier" 0 0 300 900 1 1800 true true false false 4 "$commands" "$conditions" 3
## ofunctions.sh subfunction requirements:
## Spinner
## Logger
## JoinString
## KillChilds
function ExecTasks {
local id="${1:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id
#TODO: not implemented yet
local softPerProcessTime="${2:-0}"
local hardPerProcessTime="${3:-0}"
local softMaxTime="${4:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${5:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local sleepTime="${6:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local keepLogging="${7:-0}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging.
local counting="${8:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${9:-true}" # Show spinner (true), do not show anything (false)
local noTimeErrorLog="${10:-false}" # Log errors when reaching soft / hard max time (false), do not log errors on those triggers (true)
local noErrorLogAtAll="${11:-false}" # Do not log errros at all (false)
local execTasksMode="${12:-1}" # In which mode the function should work, see above
local mainInput="${13}" # Contains list of pids / commands or filepath to list of pids / commands
local auxInput="${14}" # Contains list of conditional commands or filepath to list of conditional commands
local numberOfProcesses="${15:-2}" # Number of simultaneous commands to run in ParallExec mode
local i
local callerName="${FUNCNAME[1]}"
Logger "${FUNCNAME[0]} called by [${FUNCNAME[0]} < ${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} ...]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
__CheckArguments 8 $# "$@" #__WITH_PARANOIA_DEBUG
Logger "${FUNCNAME[0]} called in $execTasksMode mode by [${FUNCNAME[0]} < ${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} ...]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
__CheckArguments 13-15 $# "$@" #__WITH_PARANOIA_DEBUG
# Since ExecTasks takes up to 15 arguments, do a quick preflight check in DEBUG mode
if [ "$_DEBUG" == "yes" ]; then
declare -a booleans=(counting spinner noTimeErrorLog noErrorLogAtAll)
for i in "${num_vars[@]}"; do
test="if [ $i != false ] && [ $i != true ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
eval "$test"
done
declare -a integers=(softPerProcessTime hardPerProcessTime softMaxTime hardMaxTime keepLogging execTasksMode numberOfProcesses)
for i in "${integers[@]}"; do
test="if [ $(IsNumericExpand \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
eval "$test"
done
fi
# ParallelExec specific variables
local auxCount # Number of conditional commands
local commandsArray=() # Array containing commands
local commandsConditionArray=() # Array containing conditional commands
local currentCommand # Variable containing currently processed command
local currentCommandCondition # Variable containing currently processed conditional command
local commandsArrayPid # Array containing pids of commands currently run
local postPoneIfConditionFails # Boolean to check if command needs to be postponed if condition command fails
# Common variables
local pid # Current pid working on
local pidState # State of the process
local mainItemCount # number of given items (pids or commands)
local readFromFile # Should we read pids / commands from a file (true)
local counter=0
local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
@ -695,28 +799,131 @@ function WaitForTaskCompletion {
local retval=0 # return value of monitored pid process
local errorcount=0 # Number of pids that finished with errors
local pid # Current pid working on
local pidCount # number of given pids
local pidState # State of the process
local pidsArray # Array of currently running pids
local newPidsArray # New array of currently running pids
local newPidsArray # New array of currently running pids for next iteration
local pidsTimeArray # Array containing execution begin time of pids
local executeCommand # Boolean to check if currentCommand can be executed given a condition
local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG
if [ $counting == true ]; then # If counting == false _SOFT_ALERT should be a global value so no more than one soft alert is shown
local _SOFT_ALERT=false # Does a soft alert need to be triggered, if yes, send an alert once
fi
local functionMode
IFS=';' read -a pidsArray <<< "$pids"
pidCount=${#pidsArray[@]}
if [ $counting == true ]; then
local softAlert=false # Does a soft alert need to be triggered, if yes, send an alert once
else
local softAlert=false
fi
# Set global var default
# Initialise global variable
eval "WAIT_FOR_TASK_COMPLETION_$id=\"\""
eval "HARD_MAX_EXEC_TIME_REACHED_$id=false"
while [ ${#pidsArray[@]} -gt 0 ]; do
case $execTasksMode in
1)
IFS=';' read -r -a pidsArray <<< "$mainInput"
mainItemCount=${#pidsArray[@]}
readFromFile=false
functionMode=WaitForTaskCompletion
# Force while condition to be true
counter=$mainItemCount
Logger "Running ${FUNCNAME[0]} mode 1 for [$mainItemCount] pids." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
2)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else
mainItemCount=0
Logger "Cannot read file [$mainInput]." "WARN"
fi
functionMode=WaitForTaskCompletion
# Force while condition to be true
counter=$mainItemCount
Logger "Running ${FUNCNAME[0]} mode 2 for [$mainItemCount] pids." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
3)
IFS=';' read -r -a commandsArray <<< "$mainInput"
mainItemCount=${#commandsArray[@]}
readFromFile=false
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 3 for [$mainItemCount] commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
4)
IFS=';' read -r -a commandsArray <<< "$mainInput"
mainItemCount=${#commandsArray[@]}
IFS=';' read -r -a commandsConditionArray <<< "$auxInput"
auxItemCount=${#commandsConditionArray[@]}
readFromFile=false
postPoneIfConditionFails=false
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 4 for [$mainItemCount] commands and [$auxItemCount] conditional commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
5)
IFS=';' read -r -a commandsArray <<< "$mainInput"
mainItemCount=${#commandsArray[@]}
IFS=';' read -r -a commandsConditionArray <<< "$auxInput"
auxItemCount=${#commandsConditionArray[@]}
readFromFile=false
postPoneIfConditionFails=true
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 5 for [$mainItemCount] commands and [$auxItemCount] conditional commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
6)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else
mainItemCount=0
Logger "Cannot read file [$mainInput]." "WARN"
fi
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 6 for [$mainItemCount] commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
7)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else
mainItemCount=0
Logger "Cannot read file [$mainInput]." "WARN"
fi
if [ -f "$auxInput" ]; then
auxCount=$(wc -l < "$auxInput")
else
auxCount=0
Logger "Cannot read file [$auxInput]." "WARN"
fi
postPoneIfConditionFails=false
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 7 for [$mainItemCount] commands and [$auxItemCount] conditional commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
8)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else
mainItemCount=0
Logger "Cannot read file [$mainInput]." "WARN"
fi
if [ -f "$auxInput" ]; then
auxCount=$(wc -l < "$auxInput")
else
auxCount=0
Logger "Cannot read file [$auxInput]." "WARN"
fi
postPoneIfConditionFails=true
functionMode=ParallelExec
Logger "Running ${FUNCNAME[0]} mode 8 for [$mainItemCount] commands and [$auxItemCount] conditional commands." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
;;
*)
Logger "Unknown exec mode for ${FUNCNAME}." "CRITICAL"
exit 1
esac
#while [ ${#pidsArray[@]} -gt 0 ]; do
# The counter -lt mainItemCount has to be false for WaitForTaskCompletion modes
while [ ${#pidsArray[@]} -gt 0 ] || [ $counter -lt $mainItemCount ]; do
newPidsArray=()
if [ $spinner == true ]; then
@ -730,23 +937,27 @@ function WaitForTaskCompletion {
if [ $keepLogging -ne 0 ]; then
if [ $((($exec_time + 1) % $keepLogging)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1s
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1 second
log_ttime=$exec_time
if [ $functionMode == "Wait" ]; then
Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
elif [ $functionMode == "ParallelExec" ]; then
Logger "There are $((mainItemCount-counter)) / $mainItemCount tasks in the queue. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
fi
fi
fi
fi
if [ $exec_time -gt $softMaxTime ]; then
if [ "$_SOFT_ALERT" != true ] && [ $softMaxTime -ne 0 ] && [ $noErrorLog != true ]; then
if [ "$softAlert" != true ] && [ $softMaxTime -ne 0 ] && [ $noTimeErrorLog != true ]; then
Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
_SOFT_ALERT=true
softAlert=true
SendAlert true
fi
fi
if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
if [ $noErrorLog != true ]; then
if [ $noTimeErrorLog != true ]; then
Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
fi
for pid in "${pidsArray[@]}"; do
@ -758,205 +969,95 @@ function WaitForTaskCompletion {
fi
errorcount=$((errorcount+1))
done
if [ $noErrorLog != true ]; then
if [ $noTimeErrorLog != true ]; then
SendAlert true
fi
eval "HARD_MAX_EXEC_TIME_REACHED_$id=true"
if [ $functionMode == "WaitForTaskCompletion" ]; then
return $errorcount
fi
for pid in "${pidsArray[@]}"; do
if [ $(IsInteger $pid) -eq 1 ]; then
if kill -0 $pid > /dev/null 2>&1; then
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
pidState="$(eval $PROCESS_STATE_CMD)"
if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
newPidsArray+=($pid)
fi
else
# pid is dead, get it's exit code from wait command
wait $pid
retval=$?
if [ $retval -ne 0 ]; then
Logger "${FUNCNAME[0]} called by [$callerName] finished monitoring [$pid] with exitcode [$retval]." "DEBUG"
errorcount=$((errorcount+1))
# Welcome to variable variable bash hell
if [ "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_$id\")" == "" ]; then
eval "WAIT_FOR_TASK_COMPLETION_$id=\"$pid:$retval\""
else
eval "WAIT_FOR_TASK_COMPLETION_$id=\";$pid:$retval\""
fi
elif [ $functionMode == "ParallelExec" ]; then
return $((mainItemCount - counter + ${#pidsArray[@]}))
fi
fi
hasPids=true ##__WITH_PARANOIA_DEBUG
fi
done
if [ $hasPids == false ]; then ##__WITH_PARANOIA_DEBUG
Logger "No valable pids given." "ERROR" ##__WITH_PARANOIA_DEBUG
fi ##__WITH_PARANOIA_DEBUG
pidsArray=("${newPidsArray[@]}")
# Trivial wait time for bash to not eat up all CPU
sleep $sleepTime
if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG
_PerfProfiler ##__WITH_PARANOIA_DEBUG
fi ##__WITH_PARANOIA_DEBUG
done
Logger "${FUNCNAME[0]} ended for [$callerName] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
# Return exit code if only one process was monitored, else return number of errors
# As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value
if [ $pidCount -eq 1 ]; then
return $retval
# The following execution bloc is only needed in ParallelExec mode since WaitForTaskCompletion does not execute commands, but only monitors them
if [ $functionMode == "ParallelExec" ]; then
while [ $counter -lt "$mainItemCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do
if [ $readFromFile == true ]; then
currentCommand=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$mainInput")
currentCommandCondition=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$auxInput")
else
return $errorcount
fi
}
# Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs
# Returns the number of non zero exit codes from commands
# Use cmd1;cmd2;cmd3 syntax for small sets, use file for large command sets
# Only 2 first arguments are mandatory
# Sets a global variable called HARD_MAX_EXEC_TIME_REACHED to true if hardMaxTime is reached
function ParallelExec {
local numberOfProcesses="${1}" # Number of simultaneous commands to run
local commandsArg="${2}" # Semi-colon separated list of commands, or path to file containing one command per line
local readFromFile="${3:-false}" # commandsArg is a file (true), or a string (false)
local softMaxTime="${4:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${5:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local sleepTime="${6:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local keepLogging="${7:-0}" # Every keepLogging seconds, an alive log message is send. Setting this value to zero disables any alive logging.
local counting="${8:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${9:-false}" # Show spinner (true), don't show spinner (false)
local noErrorLog="${10:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true)
local callerName="${FUNCNAME[1]}"
Logger "${FUNCNAME[0]} called by [${FUNCNAME[0]} < ${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} ...]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
__CheckArguments 2-10 $# "$@" #__WITH_PARANOIA_DEBUG
local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function
local commandCount
local command
local pid
local counter=0
local commandsArray
local pidsArray
local newPidsArray
local retval
local errorCount=0
local pidState
local commandsArrayPid
local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG
# Set global var default
eval "HARD_MAX_EXEC_TIME_REACHED_$callerName=false"
if [ $counting == true ]; then # If counting == false _SOFT_ALERT should be a global value so no more than one soft alert is shown
local _SOFT_ALERT=false # Does a soft alert need to be triggered, if yes, send an alert once
currentCommand="${commandArray[$counter]}"
currentCommandCondition="${commandConditionArray[$counter]}"
fi
executeCommand=false
if [ $auxCount -ne 0 ]; then
Logger "Checking condition [$currentCommandCondition] for command [$currentCommand]." "DEBUG"
eval "$currentCommandCondition" &
ExecTasks "subConditionCheck" 0 0 1800 3600 1 $KEEP_LOGGING true true true true 1 $!
if [ $? -ne 0 ]; then
if [ $postPoneIfConditionFails == true ]; then
Logger "Condition not met for command [$currentCommand]. Postponing command." "NOTICE"
if [ $readFromFile == true ]; then
if [ -f "$commandsArg" ]; then
commandCount=$(wc -l < "$commandsArg")
else
commandCount=0
fi
# TODO: we should not write to the original file, but create a copy instead we can write postponed commands to
echo "$currentCommand" >> "$mainInput"
echo "$currentCommandCondition" >> "$auxInput"
else
IFS=';' read -r -a commandsArray <<< "$commandsArg"
commandCount=${#commandsArray[@]}
commansdArray+=($currentCommand)
commandsConditionArray+=($currentCommandCondition)
fi
mainItemCount=$((mainItemCount+1))
Logger "Runnning $commandCount commands in $numberOfProcesses simultaneous processes." "DEBUG"
while [ $counter -lt "$commandCount" ] || [ ${#pidsArray[@]} -gt 0 ]; do
if [ $spinner == true ]; then
Spinner
fi
if [ $counting == true ]; then
exec_time=$((SECONDS - seconds_begin))
# Trivial sleeptime so postponed commands will not stack too fast
sleep $sleepTime
else
exec_time=$SECONDS
Logger "Condition not met for command [$currentCommand]. Ignoring command." "NOTICE"
fi
if [ $keepLogging -ne 0 ]; then
if [ $((($exec_time + 1) % $keepLogging)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1s
log_ttime=$exec_time
Logger "There are $((commandCount-counter)) / $commandCount tasks in the queue. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
fi
fi
fi
if [ $exec_time -gt $softMaxTime ]; then
if [ "$_SOFT_ALERT" != true ] && [ $softMaxTime -ne 0 ] && [ $noErrorLog != true ]; then
Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
_SOFT_ALERT=true
SendAlert true
fi
fi
if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
if [ $noErrorLog != true ]; then
Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
fi
for pid in "${pidsArray[@]}"; do
KillChilds $pid true
if [ $? == 0 ]; then
Logger "Task with pid [$pid] stopped successfully." "NOTICE"
else
Logger "Could not stop task with pid [$pid]." "ERROR"
fi
done
if [ $noErrorLog != true ]; then
SendAlert true
fi
eval "HARD_MAX_EXEC_TIME_REACHED_$callerName=true"
# Return the number of commands that haven't run / finished run
return $((commandCount - counter + ${#pidsArray[@]}))
executeCommand=true
fi
while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do
if [ $readFromFile == true ]; then
command=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$commandsArg")
else
command="${commandsArray[$counter]}"
executeCommand=true
fi
Logger "Running command [$command]." "DEBUG"
eval "$command" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$callerName.$SCRIPT_PID.$TSTAMP" 2>&1 &
if [ $executeCommand == true ]; then
Logger "Running command [$currentCommand]." "DEBUG"
eval "$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$callerName.$SCRIPT_PID.$TSTAMP" 2>&1 &
pid=$!
pidsArray+=($pid)
commandsArrayPid[$pid]="$command"
commandsArrayPid[$pid]="$currentCommand"
#TODO not implemented
pidsTimeArray[$pid]=$((SECONDS - seconds_begin))
fi
counter=$((counter+1))
done
fi
newPidsArray=()
for pid in "${pidsArray[@]}"; do
if [ $(IsInteger $pid) -eq 1 ]; then
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
if kill -0 $pid > /dev/null 2>&1; then
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
pidState="$(eval $PROCESS_STATE_CMD)"
if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
#TODO: implement pidsTimeArray[$pid] check here
newPidsArray+=($pid)
fi
else
# pid is dead, get it's exit code from wait command
# pid is dead, get its exit code from wait command
wait $pid
retval=$?
if [ $retval -ne 0 ]; then
Logger "Command [${commandsArrayPid[$pid]}] failed with exit code [$retval]." "ERROR"
errorCount=$((errorCount+1))
Logger "${FUNCNAME[0]} called by [$callerName] finished monitoring [$pid] [$currentCommad] with exitcode [$retval]." "DEBUG"
errorcount=$((errorcount+1))
# Welcome to variable variable bash hell
if [ "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_$id\")" == "" ]; then
eval "WAIT_FOR_TASK_COMPLETION_$id=\"$pid:$retval\""
else
eval "WAIT_FOR_TASK_COMPLETION_$id=\";$pid:$retval\""
fi
else
Logger "${FUNCNAME[0]} called by [$callerName] finished monitoring [$pid] [$currentCommand] with exitcode [$retval]." "DEBUG"
fi
fi
hasPids=true ##__WITH_PARANOIA_DEBUG
@ -976,7 +1077,19 @@ function ParallelExec {
fi ##__WITH_PARANOIA_DEBUG
done
return $errorCount
Logger "${FUNCNAME[0]} ended for [$callerName] using [$mainItemCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
# Return exit code if only one process was monitored, else return number of errors
# As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value
if [ $noErrorLogAtAll == true ]; then
return 0
fi
if [ $mainItemCount -eq 1 ]; then
return $retval
else
return $errorcount
fi
}
function CleanUp {
@ -1130,7 +1243,7 @@ function GetLocalOS {
local localOsName
local localOsVer
# There's no good way to tell if currently running in BusyBox shell. Using sluggish way.
# There is no good way to tell if currently running in BusyBox shell. Using sluggish way.
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox"
else
@ -1276,7 +1389,7 @@ function GetOs {
local osInfo="/etc/os-release"
# There's no good way to tell if currently running in BusyBox shell. Using sluggish way.
# There is no good way to tell if currently running in BusyBox shell. Using sluggish way.
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox"
else
@ -1367,7 +1480,7 @@ function RunLocalCommand {
Logger "Running command [$command] on local host." "NOTICE"
eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 &
WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE"
@ -1408,7 +1521,7 @@ function RunRemoteCommand {
cmd=$SSH_CMD' "env LC_ALL=C env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1'
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE"
@ -1442,7 +1555,7 @@ function RunBeforeHook {
pids="$pids;$!"
fi
if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 0 0 true true false false 1 $pids
fi
}
@ -1461,7 +1574,7 @@ function RunAfterHook {
pids="$pids;$!"
fi
if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 0 0 true true false false 1 $pids
fi
}
@ -1474,7 +1587,7 @@ function CheckConnectivityRemoteHost {
if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then
eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" &
WaitForTaskCompletion $! 60 180 $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 60 180 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval != 0 ]; then
Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN"
@ -1489,6 +1602,7 @@ function CheckConnectivity3rdPartyHosts {
local remote3rdPartySuccess
local retval
local i
if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug #__WITH_PARANOIA_DEBUG
@ -1497,7 +1611,7 @@ function CheckConnectivity3rdPartyHosts {
for i in $REMOTE_3RD_PARTY_HOSTS
do
eval "$PING_CMD $i > /dev/null 2>&1" &
WaitForTaskCompletion $! 180 360 $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 180 360 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval != 0 ]; then
Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE"
@ -1662,7 +1776,7 @@ function PostInit {
}
function SetCompression {
## Busybox fix (Termux xz command doesn't support compression at all)
## Busybox fix (Termux xz command does not support compression at all)
if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then
compressionString=""
if type gzip > /dev/null 2>&1
@ -1752,7 +1866,7 @@ function InitLocalOSDependingSettings {
SetCompression
}
# Gets executed regardless of the need of remote connections. It's just that this code needs to get executed after we know if there is a remote os, and if yes, which one
# Gets executed regardless of the need of remote connections. It is just that this code needs to get executed after we know if there is a remote os, and if yes, which one
function InitRemoteOSDependingSettings {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
@ -1845,7 +1959,7 @@ function InitRemoteOSDependingSettings {
RSYNC_ARGS=$RSYNC_ARGS" --whole-file"
fi
# Set compression options again after we know what remote OS we're dealing with
# Set compression options again after we know what remote OS we are dealing with
SetCompression
}

2
install.sh
Unescape Escape View File

@ -126,7 +126,7 @@ function GetLocalOS {
local localOsName
local localOsVer
# There's no good way to tell if currently running in BusyBox shell. Using sluggish way.
# There is no good way to tell if currently running in BusyBox shell. Using sluggish way.
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox"
else

581
osync.sh
Unescape Escape View File

@ -7,15 +7,15 @@
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2017 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.2.2-dev
PROGRAM_BUILD=2017112302
PROGRAM_VERSION=1.2.3-dev
PROGRAM_BUILD=2018010201
IS_STABLE=no
_OFUNCTIONS_VERSION=2.1.4-rc1+
_OFUNCTIONS_BUILD=2017123002
_OFUNCTIONS_VERSION=2.2.0-dev
_OFUNCTIONS_BUILD=2018010201
_OFUNCTIONS_BOOTSTRAP=true
## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
@ -32,7 +32,7 @@ _OFUNCTIONS_BOOTSTRAP=true
#TODO: global WAIT_FOR_TASK_COMPLETION_id instead of callerName has to be backported to ParallelExec and osync / obackup / pmocr ocde
## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel
## When called from subprocesses, variable of main process can't be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID.$TSTAMP
## When called from subprocesses, variable of main process cannot be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID.$TSTAMP
if ! type "$BASH" > /dev/null; then
echo "Please run this script only with bash shell. Tested on bash >= 3.2"
@ -244,7 +244,7 @@ function Logger {
if [ "$level" == "CRITICAL" ]; then
_Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true
ERROR_ALERT=true
# ERROR_ALERT / WARN_ALERT isn't set in main when Logger is called from a subprocess. Need to keep this flag.
# ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. Need to keep this flag.
echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
return
elif [ "$level" == "ERROR" ]; then
@ -310,7 +310,7 @@ function KillChilds {
local pid="${1}" # Parent pid to kill childs
local self="${2:-false}" # Should parent be killed too ?
# Paranoid checks, we can safely assume that $pid shouldn't be 0 nor 1
# Paranoid checks, we can safely assume that $pid should not be 0 nor 1
if [ $(IsInteger "$pid") -eq 0 ] || [ "$pid" == "" ] || [ "$pid" == "0" ] || [ "$pid" == "1" ]; then
Logger "Bogus pid given [$pid]." "CRITICAL"
return 1
@ -484,13 +484,13 @@ function SendEmail {
if [ $? != 0 ]; then
Logger "Cannot send alert mail via $(type -p sendmail) !!!" "WARN"
# Don't bother try other mail systems with busybox
# Do not bother try other mail systems with busybox
return 1
else
return 0
fi
else
Logger "Sendmail not present. Won't send any mail" "WARN"
Logger "Sendmail not present. Will not send any mail" "WARN"
return 1
fi
fi
@ -624,6 +624,8 @@ function LoadConfigFile {
CONFIG_FILE="$configFile"
}
# Quick and dirty performance logger only used for debugging
_OFUNCTIONS_SPINNER="|/-\\"
function Spinner {
if [ $_LOGGER_SILENT == true ] || [ "$_LOGGER_ERR_ONLY" == true ]; then
@ -636,27 +638,128 @@ function Spinner {
fi
}
# WaitForTaskCompletion function emulation, now uses ExecTasks
function WaitForTaskCompletion {
local pids="${1}"
local softMaxTime="${2:-0}"
local hardMaxTime="${3:-0}"
local sleepTime="${4:-.05}"
local keepLogging="${5:-0}"
local counting="${6:-true}"
local spinner="${7:-true}"
local noErrorLog="${8:-false}"
local id="${9-base}"
ExecTasks "$id" 0 0 "$softMaxTime" "$hardMaxTime" "$sleepTime" "$keepLogging" "$counting" "$spinner" "$noErrorlog" false 1 "$pids" "" ""
}
# ParallelExec function emulation, now uses ExecTasks
function ParallelExec {
local numberOfProcesses="${1}"
local commandsArg="${2}"
local readFromFile="${3:-false}"
local softMaxTime="${4:-0}"
local hardMaxTime="${5:-0}"
local sleepTime="${6:-.05}"
local keepLogging="${7:-0}"
local counting="${8:-true}"
local spinner="${9:-false}"
local noErrorLog="${10:-false}"
ExecTasks "base" 0 0 "$softMaxTime" "$hardMaxTime" "$sleepTime" "$keepLogging" "$counting" "$spinner" "$noErrorLog" false 3 "$commandsArg" "" "$numberOfProcesses"
}
# Time control function for background processes, suitable for multiple synchronous processes
# Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2
# Also sets a global variable called HARD_MAX_EXEC_TIME_REACHED_$callerName to true if hardMaxTime is reached
# Standard wait $! emulation would be WaitForTaskCompletion $! 0 0 1 0 true false true false
## Main asynchronous execution function
## This function can monitor given pids as background processes, and stop them if max execution time is reached. Suitable for multiple synchronous processes.
## It can also take a list of commands to execute in parallel, and stop them if max execution time is reached.
## Function has 8 execution modes
# WaitForTaskCompletion mode: Monitor given pids as background processes, stop them if max execution time is reached. Suitaable for multiple synhronous processes.
# 1 : WaitForTaskCompletion, semi-colon separated list of pids to monitor
# 2 : WaitForTaskCompletion, list of pids to monior, from file, one per line
# Example of improved wait $! emulation
# ExecTasks "some_identifier" 0 0 0 0 1 1800 true false true true 1 $!
# Example: monitor two sleep processes, warn if execution time is higher than 10 seconds, stop after 20 seconds
# sleep 15 &
# pid=$!
# sleep 20 &
# pid2=$!
# ExecTasks "some_identifier" 0 0 10 20 1 1800 true true false false 1 "$pid;$pid2"
# ParallelExecMode: Take list of commands to execute in parallel, stop them if max execution time is reached.
# Also take optional conditional arguments to verifiy before execution main commands. Conditional command exit code 0 means ready to execute. Other exit codes will ignore/postpone main command.
# 3 : ParallelExec, semi-colon separated list of commands to execute in parallel, no conditions
# 4 : ParallelExec, semi-colon separated list of commands to execute in parallel , semi-colon separated list of conditional commands, ignoring main commands on condition failures
# 5 : ParallelExec, semi-colon separated list of commands, semi-colon separated list of conditional commands, postponing main commands on condition failures
# 6 : ParallelExec, list of commands from file, one per line, no conditions
# 7 : ParallelExec, list of commands from file, one per line, list of conditional commands from file, one per line, ignoring main commands on condition failures
# 8 : ParallelExec, list of commands from file, one per line, list of conditional commands from file, one per line, postponing main commands on condition failures
# Exmaple: execute four du commands, only if directories exist, warn if execution takes more than 300 seconds, stop if takes longer than 900 seconds. Execute max 3 commands in parallel.
# commands="du -csh /var;du -csh /etc;du -csh /home;du -csh /usr"
# conditions="[ -d /var ];[ -d /etc ];[ -d /home];[ -d /usr]"
# ExecTasks "some_identifier" 0 0 300 900 1 1800 true true false false 4 "$commands" "$conditions" 3
## ofunctions.sh subfunction requirements:
## Spinner
## Logger
## JoinString
## KillChilds
function ExecTasks {
local id="${1:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id
#TODO: not implemented yet
local softPerProcessTime="${2:-0}"
local hardPerProcessTime="${3:-0}"
local softMaxTime="${4:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${5:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local sleepTime="${6:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local keepLogging="${7:-0}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging.
local counting="${8:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${9:-true}" # Show spinner (true), do not show anything (false)
local noTimeErrorLog="${10:-false}" # Log errors when reaching soft / hard max time (false), do not log errors on those triggers (true)
local noErrorLogAtAll="${11:-false}" # Do not log errros at all (false)
local execTasksMode="${12:-1}" # In which mode the function should work, see above
local mainInput="${13}" # Contains list of pids / commands or filepath to list of pids / commands
local auxInput="${14}" # Contains list of conditional commands or filepath to list of conditional commands
local numberOfProcesses="${15:-2}" # Number of simultaneous commands to run in ParallExec mode
function WaitForTaskCompletion {
local pids="${1}" # pids to wait for, separated by semi-colon
local softMaxTime="${2:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${3:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local sleepTime="${4:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local keepLogging="${5:-0}" # Every keepLogging seconds, an alive log message is send. Setting this value to zero disables any alive logging.
local counting="${6:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${7:-true}" # Show spinner (true), don't show anything (false)
local noErrorLog="${8:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true)
local id="${9-base}" # Optional id in order to get $WAIT_FOR_TASK_COMPLETION_id global variable
local i
local callerName="${FUNCNAME[1]}"
# Since ExecTasks takes up to 15 arguments, do a quick preflight check in DEBUG mode
if [ "$_DEBUG" == "yes" ]; then
declare -a booleans=(counting spinner noTimeErrorLog noErrorLogAtAll)
for i in "${num_vars[@]}"; do
test="if [ $i != false ] && [ $i != true ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
eval "$test"
done
declare -a integers=(softPerProcessTime hardPerProcessTime softMaxTime hardMaxTime keepLogging execTasksMode numberOfProcesses)
for i in "${integers[@]}"; do
test="if [ $(IsNumericExpand \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
eval "$test"
done
fi
# ParallelExec specific variables
local auxCount # Number of conditional commands
local commandsArray=() # Array containing commands
local commandsConditionArray=() # Array containing conditional commands
local currentCommand # Variable containing currently processed command
local currentCommandCondition # Variable containing currently processed conditional command
local commandsArrayPid # Array containing pids of commands currently run
local postPoneIfConditionFails # Boolean to check if command needs to be postponed if condition command fails
# Common variables
local pid # Current pid working on
local pidState # State of the process
local mainItemCount # number of given items (pids or commands)
local readFromFile # Should we read pids / commands from a file (true)
local counter=0
local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
@ -664,27 +767,122 @@ function WaitForTaskCompletion {
local retval=0 # return value of monitored pid process
local errorcount=0 # Number of pids that finished with errors
local pid # Current pid working on
local pidCount # number of given pids
local pidState # State of the process
local pidsArray # Array of currently running pids
local newPidsArray # New array of currently running pids
local newPidsArray # New array of currently running pids for next iteration
local pidsTimeArray # Array containing execution begin time of pids
local executeCommand # Boolean to check if currentCommand can be executed given a condition
if [ $counting == true ]; then # If counting == false _SOFT_ALERT should be a global value so no more than one soft alert is shown
local _SOFT_ALERT=false # Does a soft alert need to be triggered, if yes, send an alert once
fi
local functionMode
IFS=';' read -a pidsArray <<< "$pids"
pidCount=${#pidsArray[@]}
if [ $counting == true ]; then
local softAlert=false # Does a soft alert need to be triggered, if yes, send an alert once
else
local softAlert=false
fi
# Set global var default
# Initialise global variable
eval "WAIT_FOR_TASK_COMPLETION_$id=\"\""
eval "HARD_MAX_EXEC_TIME_REACHED_$id=false"
while [ ${#pidsArray[@]} -gt 0 ]; do
case $execTasksMode in
1)
IFS=';' read -r -a pidsArray <<< "$mainInput"
mainItemCount=${#pidsArray[@]}
readFromFile=false
functionMode=WaitForTaskCompletion
# Force while condition to be true
counter=$mainItemCount
;;
2)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else
mainItemCount=0
Logger "Cannot read file [$mainInput]." "WARN"
fi
functionMode=WaitForTaskCompletion
# Force while condition to be true
counter=$mainItemCount
;;
3)
IFS=';' read -r -a commandsArray <<< "$mainInput"
mainItemCount=${#commandsArray[@]}
readFromFile=false
functionMode=ParallelExec
;;
4)
IFS=';' read -r -a commandsArray <<< "$mainInput"
mainItemCount=${#commandsArray[@]}
IFS=';' read -r -a commandsConditionArray <<< "$auxInput"
auxItemCount=${#commandsConditionArray[@]}
readFromFile=false
postPoneIfConditionFails=false
functionMode=ParallelExec
;;
5)
IFS=';' read -r -a commandsArray <<< "$mainInput"
mainItemCount=${#commandsArray[@]}
IFS=';' read -r -a commandsConditionArray <<< "$auxInput"
auxItemCount=${#commandsConditionArray[@]}
readFromFile=false
postPoneIfConditionFails=true
functionMode=ParallelExec
;;
6)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else
mainItemCount=0
Logger "Cannot read file [$mainInput]." "WARN"
fi
functionMode=ParallelExec
;;
7)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else
mainItemCount=0
Logger "Cannot read file [$mainInput]." "WARN"
fi
if [ -f "$auxInput" ]; then
auxCount=$(wc -l < "$auxInput")
else
auxCount=0
Logger "Cannot read file [$auxInput]." "WARN"
fi
postPoneIfConditionFails=false
functionMode=ParallelExec
;;
8)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else
mainItemCount=0
Logger "Cannot read file [$mainInput]." "WARN"
fi
if [ -f "$auxInput" ]; then
auxCount=$(wc -l < "$auxInput")
else
auxCount=0
Logger "Cannot read file [$auxInput]." "WARN"
fi
postPoneIfConditionFails=true
functionMode=ParallelExec
;;
*)
Logger "Unknown exec mode for ${FUNCNAME}." "CRITICAL"
exit 1
esac
#while [ ${#pidsArray[@]} -gt 0 ]; do
# The counter -lt mainItemCount has to be false for WaitForTaskCompletion modes
while [ ${#pidsArray[@]} -gt 0 ] || [ $counter -lt $mainItemCount ]; do
newPidsArray=()
if [ $spinner == true ]; then
@ -698,23 +896,27 @@ function WaitForTaskCompletion {
if [ $keepLogging -ne 0 ]; then
if [ $((($exec_time + 1) % $keepLogging)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1s
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1 second
log_ttime=$exec_time
if [ $functionMode == "Wait" ]; then
Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
elif [ $functionMode == "ParallelExec" ]; then
Logger "There are $((mainItemCount-counter)) / $mainItemCount tasks in the queue. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
fi
fi
fi
fi
if [ $exec_time -gt $softMaxTime ]; then
if [ "$_SOFT_ALERT" != true ] && [ $softMaxTime -ne 0 ] && [ $noErrorLog != true ]; then
if [ "$softAlert" != true ] && [ $softMaxTime -ne 0 ] && [ $noTimeErrorLog != true ]; then
Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
_SOFT_ALERT=true
softAlert=true
SendAlert true
fi
fi
if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
if [ $noErrorLog != true ]; then
if [ $noTimeErrorLog != true ]; then
Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
fi
for pid in "${pidsArray[@]}"; do
@ -726,194 +928,95 @@ function WaitForTaskCompletion {
fi
errorcount=$((errorcount+1))
done
if [ $noErrorLog != true ]; then
if [ $noTimeErrorLog != true ]; then
SendAlert true
fi
eval "HARD_MAX_EXEC_TIME_REACHED_$id=true"
if [ $functionMode == "WaitForTaskCompletion" ]; then
return $errorcount
fi
for pid in "${pidsArray[@]}"; do
if [ $(IsInteger $pid) -eq 1 ]; then
if kill -0 $pid > /dev/null 2>&1; then
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
pidState="$(eval $PROCESS_STATE_CMD)"
if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
newPidsArray+=($pid)
fi
else
# pid is dead, get it's exit code from wait command
wait $pid
retval=$?
if [ $retval -ne 0 ]; then
Logger "${FUNCNAME[0]} called by [$callerName] finished monitoring [$pid] with exitcode [$retval]." "DEBUG"
errorcount=$((errorcount+1))
# Welcome to variable variable bash hell
if [ "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_$id\")" == "" ]; then
eval "WAIT_FOR_TASK_COMPLETION_$id=\"$pid:$retval\""
else
eval "WAIT_FOR_TASK_COMPLETION_$id=\";$pid:$retval\""
elif [ $functionMode == "ParallelExec" ]; then
return $((mainItemCount - counter + ${#pidsArray[@]}))
fi
fi
fi
fi
done
pidsArray=("${newPidsArray[@]}")
# Trivial wait time for bash to not eat up all CPU
sleep $sleepTime
done
# Return exit code if only one process was monitored, else return number of errors
# As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value
if [ $pidCount -eq 1 ]; then
return $retval
# The following execution bloc is only needed in ParallelExec mode since WaitForTaskCompletion does not execute commands, but only monitors them
if [ $functionMode == "ParallelExec" ]; then
while [ $counter -lt "$mainItemCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do
if [ $readFromFile == true ]; then
currentCommand=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$mainInput")
currentCommandCondition=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$auxInput")
else
return $errorcount
fi
}
# Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs
# Returns the number of non zero exit codes from commands
# Use cmd1;cmd2;cmd3 syntax for small sets, use file for large command sets
# Only 2 first arguments are mandatory
# Sets a global variable called HARD_MAX_EXEC_TIME_REACHED to true if hardMaxTime is reached
function ParallelExec {
local numberOfProcesses="${1}" # Number of simultaneous commands to run
local commandsArg="${2}" # Semi-colon separated list of commands, or path to file containing one command per line
local readFromFile="${3:-false}" # commandsArg is a file (true), or a string (false)
local softMaxTime="${4:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${5:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local sleepTime="${6:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local keepLogging="${7:-0}" # Every keepLogging seconds, an alive log message is send. Setting this value to zero disables any alive logging.
local counting="${8:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${9:-false}" # Show spinner (true), don't show spinner (false)
local noErrorLog="${10:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true)
local callerName="${FUNCNAME[1]}"
local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function
local commandCount
local command
local pid
local counter=0
local commandsArray
local pidsArray
local newPidsArray
local retval
local errorCount=0
local pidState
local commandsArrayPid
# Set global var default
eval "HARD_MAX_EXEC_TIME_REACHED_$callerName=false"
if [ $counting == true ]; then # If counting == false _SOFT_ALERT should be a global value so no more than one soft alert is shown
local _SOFT_ALERT=false # Does a soft alert need to be triggered, if yes, send an alert once
currentCommand="${commandArray[$counter]}"
currentCommandCondition="${commandConditionArray[$counter]}"
fi
executeCommand=false
if [ $auxCount -ne 0 ]; then
Logger "Checking condition [$currentCommandCondition] for command [$currentCommand]." "DEBUG"
eval "$currentCommandCondition" &
ExecTasks "subConditionCheck" 0 0 1800 3600 1 $KEEP_LOGGING true true true true 1 $!
if [ $? -ne 0 ]; then
if [ $postPoneIfConditionFails == true ]; then
Logger "Condition not met for command [$currentCommand]. Postponing command." "NOTICE"
if [ $readFromFile == true ]; then
if [ -f "$commandsArg" ]; then
commandCount=$(wc -l < "$commandsArg")
# TODO: we should not write to the original file, but create a copy instead we can write postponed commands to
echo "$currentCommand" >> "$mainInput"
echo "$currentCommandCondition" >> "$auxInput"
else
commandCount=0
commansdArray+=($currentCommand)
commandsConditionArray+=($currentCommandCondition)
fi
else
IFS=';' read -r -a commandsArray <<< "$commandsArg"
commandCount=${#commandsArray[@]}
fi
Logger "Runnning $commandCount commands in $numberOfProcesses simultaneous processes." "DEBUG"
mainItemCount=$((mainItemCount+1))
while [ $counter -lt "$commandCount" ] || [ ${#pidsArray[@]} -gt 0 ]; do
if [ $spinner == true ]; then
Spinner
fi
if [ $counting == true ]; then
exec_time=$((SECONDS - seconds_begin))
# Trivial sleeptime so postponed commands will not stack too fast
sleep $sleepTime
else
exec_time=$SECONDS
fi
if [ $keepLogging -ne 0 ]; then
if [ $((($exec_time + 1) % $keepLogging)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1s
log_ttime=$exec_time
Logger "There are $((commandCount-counter)) / $commandCount tasks in the queue. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
fi
fi
fi
if [ $exec_time -gt $softMaxTime ]; then
if [ "$_SOFT_ALERT" != true ] && [ $softMaxTime -ne 0 ] && [ $noErrorLog != true ]; then
Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
_SOFT_ALERT=true
SendAlert true
Logger "Condition not met for command [$currentCommand]. Ignoring command." "NOTICE"
fi
fi
if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
if [ $noErrorLog != true ]; then
Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
fi
for pid in "${pidsArray[@]}"; do
KillChilds $pid true
if [ $? == 0 ]; then
Logger "Task with pid [$pid] stopped successfully." "NOTICE"
else
Logger "Could not stop task with pid [$pid]." "ERROR"
executeCommand=true
fi
done
if [ $noErrorLog != true ]; then
SendAlert true
fi
eval "HARD_MAX_EXEC_TIME_REACHED_$callerName=true"
# Return the number of commands that haven't run / finished run
return $((commandCount - counter + ${#pidsArray[@]}))
fi
while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do
if [ $readFromFile == true ]; then
command=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$commandsArg")
else
command="${commandsArray[$counter]}"
executeCommand=true
fi
Logger "Running command [$command]." "DEBUG"
eval "$command" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$callerName.$SCRIPT_PID.$TSTAMP" 2>&1 &
if [ $executeCommand == true ]; then
Logger "Running command [$currentCommand]." "DEBUG"
eval "$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$callerName.$SCRIPT_PID.$TSTAMP" 2>&1 &
pid=$!
pidsArray+=($pid)
commandsArrayPid[$pid]="$command"
commandsArrayPid[$pid]="$currentCommand"
#TODO not implemented
pidsTimeArray[$pid]=$((SECONDS - seconds_begin))
fi
counter=$((counter+1))
done
fi
newPidsArray=()
for pid in "${pidsArray[@]}"; do
if [ $(IsInteger $pid) -eq 1 ]; then
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
if kill -0 $pid > /dev/null 2>&1; then
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
pidState="$(eval $PROCESS_STATE_CMD)"
if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
#TODO: implement pidsTimeArray[$pid] check here
newPidsArray+=($pid)
fi
else
# pid is dead, get it's exit code from wait command
# pid is dead, get its exit code from wait command
wait $pid
retval=$?
if [ $retval -ne 0 ]; then
Logger "Command [${commandsArrayPid[$pid]}] failed with exit code [$retval]." "ERROR"
errorCount=$((errorCount+1))
Logger "${FUNCNAME[0]} called by [$callerName] finished monitoring [$pid] [$currentCommad] with exitcode [$retval]." "DEBUG"
errorcount=$((errorcount+1))
# Welcome to variable variable bash hell
if [ "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_$id\")" == "" ]; then
eval "WAIT_FOR_TASK_COMPLETION_$id=\"$pid:$retval\""
else
eval "WAIT_FOR_TASK_COMPLETION_$id=\";$pid:$retval\""
fi
else
Logger "${FUNCNAME[0]} called by [$callerName] finished monitoring [$pid] [$currentCommand] with exitcode [$retval]." "DEBUG"
fi
fi
fi
@ -926,7 +1029,18 @@ function ParallelExec {
done
return $errorCount
# Return exit code if only one process was monitored, else return number of errors
# As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value
if [ $noErrorLogAtAll == true ]; then
return 0
fi
if [ $mainItemCount -eq 1 ]; then
return $retval
else
return $errorcount
fi
}
function CleanUp {
@ -1079,7 +1193,7 @@ function GetLocalOS {
local localOsName
local localOsVer
# There's no good way to tell if currently running in BusyBox shell. Using sluggish way.
# There is no good way to tell if currently running in BusyBox shell. Using sluggish way.
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox"
else
@ -1169,7 +1283,7 @@ function GetOs {
local osInfo="/etc/os-release"
# There's no good way to tell if currently running in BusyBox shell. Using sluggish way.
# There is no good way to tell if currently running in BusyBox shell. Using sluggish way.
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox"
else
@ -1259,7 +1373,7 @@ function RunLocalCommand {
Logger "Running command [$command] on local host." "NOTICE"
eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 &
WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE"
@ -1299,7 +1413,7 @@ function RunRemoteCommand {
cmd=$SSH_CMD' "env LC_ALL=C env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1'
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE"
@ -1332,7 +1446,7 @@ function RunBeforeHook {
pids="$pids;$!"
fi
if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 0 0 true true false false 1 $pids
fi
}
@ -1350,7 +1464,7 @@ function RunAfterHook {
pids="$pids;$!"
fi
if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 0 0 true true false false 1 $pids
fi
}
@ -1361,7 +1475,7 @@ function CheckConnectivityRemoteHost {
if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then
eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" &
WaitForTaskCompletion $! 60 180 $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 60 180 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval != 0 ]; then
Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN"
@ -1374,6 +1488,7 @@ function CheckConnectivity3rdPartyHosts {
local remote3rdPartySuccess
local retval
local i
if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then
@ -1381,7 +1496,7 @@ function CheckConnectivity3rdPartyHosts {
for i in $REMOTE_3RD_PARTY_HOSTS
do
eval "$PING_CMD $i > /dev/null 2>&1" &
WaitForTaskCompletion $! 180 360 $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 180 360 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval != 0 ]; then
Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE"
@ -1540,7 +1655,7 @@ function PostInit {
}
function SetCompression {
## Busybox fix (Termux xz command doesn't support compression at all)
## Busybox fix (Termux xz command does not support compression at all)
if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then
compressionString=""
if type gzip > /dev/null 2>&1
@ -1629,7 +1744,7 @@ function InitLocalOSDependingSettings {
SetCompression
}
# Gets executed regardless of the need of remote connections. It's just that this code needs to get executed after we know if there is a remote os, and if yes, which one
# Gets executed regardless of the need of remote connections. It is just that this code needs to get executed after we know if there is a remote os, and if yes, which one
function InitRemoteOSDependingSettings {
if [ "$REMOTE_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then
@ -1721,7 +1836,7 @@ function InitRemoteOSDependingSettings {
RSYNC_ARGS=$RSYNC_ARGS" --whole-file"
fi
# Set compression options again after we know what remote OS we're dealing with
# Set compression options again after we know what remote OS we are dealing with
SetCompression
}
@ -2297,7 +2412,7 @@ function CheckReplicas {
_CheckReplicasRemote "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" &
pids="$pids;$!"
fi
WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $pids
retval=$?
if [ $retval -ne 0 ]; then
Logger "Cancelling task." "CRITICAL" $retval
@ -2635,7 +2750,8 @@ function HandleLocks {
# Do not bother checking for locks when FORCE_UNLOCK is set
if [ $FORCE_UNLOCK == true ]; then
overwrite=true
else
fi
_HandleLocksLocal "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}" "${INITIATOR[$__lockFile]}" "${INITIATOR[$__type]}" $overwrite &
initiatorPid=$!
pids="$initiatorPid"
@ -2648,14 +2764,12 @@ function HandleLocks {
targetPid=$!
pids="$pids;$targetPid"
fi
WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
ExecTasks "bulldog" 0 0 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $pids
retval=$?
if [ $retval -ne 0 ]; then
echo $WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
for pid in "${pidArray[@]}"; do
pid=${pid%:*}
echo "$pid - $initiatorPid - $targetPid"
if [ "$pid" == "$initiatorPid" ]; then
initiatorLockSuccess=false
elif [ "$pid" == "$targetPid" ]; then
@ -2663,17 +2777,22 @@ function HandleLocks {
fi
done
#WIP refactor the following
if [ $initiatorLockSuccess == true ]; then
INITIATOR_LOCK_FILE_EXISTS=true
fi
if [ $targetLockSuccess == true ]; then
TARGET_LOCK_FILE_EXISTS=true
fi
Logger "Cancelling task." "CRITICAL" $retval
exit 1
fi
if [ $initiatorLockSuccess == true ]; then
INITIATOR_LOCK_FILE_EXISTS=true
fi
if [ $targetLockSuccess == true ]; then
TARGET_LOCK_FILE_EXISTS=true
fi
}
@ -2747,7 +2866,7 @@ function UnlockReplicas {
fi
if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $pids
fi
}
@ -3073,7 +3192,7 @@ function syncAttrs {
fi
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
eval "$rsyncCmd"
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
retval=$?
if [ $retval -ne 0 ] && [ $retval -ne 24 ]; then
@ -3107,7 +3226,7 @@ function syncAttrs {
_getFileCtimeMtimeRemote "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-cleaned.$SCRIPT_PID.$TSTAMP" "$RUN_DIR/$PROGRAM.ctime_mtime___.${TARGET[$__type]}.$SCRIPT_PID.$TSTAMP" &
pids="$pids;$!"
fi
WaitForTaskCompletion $pids $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $pids
retval=$?
if [ $retval -ne 0 ]; then
Logger "Getting ctime attributes failed." "CRITICAL" $retval
@ -3160,7 +3279,7 @@ function syncAttrs {
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
eval "$rsyncCmd"
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
retval=$?
if [ $retval -ne 0 ] && [ $retval -ne 24 ]; then
@ -3708,7 +3827,7 @@ function Sync {
targetPid="$!"
fi
WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false
@ -3753,7 +3872,7 @@ function Sync {
targetPid="$!"
fi
WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false
@ -3802,7 +3921,7 @@ function Sync {
targetPid="$!"
fi
WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false
@ -3845,7 +3964,7 @@ function Sync {
if [ "$resumeInitiator" == "${SYNC_ACTION[3]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[3]}" ]; then
if [ "$LOG_CONFLICTS" == "yes" ]; then
conflictList "${INITIATOR[$__timestampCurrentFile]}" "${INITIATOR[$__timestampAfterFileNoSuffix]}" "${INITIATOR[$__conflictListFile]}" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[3]}" > "${INITIATOR[$__targetLastActionFile]}"
@ -3869,7 +3988,7 @@ function Sync {
if [ "$resumeInitiator" == "${SYNC_ACTION[4]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[4]}" ]; then
if [[ "$RSYNC_ATTR_ARGS" == *"-X"* ]] || [[ "$RSYNC_ATTR_ARGS" == *"-A"* ]]; then
syncAttrs "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[4]}" > "${INITIATOR[$__targetLastActionFile]}"
@ -3894,7 +4013,7 @@ function Sync {
if [ "$CONFLICT_PREVALANCE" == "${TARGET[$__type]}" ]; then
if [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}"
resumeTarget="${SYNC_ACTION[5]}"
@ -3906,7 +4025,7 @@ function Sync {
fi
if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}"
resumeInitiator="${SYNC_ACTION[5]}"
@ -3919,7 +4038,7 @@ function Sync {
else
if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}"
resumeInitiator="${SYNC_ACTION[5]}"
@ -3931,7 +4050,7 @@ function Sync {
fi
if [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 $!
if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}"
resumeTarget="${SYNC_ACTION[5]}"
@ -3956,7 +4075,7 @@ function Sync {
targetPid="$!"
fi
WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false
@ -4002,7 +4121,7 @@ function Sync {
targetPid="$!"
fi
WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false
@ -4051,7 +4170,7 @@ function Sync {
targetPid="$!"
fi
WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$initiatorPid;$targetPid"
if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_${FUNCNAME[0]}\")"
initiatorFail=false
@ -4209,7 +4328,7 @@ function SoftDelete {
_SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__backupDir]}" $CONFLICT_BACKUP_DAYS "conflict backup" &
pids="$pids;$!"
fi
WaitForTaskCompletion $pids $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$pids"
if [ $? -ne 0 ] && [ "$(eval echo \"\$HARD_MAX_EXEC_TIME_REACHED_${FUNCNAME[0]}\")" == true ]; then
exit 1
fi
@ -4227,7 +4346,7 @@ function SoftDelete {
_SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__deleteDir]}" $SOFT_DELETE_DAYS "softdelete" &
pids="$pids;$!"
fi
WaitForTaskCompletion $pids $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false false 1 "$pids"
if [ $? -ne 0 ] && [ "$(eval echo \"\$HARD_MAX_EXEC_TIME_REACHED_${FUNCNAME[0]}\")" == true ]; then
exit 1
fi
@ -4595,7 +4714,7 @@ function SyncOnChanges {
if [ "$LOCAL_OS" == "MacOSX" ]; then
fswatch $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude "$OSYNC_DIR" -1 "$INITIATOR_SYNC_DIR" > /dev/null &
# Mac fswatch doesn't have timeout switch, replacing wait $! with WaitForTaskCompletion without warning nor spinner and increased SLEEP_TIME to avoid cpu hogging. This sims wait $! with timeout
WaitForTaskCompletion $! 0 $MAX_WAIT 1 0 true false true
ExecTasks "MonitorMacOSXWait" 0 0 0 $MAX_WAIT 1 0 true false true false $!
else
inotifywait $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude "$OSYNC_DIR" -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" &
wait $!

521
osync_target_helper.sh
Unescape Escape View File

@ -8,8 +8,8 @@ PROGRAM_BUILD=2017061901
IS_STABLE=no
_OFUNCTIONS_VERSION=2.1.4-rc1+
_OFUNCTIONS_BUILD=2017123002
_OFUNCTIONS_VERSION=2.2.0-dev
_OFUNCTIONS_BUILD=2018010201
_OFUNCTIONS_BOOTSTRAP=true
## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
@ -26,7 +26,7 @@ _OFUNCTIONS_BOOTSTRAP=true
#TODO: global WAIT_FOR_TASK_COMPLETION_id instead of callerName has to be backported to ParallelExec and osync / obackup / pmocr ocde
## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel
## When called from subprocesses, variable of main process can't be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID.$TSTAMP
## When called from subprocesses, variable of main process cannot be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID.$TSTAMP
if ! type "$BASH" > /dev/null; then
echo "Please run this script only with bash shell. Tested on bash >= 3.2"
@ -238,7 +238,7 @@ function Logger {
if [ "$level" == "CRITICAL" ]; then
_Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true
ERROR_ALERT=true
# ERROR_ALERT / WARN_ALERT isn't set in main when Logger is called from a subprocess. Need to keep this flag.
# ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. Need to keep this flag.
echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
return
elif [ "$level" == "ERROR" ]; then
@ -304,7 +304,7 @@ function KillChilds {
local pid="${1}" # Parent pid to kill childs
local self="${2:-false}" # Should parent be killed too ?
# Paranoid checks, we can safely assume that $pid shouldn't be 0 nor 1
# Paranoid checks, we can safely assume that $pid should not be 0 nor 1
if [ $(IsInteger "$pid") -eq 0 ] || [ "$pid" == "" ] || [ "$pid" == "0" ] || [ "$pid" == "1" ]; then
Logger "Bogus pid given [$pid]." "CRITICAL"
return 1
@ -478,13 +478,13 @@ function SendEmail {
if [ $? != 0 ]; then
Logger "Cannot send alert mail via $(type -p sendmail) !!!" "WARN"
# Don't bother try other mail systems with busybox
# Do not bother try other mail systems with busybox
return 1
else
return 0
fi
else
Logger "Sendmail not present. Won't send any mail" "WARN"
Logger "Sendmail not present. Will not send any mail" "WARN"
return 1
fi
fi
@ -618,6 +618,8 @@ function LoadConfigFile {
CONFIG_FILE="$configFile"
}
# Quick and dirty performance logger only used for debugging
_OFUNCTIONS_SPINNER="|/-\\"
function Spinner {
if [ $_LOGGER_SILENT == true ] || [ "$_LOGGER_ERR_ONLY" == true ]; then
@ -630,27 +632,128 @@ function Spinner {
fi
}
# WaitForTaskCompletion function emulation, now uses ExecTasks
function WaitForTaskCompletion {
local pids="${1}"
local softMaxTime="${2:-0}"
local hardMaxTime="${3:-0}"
local sleepTime="${4:-.05}"
local keepLogging="${5:-0}"
local counting="${6:-true}"
local spinner="${7:-true}"
local noErrorLog="${8:-false}"
local id="${9-base}"
ExecTasks "$id" 0 0 "$softMaxTime" "$hardMaxTime" "$sleepTime" "$keepLogging" "$counting" "$spinner" "$noErrorlog" false 1 "$pids" "" ""
}
# ParallelExec function emulation, now uses ExecTasks
function ParallelExec {
local numberOfProcesses="${1}"
local commandsArg="${2}"
local readFromFile="${3:-false}"
local softMaxTime="${4:-0}"
local hardMaxTime="${5:-0}"
local sleepTime="${6:-.05}"
local keepLogging="${7:-0}"
local counting="${8:-true}"
local spinner="${9:-false}"
local noErrorLog="${10:-false}"
ExecTasks "base" 0 0 "$softMaxTime" "$hardMaxTime" "$sleepTime" "$keepLogging" "$counting" "$spinner" "$noErrorLog" false 3 "$commandsArg" "" "$numberOfProcesses"
}
# Time control function for background processes, suitable for multiple synchronous processes
# Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2
# Also sets a global variable called HARD_MAX_EXEC_TIME_REACHED_$callerName to true if hardMaxTime is reached
# Standard wait $! emulation would be WaitForTaskCompletion $! 0 0 1 0 true false true false
## Main asynchronous execution function
## This function can monitor given pids as background processes, and stop them if max execution time is reached. Suitable for multiple synchronous processes.
## It can also take a list of commands to execute in parallel, and stop them if max execution time is reached.
## Function has 8 execution modes
# WaitForTaskCompletion mode: Monitor given pids as background processes, stop them if max execution time is reached. Suitaable for multiple synhronous processes.
# 1 : WaitForTaskCompletion, semi-colon separated list of pids to monitor
# 2 : WaitForTaskCompletion, list of pids to monior, from file, one per line
# Example of improved wait $! emulation
# ExecTasks "some_identifier" 0 0 0 0 1 1800 true false true true 1 $!
# Example: monitor two sleep processes, warn if execution time is higher than 10 seconds, stop after 20 seconds
# sleep 15 &
# pid=$!
# sleep 20 &
# pid2=$!
# ExecTasks "some_identifier" 0 0 10 20 1 1800 true true false false 1 "$pid;$pid2"
# ParallelExecMode: Take list of commands to execute in parallel, stop them if max execution time is reached.
# Also take optional conditional arguments to verifiy before execution main commands. Conditional command exit code 0 means ready to execute. Other exit codes will ignore/postpone main command.
# 3 : ParallelExec, semi-colon separated list of commands to execute in parallel, no conditions
# 4 : ParallelExec, semi-colon separated list of commands to execute in parallel , semi-colon separated list of conditional commands, ignoring main commands on condition failures
# 5 : ParallelExec, semi-colon separated list of commands, semi-colon separated list of conditional commands, postponing main commands on condition failures
# 6 : ParallelExec, list of commands from file, one per line, no conditions
# 7 : ParallelExec, list of commands from file, one per line, list of conditional commands from file, one per line, ignoring main commands on condition failures
# 8 : ParallelExec, list of commands from file, one per line, list of conditional commands from file, one per line, postponing main commands on condition failures
# Exmaple: execute four du commands, only if directories exist, warn if execution takes more than 300 seconds, stop if takes longer than 900 seconds. Execute max 3 commands in parallel.
# commands="du -csh /var;du -csh /etc;du -csh /home;du -csh /usr"
# conditions="[ -d /var ];[ -d /etc ];[ -d /home];[ -d /usr]"
# ExecTasks "some_identifier" 0 0 300 900 1 1800 true true false false 4 "$commands" "$conditions" 3
## ofunctions.sh subfunction requirements:
## Spinner
## Logger
## JoinString
## KillChilds
function ExecTasks {
local id="${1:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id
#TODO: not implemented yet
local softPerProcessTime="${2:-0}"
local hardPerProcessTime="${3:-0}"
local softMaxTime="${4:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${5:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local sleepTime="${6:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local keepLogging="${7:-0}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging.
local counting="${8:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${9:-true}" # Show spinner (true), do not show anything (false)
local noTimeErrorLog="${10:-false}" # Log errors when reaching soft / hard max time (false), do not log errors on those triggers (true)
local noErrorLogAtAll="${11:-false}" # Do not log errros at all (false)
local execTasksMode="${12:-1}" # In which mode the function should work, see above
local mainInput="${13}" # Contains list of pids / commands or filepath to list of pids / commands
local auxInput="${14}" # Contains list of conditional commands or filepath to list of conditional commands
local numberOfProcesses="${15:-2}" # Number of simultaneous commands to run in ParallExec mode
function WaitForTaskCompletion {
local pids="${1}" # pids to wait for, separated by semi-colon
local softMaxTime="${2:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${3:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local sleepTime="${4:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local keepLogging="${5:-0}" # Every keepLogging seconds, an alive log message is send. Setting this value to zero disables any alive logging.
local counting="${6:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${7:-true}" # Show spinner (true), don't show anything (false)
local noErrorLog="${8:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true)
local id="${9-base}" # Optional id in order to get $WAIT_FOR_TASK_COMPLETION_id global variable
local i
local callerName="${FUNCNAME[1]}"
# Since ExecTasks takes up to 15 arguments, do a quick preflight check in DEBUG mode
if [ "$_DEBUG" == "yes" ]; then
declare -a booleans=(counting spinner noTimeErrorLog noErrorLogAtAll)
for i in "${num_vars[@]}"; do
test="if [ $i != false ] && [ $i != true ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
eval "$test"
done
declare -a integers=(softPerProcessTime hardPerProcessTime softMaxTime hardMaxTime keepLogging execTasksMode numberOfProcesses)
for i in "${integers[@]}"; do
test="if [ $(IsNumericExpand \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
eval "$test"
done
fi
# ParallelExec specific variables
local auxCount # Number of conditional commands
local commandsArray=() # Array containing commands
local commandsConditionArray=() # Array containing conditional commands
local currentCommand # Variable containing currently processed command
local currentCommandCondition # Variable containing currently processed conditional command
local commandsArrayPid # Array containing pids of commands currently run
local postPoneIfConditionFails # Boolean to check if command needs to be postponed if condition command fails
# Common variables
local pid # Current pid working on
local pidState # State of the process
local mainItemCount # number of given items (pids or commands)
local readFromFile # Should we read pids / commands from a file (true)
local counter=0
local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
@ -658,27 +761,122 @@ function WaitForTaskCompletion {
local retval=0 # return value of monitored pid process
local errorcount=0 # Number of pids that finished with errors
local pid # Current pid working on
local pidCount # number of given pids
local pidState # State of the process
local pidsArray # Array of currently running pids
local newPidsArray # New array of currently running pids
local newPidsArray # New array of currently running pids for next iteration
local pidsTimeArray # Array containing execution begin time of pids
local executeCommand # Boolean to check if currentCommand can be executed given a condition
if [ $counting == true ]; then # If counting == false _SOFT_ALERT should be a global value so no more than one soft alert is shown
local _SOFT_ALERT=false # Does a soft alert need to be triggered, if yes, send an alert once
fi
local functionMode
IFS=';' read -a pidsArray <<< "$pids"
pidCount=${#pidsArray[@]}
if [ $counting == true ]; then
local softAlert=false # Does a soft alert need to be triggered, if yes, send an alert once
else
local softAlert=false
fi
# Set global var default
# Initialise global variable
eval "WAIT_FOR_TASK_COMPLETION_$id=\"\""
eval "HARD_MAX_EXEC_TIME_REACHED_$id=false"
while [ ${#pidsArray[@]} -gt 0 ]; do
case $execTasksMode in
1)
IFS=';' read -r -a pidsArray <<< "$mainInput"
mainItemCount=${#pidsArray[@]}
readFromFile=false
functionMode=WaitForTaskCompletion
# Force while condition to be true
counter=$mainItemCount
;;
2)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else
mainItemCount=0
Logger "Cannot read file [$mainInput]." "WARN"
fi
functionMode=WaitForTaskCompletion
# Force while condition to be true
counter=$mainItemCount
;;
3)
IFS=';' read -r -a commandsArray <<< "$mainInput"
mainItemCount=${#commandsArray[@]}
readFromFile=false
functionMode=ParallelExec
;;
4)
IFS=';' read -r -a commandsArray <<< "$mainInput"
mainItemCount=${#commandsArray[@]}
IFS=';' read -r -a commandsConditionArray <<< "$auxInput"
auxItemCount=${#commandsConditionArray[@]}
readFromFile=false
postPoneIfConditionFails=false
functionMode=ParallelExec
;;
5)
IFS=';' read -r -a commandsArray <<< "$mainInput"
mainItemCount=${#commandsArray[@]}
IFS=';' read -r -a commandsConditionArray <<< "$auxInput"
auxItemCount=${#commandsConditionArray[@]}
readFromFile=false
postPoneIfConditionFails=true
functionMode=ParallelExec
;;
6)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else
mainItemCount=0
Logger "Cannot read file [$mainInput]." "WARN"
fi
functionMode=ParallelExec
;;
7)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else
mainItemCount=0
Logger "Cannot read file [$mainInput]." "WARN"
fi
if [ -f "$auxInput" ]; then
auxCount=$(wc -l < "$auxInput")
else
auxCount=0
Logger "Cannot read file [$auxInput]." "WARN"
fi
postPoneIfConditionFails=false
functionMode=ParallelExec
;;
8)
if [ -f "$mainInput" ]; then
mainItemCount=$(wc -l < "$mainInput")
readFromFile=true
else
mainItemCount=0
Logger "Cannot read file [$mainInput]." "WARN"
fi
if [ -f "$auxInput" ]; then
auxCount=$(wc -l < "$auxInput")
else
auxCount=0
Logger "Cannot read file [$auxInput]." "WARN"
fi
postPoneIfConditionFails=true
functionMode=ParallelExec
;;
*)
Logger "Unknown exec mode for ${FUNCNAME}." "CRITICAL"
exit 1
esac
#while [ ${#pidsArray[@]} -gt 0 ]; do
# The counter -lt mainItemCount has to be false for WaitForTaskCompletion modes
while [ ${#pidsArray[@]} -gt 0 ] || [ $counter -lt $mainItemCount ]; do
newPidsArray=()
if [ $spinner == true ]; then
@ -692,23 +890,27 @@ function WaitForTaskCompletion {
if [ $keepLogging -ne 0 ]; then
if [ $((($exec_time + 1) % $keepLogging)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1s
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1 second
log_ttime=$exec_time
if [ $functionMode == "Wait" ]; then
Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
elif [ $functionMode == "ParallelExec" ]; then
Logger "There are $((mainItemCount-counter)) / $mainItemCount tasks in the queue. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
fi
fi
fi
fi
if [ $exec_time -gt $softMaxTime ]; then
if [ "$_SOFT_ALERT" != true ] && [ $softMaxTime -ne 0 ] && [ $noErrorLog != true ]; then
if [ "$softAlert" != true ] && [ $softMaxTime -ne 0 ] && [ $noTimeErrorLog != true ]; then
Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
_SOFT_ALERT=true
softAlert=true
SendAlert true
fi
fi
if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
if [ $noErrorLog != true ]; then
if [ $noTimeErrorLog != true ]; then
Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
fi
for pid in "${pidsArray[@]}"; do
@ -720,194 +922,95 @@ function WaitForTaskCompletion {
fi
errorcount=$((errorcount+1))
done
if [ $noErrorLog != true ]; then
if [ $noTimeErrorLog != true ]; then
SendAlert true
fi
eval "HARD_MAX_EXEC_TIME_REACHED_$id=true"
if [ $functionMode == "WaitForTaskCompletion" ]; then
return $errorcount
elif [ $functionMode == "ParallelExec" ]; then
return $((mainItemCount - counter + ${#pidsArray[@]}))
fi
for pid in "${pidsArray[@]}"; do
if [ $(IsInteger $pid) -eq 1 ]; then
if kill -0 $pid > /dev/null 2>&1; then
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
pidState="$(eval $PROCESS_STATE_CMD)"
if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
newPidsArray+=($pid)
fi
else
# pid is dead, get it's exit code from wait command
wait $pid
retval=$?
if [ $retval -ne 0 ]; then
Logger "${FUNCNAME[0]} called by [$callerName] finished monitoring [$pid] with exitcode [$retval]." "DEBUG"
errorcount=$((errorcount+1))
# Welcome to variable variable bash hell
if [ "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_$id\")" == "" ]; then
eval "WAIT_FOR_TASK_COMPLETION_$id=\"$pid:$retval\""
else
eval "WAIT_FOR_TASK_COMPLETION_$id=\";$pid:$retval\""
fi
fi
fi
fi
done
pidsArray=("${newPidsArray[@]}")
# Trivial wait time for bash to not eat up all CPU
sleep $sleepTime
done
# Return exit code if only one process was monitored, else return number of errors
# As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value
if [ $pidCount -eq 1 ]; then
return $retval
# The following execution bloc is only needed in ParallelExec mode since WaitForTaskCompletion does not execute commands, but only monitors them
if [ $functionMode == "ParallelExec" ]; then
while [ $counter -lt "$mainItemCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do
if [ $readFromFile == true ]; then
currentCommand=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$mainInput")
currentCommandCondition=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$auxInput")
else
return $errorcount
fi
}
# Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs
# Returns the number of non zero exit codes from commands
# Use cmd1;cmd2;cmd3 syntax for small sets, use file for large command sets
# Only 2 first arguments are mandatory
# Sets a global variable called HARD_MAX_EXEC_TIME_REACHED to true if hardMaxTime is reached
function ParallelExec {
local numberOfProcesses="${1}" # Number of simultaneous commands to run
local commandsArg="${2}" # Semi-colon separated list of commands, or path to file containing one command per line
local readFromFile="${3:-false}" # commandsArg is a file (true), or a string (false)
local softMaxTime="${4:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${5:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local sleepTime="${6:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local keepLogging="${7:-0}" # Every keepLogging seconds, an alive log message is send. Setting this value to zero disables any alive logging.
local counting="${8:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${9:-false}" # Show spinner (true), don't show spinner (false)
local noErrorLog="${10:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true)
local callerName="${FUNCNAME[1]}"
local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function
local commandCount
local command
local pid
local counter=0
local commandsArray
local pidsArray
local newPidsArray
local retval
local errorCount=0
local pidState
local commandsArrayPid
# Set global var default
eval "HARD_MAX_EXEC_TIME_REACHED_$callerName=false"
if [ $counting == true ]; then # If counting == false _SOFT_ALERT should be a global value so no more than one soft alert is shown
local _SOFT_ALERT=false # Does a soft alert need to be triggered, if yes, send an alert once
currentCommand="${commandArray[$counter]}"
currentCommandCondition="${commandConditionArray[$counter]}"
fi
executeCommand=false
if [ $auxCount -ne 0 ]; then
Logger "Checking condition [$currentCommandCondition] for command [$currentCommand]." "DEBUG"
eval "$currentCommandCondition" &
ExecTasks "subConditionCheck" 0 0 1800 3600 1 $KEEP_LOGGING true true true true 1 $!
if [ $? -ne 0 ]; then
if [ $postPoneIfConditionFails == true ]; then
Logger "Condition not met for command [$currentCommand]. Postponing command." "NOTICE"
if [ $readFromFile == true ]; then
if [ -f "$commandsArg" ]; then
commandCount=$(wc -l < "$commandsArg")
else
commandCount=0
fi
# TODO: we should not write to the original file, but create a copy instead we can write postponed commands to
echo "$currentCommand" >> "$mainInput"
echo "$currentCommandCondition" >> "$auxInput"
else
IFS=';' read -r -a commandsArray <<< "$commandsArg"
commandCount=${#commandsArray[@]}
fi
Logger "Runnning $commandCount commands in $numberOfProcesses simultaneous processes." "DEBUG"
while [ $counter -lt "$commandCount" ] || [ ${#pidsArray[@]} -gt 0 ]; do
if [ $spinner == true ]; then
Spinner
commansdArray+=($currentCommand)
commandsConditionArray+=($currentCommandCondition)
fi
mainItemCount=$((mainItemCount+1))
if [ $counting == true ]; then
exec_time=$((SECONDS - seconds_begin))
# Trivial sleeptime so postponed commands will not stack too fast
sleep $sleepTime
else
exec_time=$SECONDS
fi
if [ $keepLogging -ne 0 ]; then
if [ $((($exec_time + 1) % $keepLogging)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1s
log_ttime=$exec_time
Logger "There are $((commandCount-counter)) / $commandCount tasks in the queue. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
fi
fi
Logger "Condition not met for command [$currentCommand]. Ignoring command." "NOTICE"
fi
if [ $exec_time -gt $softMaxTime ]; then
if [ "$_SOFT_ALERT" != true ] && [ $softMaxTime -ne 0 ] && [ $noErrorLog != true ]; then
Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
_SOFT_ALERT=true
SendAlert true
fi
fi
if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
if [ $noErrorLog != true ]; then
Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
fi
for pid in "${pidsArray[@]}"; do
KillChilds $pid true
if [ $? == 0 ]; then
Logger "Task with pid [$pid] stopped successfully." "NOTICE"
else
Logger "Could not stop task with pid [$pid]." "ERROR"
fi
done
if [ $noErrorLog != true ]; then
SendAlert true
fi
eval "HARD_MAX_EXEC_TIME_REACHED_$callerName=true"
# Return the number of commands that haven't run / finished run
return $((commandCount - counter + ${#pidsArray[@]}))
executeCommand=true
fi
while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do
if [ $readFromFile == true ]; then
command=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$commandsArg")
else
command="${commandsArray[$counter]}"
executeCommand=true
fi
Logger "Running command [$command]." "DEBUG"
eval "$command" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$callerName.$SCRIPT_PID.$TSTAMP" 2>&1 &
if [ $executeCommand == true ]; then
Logger "Running command [$currentCommand]." "DEBUG"
eval "$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$callerName.$SCRIPT_PID.$TSTAMP" 2>&1 &
pid=$!
pidsArray+=($pid)
commandsArrayPid[$pid]="$command"
commandsArrayPid[$pid]="$currentCommand"
#TODO not implemented
pidsTimeArray[$pid]=$((SECONDS - seconds_begin))
fi
counter=$((counter+1))
done
fi
newPidsArray=()
for pid in "${pidsArray[@]}"; do
if [ $(IsInteger $pid) -eq 1 ]; then
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
if kill -0 $pid > /dev/null 2>&1; then
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
pidState="$(eval $PROCESS_STATE_CMD)"
if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
#TODO: implement pidsTimeArray[$pid] check here
newPidsArray+=($pid)
fi
else
# pid is dead, get it's exit code from wait command
# pid is dead, get its exit code from wait command
wait $pid
retval=$?
if [ $retval -ne 0 ]; then
Logger "Command [${commandsArrayPid[$pid]}] failed with exit code [$retval]." "ERROR"
errorCount=$((errorCount+1))
Logger "${FUNCNAME[0]} called by [$callerName] finished monitoring [$pid] [$currentCommad] with exitcode [$retval]." "DEBUG"
errorcount=$((errorcount+1))
# Welcome to variable variable bash hell
if [ "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_$id\")" == "" ]; then
eval "WAIT_FOR_TASK_COMPLETION_$id=\"$pid:$retval\""
else
eval "WAIT_FOR_TASK_COMPLETION_$id=\";$pid:$retval\""
fi
else
Logger "${FUNCNAME[0]} called by [$callerName] finished monitoring [$pid] [$currentCommand] with exitcode [$retval]." "DEBUG"
fi
fi
fi
@ -920,7 +1023,18 @@ function ParallelExec {
done
return $errorCount
# Return exit code if only one process was monitored, else return number of errors
# As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value
if [ $noErrorLogAtAll == true ]; then
return 0
fi
if [ $mainItemCount -eq 1 ]; then
return $retval
else
return $errorcount
fi
}
function CleanUp {
@ -1073,7 +1187,7 @@ function GetLocalOS {
local localOsName
local localOsVer
# There's no good way to tell if currently running in BusyBox shell. Using sluggish way.
# There is no good way to tell if currently running in BusyBox shell. Using sluggish way.
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox"
else
@ -1163,7 +1277,7 @@ function GetOs {
local osInfo="/etc/os-release"
# There's no good way to tell if currently running in BusyBox shell. Using sluggish way.
# There is no good way to tell if currently running in BusyBox shell. Using sluggish way.
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox"
else
@ -1253,7 +1367,7 @@ function RunLocalCommand {
Logger "Running command [$command] on local host." "NOTICE"
eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 &
WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE"
@ -1293,7 +1407,7 @@ function RunRemoteCommand {
cmd=$SSH_CMD' "env LC_ALL=C env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1'
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE"
@ -1326,7 +1440,7 @@ function RunBeforeHook {
pids="$pids;$!"
fi
if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 0 0 true true false false 1 $pids
fi
}
@ -1344,7 +1458,7 @@ function RunAfterHook {
pids="$pids;$!"
fi
if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 0 0 true true false false 1 $pids
fi
}
@ -1355,7 +1469,7 @@ function CheckConnectivityRemoteHost {
if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then
eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" &
WaitForTaskCompletion $! 60 180 $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 60 180 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval != 0 ]; then
Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN"
@ -1368,6 +1482,7 @@ function CheckConnectivity3rdPartyHosts {
local remote3rdPartySuccess
local retval
local i
if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then
@ -1375,7 +1490,7 @@ function CheckConnectivity3rdPartyHosts {
for i in $REMOTE_3RD_PARTY_HOSTS
do
eval "$PING_CMD $i > /dev/null 2>&1" &
WaitForTaskCompletion $! 180 360 $SLEEP_TIME $KEEP_LOGGING true true false
ExecTasks "${FUNCNAME[0]}" 0 0 180 360 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
retval=$?
if [ $retval != 0 ]; then
Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE"
@ -1534,7 +1649,7 @@ function PostInit {
}
function SetCompression {
## Busybox fix (Termux xz command doesn't support compression at all)
## Busybox fix (Termux xz command does not support compression at all)
if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then
compressionString=""
if type gzip > /dev/null 2>&1
@ -1623,7 +1738,7 @@ function InitLocalOSDependingSettings {
SetCompression
}
# Gets executed regardless of the need of remote connections. It's just that this code needs to get executed after we know if there is a remote os, and if yes, which one
# Gets executed regardless of the need of remote connections. It is just that this code needs to get executed after we know if there is a remote os, and if yes, which one
function InitRemoteOSDependingSettings {
if [ "$REMOTE_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then
@ -1715,7 +1830,7 @@ function InitRemoteOSDependingSettings {
RSYNC_ARGS=$RSYNC_ARGS" --whole-file"
fi
# Set compression options again after we know what remote OS we're dealing with
# Set compression options again after we know what remote OS we are dealing with
SetCompression
}

Write Preview
Loading…
Cancel
Save