|
|
|
@ -1,6 +1,6 @@
|
|
|
|
|
#### MINIMAL-FUNCTION-SET BEGIN ####
|
|
|
|
|
|
|
|
|
|
## FUNC_BUILD=2016082802
|
|
|
|
|
## FUNC_BUILD=2016082901
|
|
|
|
|
## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
|
|
|
|
|
|
|
|
|
|
## To use in a program, define the following variables:
|
|
|
|
@ -20,19 +20,20 @@ export LC_ALL=C
|
|
|
|
|
MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
|
|
|
|
|
|
|
|
|
|
# Environment variables that can be overriden by programs
|
|
|
|
|
_DRYRUN=0
|
|
|
|
|
_SILENT=0
|
|
|
|
|
_DRYRUN=false
|
|
|
|
|
_SILENT=false
|
|
|
|
|
_VERBOSE=false
|
|
|
|
|
_LOGGER_PREFIX="date"
|
|
|
|
|
_LOGGER_STDERR=0
|
|
|
|
|
_LOGGER_STDERR=false
|
|
|
|
|
if [ "$KEEP_LOGGING" == "" ]; then
|
|
|
|
|
KEEP_LOGGING=1801
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# Initial error status, logging 'WARN', 'ERROR' or 'CRITICAL' will enable alerts flags
|
|
|
|
|
ERROR_ALERT=0
|
|
|
|
|
WARN_ALERT=0
|
|
|
|
|
ERROR_ALERT=false
|
|
|
|
|
WARN_ALERT=false
|
|
|
|
|
|
|
|
|
|
# Current log
|
|
|
|
|
# Log from current run
|
|
|
|
|
CURRENT_LOG=
|
|
|
|
|
|
|
|
|
|
## allow function call checks #__WITH_PARANOIA_DEBUG
|
|
|
|
@ -44,11 +45,11 @@ fi #__WITH_PARANOIA_DEBUG
|
|
|
|
|
if [ ! "$_DEBUG" == "yes" ]; then
|
|
|
|
|
_DEBUG=no
|
|
|
|
|
SLEEP_TIME=.05 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys
|
|
|
|
|
_VERBOSE=0
|
|
|
|
|
_VERBOSE=false
|
|
|
|
|
else
|
|
|
|
|
SLEEP_TIME=1
|
|
|
|
|
trap 'TrapError ${LINENO} $?' ERR
|
|
|
|
|
_VERBOSE=1
|
|
|
|
|
_VERBOSE=true
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
SCRIPT_PID=$$
|
|
|
|
@ -102,9 +103,9 @@ function _Logger {
|
|
|
|
|
echo -e "$lvalue" >> "$LOG_FILE"
|
|
|
|
|
CURRENT_LOG="$CURRENT_LOG"$'\n'"$lvalue"
|
|
|
|
|
|
|
|
|
|
if [ "$_LOGGER_STDERR" -eq 1 ]; then
|
|
|
|
|
if [ $_LOGGER_STDERR == true ]; then
|
|
|
|
|
cat <<< "$evalue" 1>&2
|
|
|
|
|
elif [ "$_SILENT" -eq 0 ]; then
|
|
|
|
|
elif [ "$_SILENT" == false ]; then
|
|
|
|
|
echo -e "$svalue"
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
@ -112,7 +113,7 @@ function _Logger {
|
|
|
|
|
# General log function with log levels
|
|
|
|
|
function Logger {
|
|
|
|
|
local value="${1}" # Sentence to log (in double quotes)
|
|
|
|
|
local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, NOTICE, WARN, ERROR, CRITIAL
|
|
|
|
|
local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, VERBOSE, NOTICE, WARN, ERROR, CRITIAL
|
|
|
|
|
|
|
|
|
|
if [ "$_LOGGER_PREFIX" == "time" ]; then
|
|
|
|
|
prefix="TIME: $SECONDS - "
|
|
|
|
@ -124,19 +125,22 @@ function Logger {
|
|
|
|
|
|
|
|
|
|
if [ "$level" == "CRITICAL" ]; then
|
|
|
|
|
_Logger "$prefix\e[41m$value\e[0m" "$prefix$level:$value" "$level:$value"
|
|
|
|
|
ERROR_ALERT=1
|
|
|
|
|
ERROR_ALERT=true
|
|
|
|
|
return
|
|
|
|
|
elif [ "$level" == "ERROR" ]; then
|
|
|
|
|
_Logger "$prefix\e[91m$value\e[0m" "$prefix$level:$value" "$level:$value"
|
|
|
|
|
ERROR_ALERT=1
|
|
|
|
|
ERROR_ALERT=true
|
|
|
|
|
return
|
|
|
|
|
elif [ "$level" == "WARN" ]; then
|
|
|
|
|
_Logger "$prefix\e[93m$value\e[0m" "$prefix$level:$value" "$level:$value"
|
|
|
|
|
WARN_ALERT=1
|
|
|
|
|
WARN_ALERT=true
|
|
|
|
|
return
|
|
|
|
|
elif [ "$level" == "NOTICE" ]; then
|
|
|
|
|
_Logger "$prefix$value"
|
|
|
|
|
return
|
|
|
|
|
elif [ "$level" == "VERBOSE" ] && [ $_VERBOSE == true ]; then
|
|
|
|
|
_Logger "$prefix$value"
|
|
|
|
|
return
|
|
|
|
|
elif [ "$level" == "DEBUG" ]; then
|
|
|
|
|
if [ "$_DEBUG" == "yes" ]; then
|
|
|
|
|
_Logger "$prefix$value"
|
|
|
|
@ -173,7 +177,7 @@ function QuickLogger {
|
|
|
|
|
|
|
|
|
|
__CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
|
|
|
|
|
|
|
|
|
|
if [ "$_SILENT" -eq 1 ]; then
|
|
|
|
|
if [ $_SILENT == true ]; then
|
|
|
|
|
_QuickLogger "$value" "log"
|
|
|
|
|
else
|
|
|
|
|
_QuickLogger "$value" "stdout"
|
|
|
|
@ -270,9 +274,9 @@ function SendAlert {
|
|
|
|
|
fi
|
|
|
|
|
body="$MAIL_ALERT_MSG"$'\n\n'"$CURRENT_LOG"
|
|
|
|
|
|
|
|
|
|
if [ $ERROR_ALERT -eq 1 ]; then
|
|
|
|
|
if [ $ERROR_ALERT == true ]; then
|
|
|
|
|
subject="Error alert for $INSTANCE_ID"
|
|
|
|
|
elif [ $WARN_ALERT -eq 1 ]; then
|
|
|
|
|
elif [ $WARN_ALERT == true ]; then
|
|
|
|
|
subject="Warning alert for $INSTANCE_ID"
|
|
|
|
|
else
|
|
|
|
|
subject="Alert for $INSTANCE_ID"
|
|
|
|
@ -524,7 +528,7 @@ function TrapError {
|
|
|
|
|
local job="$0"
|
|
|
|
|
local line="$1"
|
|
|
|
|
local code="${2:-1}"
|
|
|
|
|
if [ $_SILENT -eq 0 ]; then
|
|
|
|
|
if [ $_SILENT == false ]; then
|
|
|
|
|
echo -e " /!\ ERROR in ${job}: Near line ${line}, exit code ${code}"
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
@ -550,7 +554,7 @@ function LoadConfigFile {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function Spinner {
|
|
|
|
|
if [ $_SILENT -eq 1 ]; then
|
|
|
|
|
if [ $_SILENT == true ]; then
|
|
|
|
|
return 0
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
@ -951,7 +955,7 @@ function RunLocalCommand {
|
|
|
|
|
local hard_max_time="${2}" # Max time to wait for command to compleet
|
|
|
|
|
__CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
|
|
|
|
|
|
|
|
|
|
if [ $_DRYRUN -ne 0 ]; then
|
|
|
|
|
if [ $_DRYRUN == true ]; then
|
|
|
|
|
Logger "Dryrun: Local command [$command] not run." "NOTICE"
|
|
|
|
|
return 0
|
|
|
|
|
fi
|
|
|
|
@ -966,7 +970,7 @@ function RunLocalCommand {
|
|
|
|
|
Logger "Command failed." "ERROR"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ $_VERBOSE -eq 1 ] || [ $retval -ne 0 ]; then
|
|
|
|
|
if [ $_VERBOSE == true ] || [ $retval -ne 0 ]; then
|
|
|
|
|
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
@ -984,7 +988,7 @@ function RunRemoteCommand {
|
|
|
|
|
|
|
|
|
|
CheckConnectivity3rdPartyHosts
|
|
|
|
|
CheckConnectivityRemoteHost
|
|
|
|
|
if [ $_DRYRUN -ne 0 ]; then
|
|
|
|
|
if [ $_DRYRUN == true ]; then
|
|
|
|
|
Logger "Dryrun: Local command [$command] not run." "NOTICE"
|
|
|
|
|
return 0
|
|
|
|
|
fi
|
|
|
|
@ -1001,7 +1005,7 @@ function RunRemoteCommand {
|
|
|
|
|
Logger "Command failed." "ERROR"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ] && ([ $_VERBOSE -eq 1 ] || [ $retval -ne 0 ])
|
|
|
|
|
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ] && ([ $_VERBOSE == true ] || [ $retval -ne 0 ])
|
|
|
|
|
then
|
|
|
|
|
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
|
|
|
|
|
fi
|
|
|
|
@ -1075,7 +1079,7 @@ function CheckConnectivity3rdPartyHosts {
|
|
|
|
|
if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug
|
|
|
|
|
|
|
|
|
|
if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then
|
|
|
|
|
remote_3rd_party_success=0
|
|
|
|
|
remote_3rd_party_success=false
|
|
|
|
|
for i in $REMOTE_3RD_PARTY_HOSTS
|
|
|
|
|
do
|
|
|
|
|
eval "$PING_CMD $i > /dev/null 2>&1" &
|
|
|
|
@ -1083,11 +1087,11 @@ function CheckConnectivity3rdPartyHosts {
|
|
|
|
|
if [ $? != 0 ]; then
|
|
|
|
|
Logger "Cannot ping 3rd party host $i" "NOTICE"
|
|
|
|
|
else
|
|
|
|
|
remote_3rd_party_success=1
|
|
|
|
|
remote_3rd_party_success=true
|
|
|
|
|
fi
|
|
|
|
|
done
|
|
|
|
|
|
|
|
|
|
if [ $remote_3rd_party_success -ne 1 ]; then
|
|
|
|
|
if [ $remote_3rd_party_success == false ]; then
|
|
|
|
|
Logger "No remote 3rd party host responded to ping. No internet ?" "ERROR"
|
|
|
|
|
return 1
|
|
|
|
|
else
|
|
|
|
@ -1118,14 +1122,14 @@ function __CheckArguments {
|
|
|
|
|
# In order to avoid this, we need to iterate over ${4} and count
|
|
|
|
|
|
|
|
|
|
local iterate=4
|
|
|
|
|
local fetchArguments=1
|
|
|
|
|
local fetchArguments=true
|
|
|
|
|
local argList=""
|
|
|
|
|
local countedArguments
|
|
|
|
|
while [ $fetchArguments -eq 1 ]; do
|
|
|
|
|
while [ $fetchArguments == true ]; do
|
|
|
|
|
cmd='argument=${'$iterate'}'
|
|
|
|
|
eval $cmd
|
|
|
|
|
if [ "$argument" = "" ]; then
|
|
|
|
|
fetchArguments=0
|
|
|
|
|
fetchArguments=false
|
|
|
|
|
else
|
|
|
|
|
argList="$arg_list [Argument $(($iterate-3)): $argument]"
|
|
|
|
|
iterate=$(($iterate+1))
|
|
|
|
@ -1267,7 +1271,7 @@ function PreInit {
|
|
|
|
|
|
|
|
|
|
## Set rsync default arguments
|
|
|
|
|
RSYNC_ARGS="-rltD"
|
|
|
|
|
if [ "$_DRYRUN" -eq 1 ]; then
|
|
|
|
|
if [ "$_DRYRUN" == true ]; then
|
|
|
|
|
RSYNC_DRY_ARG="-n"
|
|
|
|
|
DRY_WARNING="/!\ DRY RUN"
|
|
|
|
|
else
|
|
|
|
|