From 51a4c8bcce8216ec558e8dcdc4e89a450d5fafb0 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 15 Mar 2020 22:51:21 +0100 Subject: [PATCH] Rebuilt targets --- dev/debug_osync.sh | 3 +- install.sh | 1910 +++++++++++++++++++++++++++++++++++++++++++- osync-batch.sh | 1907 ++++++++++++++++++++++++++++++++++++++++++- osync.sh | 3 +- 4 files changed, 3816 insertions(+), 7 deletions(-) diff --git a/dev/debug_osync.sh b/dev/debug_osync.sh index 8d4e30b..4f19f9a 100755 --- a/dev/debug_osync.sh +++ b/dev/debug_osync.sh @@ -43,7 +43,7 @@ CONFIG_FILE_REVISION_REQUIRED=1.3.0 # CleanUp no #__WITH_PARANOIA_DEBUG _OFUNCTIONS_VERSION=2.3.0-RC4 -_OFUNCTIONS_BUILD=2020031501 +_OFUNCTIONS_BUILD=2020031502 _OFUNCTIONS_BOOTSTRAP=true if ! type "$BASH" > /dev/null; then @@ -436,6 +436,7 @@ function GenericTrapQuit { exit $exitcode } +#### TrapQuit SUBSET END #### function CleanUp { # Exit controlmaster before it's socket gets deleted diff --git a/install.sh b/install.sh index 45fb3dc..d5cef78 100755 --- a/install.sh +++ b/install.sh @@ -10,7 +10,7 @@ PROGRAM_BINARY=$PROGRAM".sh" PROGRAM_BATCH=$PROGRAM"-batch.sh" SSH_FILTER="ssh_filter.sh" -SCRIPT_BUILD=2020031501 +SCRIPT_BUILD=2020031502 INSTANCE_ID="installer-$SCRIPT_BUILD" ## osync / obackup / pmocr / zsnap install script @@ -18,7 +18,7 @@ INSTANCE_ID="installer-$SCRIPT_BUILD" ## Please adapt this to fit your distro needs _OFUNCTIONS_VERSION=2.3.0-RC4 -_OFUNCTIONS_BUILD=2020031501 +_OFUNCTIONS_BUILD=2020031502 _OFUNCTIONS_BOOTSTRAP=true if ! type "$BASH" > /dev/null; then @@ -395,6 +395,7 @@ function GenericTrapQuit { exit $exitcode } +#### TrapQuit SUBSET END #### function CleanUp { # Exit controlmaster before it's socket gets deleted @@ -611,6 +612,1909 @@ function GenericTrapQuit { exit $exitcode } +#### TrapQuit SUBSET END #### + +#### CleanUp SUBSET #### +function CleanUp { + # Exit controlmaster before it's socket gets deleted + if [ "$SSH_CONTROLMASTER" == true ] && [ "$SSH_CMD" != "" ]; then + $SSH_CMD -O exit + fi + + if [ "$_DEBUG" != true ]; then + # Removing optional remote $RUN_DIR that goes into local $RUN_DIR + if [ -d "$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" ]; then + rm -rf "$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" + fi + # Removing all temporary run files + rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP" + # Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements) + rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp" + fi +} + +#### CleanUp SUBSET END #### + +#### OFUNCTIONS MICRO SUBSET END #### + +# osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending +function SendAlert { + local runAlert="${1:-false}" # Specifies if current message is sent while running or at the end of a run + local attachment="${2:-true}" # Should we send the log file as attachment + + + local attachmentFile + local subject + local body + + if [ "$DESTINATION_MAILS" == "" ]; then + return 0 + fi + + if [ "$_DEBUG" == true ]; then + Logger "Debug mode, no warning mail will be sent." "NOTICE" + return 0 + fi + + if [ $attachment == true ]; then + attachmentFile="$LOG_FILE" + if type "$COMPRESSION_PROGRAM" > /dev/null 2>&1; then + eval "cat \"$LOG_FILE\" \"$COMPRESSION_PROGRAM\" > \"$ALERT_LOG_FILE\"" + if [ $? -eq 0 ]; then + attachmentFile="$ALERT_LOG_FILE" + fi + fi + fi + + body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of current log"$'\n\n'"$(tail -n 1000 "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP")" + + if [ $ERROR_ALERT == true ]; then + subject="Error alert for $INSTANCE_ID" + elif [ $WARN_ALERT == true ]; then + subject="Warning alert for $INSTANCE_ID" + else + subject="Alert for $INSTANCE_ID" + fi + + if [ $runAlert == true ]; then + subject="Currently runing - $subject" + else + subject="Finished run - $subject" + fi + + SendEmail "$subject" "$body" "$DESTINATION_MAILS" "$attachmentFile" "$SENDER_MAIL" "$SMTP_SERVER" "$SMTP_PORT" "$SMTP_ENCRYPTION" "$SMTP_USER" "$SMTP_PASSWORD" + + # Delete tmp log file + if [ "$attachment" == true ]; then + if [ -f "$ALERT_LOG_FILE" ]; then + rm -f "$ALERT_LOG_FILE" + fi + fi +} + +# Generic email sending function. +# Usage (linux / BSD), attachment is optional, can be "/path/to/my.file" or "" +# SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file" +# Usage (Windows, make sure you have mailsend.exe in executable path, see http://github.com/muquit/mailsend) +# attachment is optional but must be in windows format like "c:\\some\path\\my.file", or "" +# smtp_server.domain.tld is mandatory, as is smtpPort (should be 25, 465 or 587) +# encryption can be set to tls, ssl or none +# smtpUser and smtpPassword are optional +# SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file" "senderMail@example.com" "smtpServer.domain.tld" "smtpPort" "encryption" "smtpUser" "smtpPassword" + +# If text is received as attachment ATT00001.bin or noname, consider adding the following to /etc/mail.rc +#set ttycharset=iso-8859-1 +#set sendcharsets=iso-8859-1 +#set encoding=8bit + +function SendEmail { + local subject="${1}" + local message="${2}" + local destinationMails="${3}" + local attachment="${4}" + local senderMail="${5}" + local smtpServer="${6}" + local smtpPort="${7}" + local encryption="${8}" + local smtpUser="${9}" + local smtpPassword="${10}" + + + local mail_no_attachment= + local attachment_command= + + local encryption_string= + local auth_string= + + local i + + if [ "${destinationMails}" != "" ]; then + for i in "${destinationMails[@]}"; do + if [ $(CheckRFC822 "$i") -ne 1 ]; then + Logger "Given email [$i] does not seem to be valid." "WARN" + fi + done + else + Logger "No valid email addresses given." "WARN" + return 1 + fi + + # Prior to sending an email, convert its body if needed + if [ "$MAIL_BODY_CHARSET" != "" ]; then + if type iconv > /dev/null 2>&1; then + echo "$message" | iconv -f UTF-8 -t $MAIL_BODY_CHARSET -o "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.iconv.$SCRIPT_PID.$TSTAMP" + message="$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.iconv.$SCRIPT_PID.$TSTAMP")" + else + Logger "iconv utility not installed. Will not convert email charset." "NOTICE" + fi + fi + + if [ ! -f "$attachment" ]; then + attachment_command="-a $attachment" + mail_no_attachment=1 + else + mail_no_attachment=0 + fi + + if [ "$LOCAL_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ]; then + if [ "$smtpPort" == "" ]; then + Logger "Missing smtp port, assuming 25." "WARN" + smtpPort=25 + fi + if type sendmail > /dev/null 2>&1; then + if [ "$encryption" == "tls" ]; then + echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$senderMail" -H "exec openssl s_client -quiet -tls1_2 -starttls smtp -connect $smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails" + elif [ "$encryption" == "ssl" ]; then + echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$senderMail" -H "exec openssl s_client -quiet -connect $smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails" + elif [ "$encryption" == "none" ]; then + echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$senderMail" -S "$smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails" + else + echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$senderMail" -S "$smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails" + Logger "Bogus email encryption used [$encryption]." "WARN" + fi + + if [ $? -ne 0 ]; then + Logger "Cannot send alert mail via $(type -p sendmail) !!!" "WARN" + # Do not bother try other mail systems with busybox + return 1 + else + return 0 + fi + else + Logger "Sendmail not present. Will not send any mail" "WARN" + return 1 + fi + fi + + if type mutt > /dev/null 2>&1 ; then + # We need to replace spaces with comma in order for mutt to be able to process multiple destinations + echo "$message" | $(type -p mutt) -x -s "$subject" "${destinationMails// /,}" $attachment_command + if [ $? -ne 0 ]; then + Logger "Cannot send mail via $(type -p mutt) !!!" "WARN" + else + Logger "Sent mail using mutt." "NOTICE" + return 0 + fi + fi + + if type mail > /dev/null 2>&1 ; then + # We need to detect which version of mail is installed + if ! $(type -p mail) -V > /dev/null 2>&1; then + # This may be MacOS mail program + attachment_command="" + elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then + attachment_command="-A $attachment" + elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V > /dev/null; then + attachment_command="-a$attachment" + else + attachment_command="" + fi + + echo "$message" | $(type -p mail) $attachment_command -s "$subject" "$destinationMails" + if [ $? -ne 0 ]; then + Logger "Cannot send mail via $(type -p mail) with attachments !!!" "WARN" + echo "$message" | $(type -p mail) -s "$subject" "$destinationMails" + if [ $? -ne 0 ]; then + Logger "Cannot send mail via $(type -p mail) without attachments !!!" "WARN" + else + Logger "Sent mail using mail command without attachment." "NOTICE" + return 0 + fi + else + Logger "Sent mail using mail command." "NOTICE" + return 0 + fi + fi + + if type sendmail > /dev/null 2>&1 ; then + echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) "$destinationMails" + if [ $? -ne 0 ]; then + Logger "Cannot send mail via $(type -p sendmail) !!!" "WARN" + else + Logger "Sent mail using sendmail command without attachment." "NOTICE" + return 0 + fi + fi + + # Windows specific + if type "mailsend.exe" > /dev/null 2>&1 ; then + if [ "$senderMail" == "" ]; then + Logger "Missing sender email." "ERROR" + return 1 + fi + if [ "$smtpServer" == "" ]; then + Logger "Missing smtp port." "ERROR" + return 1 + fi + if [ "$smtpPort" == "" ]; then + Logger "Missing smtp port, assuming 25." "WARN" + smtpPort=25 + fi + if [ "$encryption" != "tls" ] && [ "$encryption" != "ssl" ] && [ "$encryption" != "none" ]; then + Logger "Bogus smtp encryption, assuming none." "WARN" + encryption_string= + elif [ "$encryption" == "tls" ]; then + encryption_string=-starttls + elif [ "$encryption" == "ssl" ]:; then + encryption_string=-ssl + fi + if [ "$smtpUser" != "" ] && [ "$smtpPassword" != "" ]; then + auth_string="-auth -user \"$smtpUser\" -pass \"$smtpPassword\"" + fi + $(type mailsend.exe) -f "$senderMail" -t "$destinationMails" -sub "$subject" -M "$message" -attach "$attachment" -smtp "$smtpServer" -port "$smtpPort" $encryption_string $auth_string + if [ $? -ne 0 ]; then + Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN" + else + Logger "Sent mail using mailsend.exe command with attachment." "NOTICE" + return 0 + fi + fi + + # pfSense specific + if [ -f /usr/local/bin/mail.php ]; then + echo "$message" | /usr/local/bin/mail.php -s="$subject" + if [ $? -ne 0 ]; then + Logger "Cannot send mail via /usr/local/bin/mail.php (pfsense) !!!" "WARN" + else + Logger "Sent mail using pfSense mail.php." "NOTICE" + return 0 + fi + fi + + # If function has not returned 0 yet, assume it is critical that no alert can be sent + Logger "Cannot send mail (neither mutt, mail, sendmail, sendemail, mailsend (windows) or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue +} + +#### TrapError SUBSET #### +function TrapError { + local job="$0" + local line="$1" + local code="${2:-1}" + + if [ $_LOGGER_SILENT == false ]; then + (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") + fi +} +#### TrapError SUBSET END #### + +function LoadConfigFile { + local configFile="${1}" + local revisionRequired="${2}" + + + local revisionPresent + + if [ ! -f "$configFile" ]; then + Logger "Cannot load configuration file [$configFile]. Cannot start." "CRITICAL" + exit 1 + elif [[ "$configFile" != *".conf" ]]; then + Logger "Wrong configuration file supplied [$configFile]. Cannot start." "CRITICAL" + exit 1 + else + revisionPresent="$(GetConfFileValue "$configFile" "CONFIG_FILE_REVISION" true)" + if [ "$(IsNumeric "${revisionPresent%%.*}")" -eq 0 ]; then + Logger "Missing CONFIG_FILE_REVISION. Please provide a valid config file, or run the config update script." "WARN" + Logger "CONFIG_FILE_REVISION does not seem numeric [$revisionPresent]." "DEBUG" + elif [ "$revisionRequired" != "" ]; then + if [ $(VerComp "$revisionPresent" "$revisionRequired") -eq 2 ]; then + Logger "Configuration file seems out of date. Required version [$revisionRequired]. Actual version [$revisionPresent]." "CRITICAL" + exit 1 + fi + fi + # Remove everything that is not a variable assignation + grep '^[^ ]*=[^;&]*' "$configFile" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + source "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi + + CONFIG_FILE="$configFile" +} + +# Quick and dirty performance logger only used for debugging + +_OFUNCTIONS_SPINNER="|/-\\" +function Spinner { + if [ $_LOGGER_SILENT == true ] || [ "$_LOGGER_ERR_ONLY" == true ] || [ "$_SYNC_ON_CHANGES" == "initiator" ] || [ "$_SYNC_ON_CHANGES" == "target" ] ; then + return 0 + else + printf " [%c] \b\b\b\b\b\b" "$_OFUNCTIONS_SPINNER" + _OFUNCTIONS_SPINNER=${_OFUNCTIONS_SPINNER#?}${_OFUNCTIONS_SPINNER%%???} + return 0 + fi +} + +# WaitForTaskCompletion function emulation, now uses ExecTasks +function WaitForTaskCompletion { + local pids="${1}" + local softMaxTime="${2:-0}" + local hardMaxTime="${3:-0}" + local sleepTime="${4:-.05}" + local keepLogging="${5:-0}" + local counting="${6:-true}" + local spinner="${7:-true}" + local noErrorLog="${8:-false}" + local id="${9-base}" + + ExecTasks "$pids" "$id" false 0 0 "$softMaxTime" "$hardMaxTime" "$counting" "$sleepTime" "$keepLogging" "$spinner" "$noErrorlog" +} + +# ParallelExec function emulation, now uses ExecTasks +function ParallelExec { + local numberOfProcesses="${1}" + local commandsArg="${2}" + local readFromFile="${3:-false}" + local softMaxTime="${4:-0}" + local hardMaxTime="${5:-0}" + local sleepTime="${6:-.05}" + local keepLogging="${7:-0}" + local counting="${8:-true}" + local spinner="${9:-false}" + local noErrorLog="${10:-false}" + + if [ $readFromFile == true ]; then + ExecTasks "$commandsArg" "base" $readFromFile 0 0 "$softMaxTime" "$hardMaxTime" "$counting" "$sleepTime" "$keepLogging" "$spinner" "$noErrorLog" false "$numberOfProcesses" + else + ExecTasks "$commandsArg" "base" $readFromFile 0 0 "$softMaxTime" "$hardMaxTime" "$counting" "$sleepTime" "$keepLogging" "$spinner" "$noErrorLog" false "$numberOfProcesses" + fi +} + +## Main asynchronous execution function +## Function can work in: +## WaitForTaskCompletion mode: monitors given pid in background, and stops them if max execution time is reached. Suitable for multiple synchronous pids to monitor and wait for +## ParallExec mode: takes list of commands to execute in parallel per batch, and stops them if max execution time is reahed. + +## Example of improved wait $! +## ExecTasks $! "some_identifier" false 0 0 0 0 true 1 1800 false +## Example: monitor two sleep processes, warn if execution time is higher than 10 seconds, stop after 20 seconds +## sleep 15 & +## pid=$! +## sleep 20 & +## pid2=$! +## ExecTasks "some_identifier" 0 0 10 20 1 1800 true true false false 1 "$pid;$pid2" + +## Example of parallel execution of four commands, only if directories exist. Warn if execution takes more than 300 seconds. Stop if takes longer than 900 seconds. Exeute max 3 commands in parallel. +## commands="du -csh /var;du -csh /etc;du -csh /home;du -csh /usr" +## conditions="[ -d /var ];[ -d /etc ];[ -d /home];[ -d /usr]" +## ExecTasks "$commands" "some_identifier" false 0 0 300 900 true 1 1800 true false false 3 "$conditions" + +## Bear in mind that given commands and conditions need to be quoted + +## ExecTasks has the following ofunctions subfunction requirements: +## Spinner +## Logger +## JoinString +## KillChilds + +## Full call +##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes + +function ExecTasks { + # Mandatory arguments + local mainInput="${1}" # Contains list of pids / commands separated by semicolons or filepath to list of pids / commands + + # Optional arguments + local id="${2:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id + local readFromFile="${3:-false}" # Is mainInput / auxInput a semicolon separated list (true) or a filepath (false) + local softPerProcessTime="${4:-0}" # Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0 + local hardPerProcessTime="${5:-0}" # Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0 + local softMaxTime="${6:-0}" # Max time (in seconds) for the whole function to run before a warning is logged, unless set to 0 + local hardMaxTime="${7:-0}" # Max time (in seconds) for the whole function to run before all pids / commands given are stopped, unless set to 0 + local counting="${8:-true}" # Should softMaxTime and hardMaxTime be accounted since function begin (true) or since script begin (false) + local sleepTime="${9:-.5}" # Seconds between each state check. The shorter the value, the snappier ExecTasks will be, but as a tradeoff, more cpu power will be used (good values are between .05 and 1) + local keepLogging="${10:-1800}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging + local spinner="${11:-true}" # Show spinner (true) or do not show anything (false) while running + local noTimeErrorLog="${12:-false}" # Log errors when reaching soft / hard execution times (false) or do not log errors on those triggers (true) + local noErrorLogsAtAll="${13:-false}" # Do not log any errros at all (useful for recursive ExecTasks checks) + + # Parallelism specific arguments + local numberOfProcesses="${14:-0}" # Number of simulanteous commands to run, given as mainInput. Set to 0 by default (WaitForTaskCompletion mode). Setting this value enables ParallelExec mode. + local auxInput="${15}" # Contains list of commands separated by semicolons or filepath fo list of commands. Exit code of those commands decide whether main commands will be executed or not + local maxPostponeRetries="${16:-3}" # If a conditional command fails, how many times shall we try to postpone the associated main command. Set this to 0 to disable postponing + local minTimeBetweenRetries="${17:-300}" # Time (in seconds) between postponed command retries + local validExitCodes="${18:-0}" # Semi colon separated list of valid main command exit codes which will not trigger errors + + + local i + + + # Since ExecTasks takes up to 17 arguments, do a quick preflight check in DEBUG mode + if [ "$_DEBUG" == true ]; then + declare -a booleans=(readFromFile counting spinner noTimeErrorLog noErrorLogsAtAll) + for i in "${booleans[@]}"; do + test="if [ \$$i != false ] && [ \$$i != true ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi" + eval "$test" + done + declare -a integers=(softPerProcessTime hardPerProcessTime softMaxTime hardMaxTime keepLogging numberOfProcesses maxPostponeRetries minTimeBetweenRetries) + for i in "${integers[@]}"; do + test="if [ $(IsNumericExpand \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi" + eval "$test" + done + fi + + # Expand validExitCodes into array + IFS=';' read -r -a validExitCodes <<< "$validExitCodes" + + # ParallelExec specific variables + local auxItemCount=0 # Number of conditional commands + local commandsArray=() # Array containing commands + local commandsConditionArray=() # Array containing conditional commands + local currentCommand # Variable containing currently processed command + local currentCommandCondition # Variable containing currently processed conditional command + local commandsArrayPid=() # Array containing commands indexed by pids + local commandsArrayOutput=() # Array containing command results indexed by pids + local postponedRetryCount=0 # Number of current postponed commands retries + local postponedItemCount=0 # Number of commands that have been postponed (keep at least one in order to check once) + local postponedCounter=0 + local isPostponedCommand=false # Is the current command from a postponed file ? + local postponedExecTime=0 # How much time has passed since last postponed condition was checked + local needsPostponing # Does currentCommand need to be postponed + local temp + + # Common variables + local pid # Current pid working on + local pidState # State of the process + local mainItemCount=0 # number of given items (pids or commands) + local readFromFile # Should we read pids / commands from a file (true) + local counter=0 + local log_ttime=0 # local time instance for comparaison + + local seconds_begin=$SECONDS # Seconds since the beginning of the script + local exec_time=0 # Seconds since the beginning of this function + + local retval=0 # return value of monitored pid process + local subRetval=0 # return value of condition commands + local errorcount=0 # Number of pids that finished with errors + local pidsArray # Array of currently running pids + local newPidsArray # New array of currently running pids for next iteration + local pidsTimeArray # Array containing execution begin time of pids + local executeCommand # Boolean to check if currentCommand can be executed given a condition + local functionMode + local softAlert=false # Does a soft alert need to be triggered, if yes, send an alert once + local failedPidsList # List containing failed pids with exit code separated by semicolons (eg : 2355:1;4534:2;2354:3) + local randomOutputName # Random filename for command outputs + local currentRunningPids # String of pids running, used for debugging purposes only + + # Initialise global variable + eval "WAIT_FOR_TASK_COMPLETION_$id=\"\"" + eval "HARD_MAX_EXEC_TIME_REACHED_$id=false" + + # Init function variables depending on mode + + if [ $numberOfProcesses -gt 0 ]; then + functionMode=ParallelExec + else + functionMode=WaitForTaskCompletion + fi + + if [ $readFromFile == false ]; then + if [ $functionMode == "WaitForTaskCompletion" ]; then + IFS=';' read -r -a pidsArray <<< "$mainInput" + mainItemCount="${#pidsArray[@]}" + else + IFS=';' read -r -a commandsArray <<< "$mainInput" + mainItemCount="${#commandsArray[@]}" + IFS=';' read -r -a commandsConditionArray <<< "$auxInput" + auxItemCount="${#commandsConditionArray[@]}" + fi + else + if [ -f "$mainInput" ]; then + mainItemCount=$(wc -l < "$mainInput") + readFromFile=true + else + Logger "Cannot read main file [$mainInput]." "WARN" + fi + if [ "$auxInput" != "" ]; then + if [ -f "$auxInput" ]; then + auxItemCount=$(wc -l < "$auxInput") + else + Logger "Cannot read aux file [$auxInput]." "WARN" + fi + fi + fi + + if [ $functionMode == "WaitForTaskCompletion" ]; then + # Force first while loop condition to be true because we don't deal with counters but pids in WaitForTaskCompletion mode + counter=$mainItemCount + fi + + + # soft / hard execution time checks that needs to be a subfunction since it is called both from main loop and from parallelExec sub loop + function _ExecTasksTimeCheck { + if [ $spinner == true ]; then + Spinner + fi + if [ $counting == true ]; then + exec_time=$((SECONDS - seconds_begin)) + else + exec_time=$SECONDS + fi + + if [ $keepLogging -ne 0 ]; then + # This log solely exists for readability purposes before having next set of logs + if [ ${#pidsArray[@]} -eq $numberOfProcesses ] && [ $log_ttime -eq 0 ]; then + log_ttime=$exec_time + Logger "There are $((mainItemCount-counter+postponedItemCount)) / $mainItemCount tasks in the queue of which $postponedItemCount are postponed. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE" + fi + if [ $(((exec_time + 1) % keepLogging)) -eq 0 ]; then + if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1 second + log_ttime=$exec_time + if [ $functionMode == "WaitForTaskCompletion" ]; then + Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE" + elif [ $functionMode == "ParallelExec" ]; then + Logger "There are $((mainItemCount-counter+postponedItemCount)) / $mainItemCount tasks in the queue of which $postponedItemCount are postponed. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE" + fi + fi + fi + fi + + if [ $exec_time -gt $softMaxTime ]; then + if [ "$softAlert" != true ] && [ $softMaxTime -ne 0 ] && [ $noTimeErrorLog != true ]; then + Logger "Max soft execution time [$softMaxTime] exceeded for task [$id] with pids [$(joinString , ${pidsArray[@]})]." "WARN" + softAlert=true + SendAlert true + fi + fi + + if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then + if [ $noTimeErrorLog != true ]; then + Logger "Max hard execution time [$hardMaxTime] exceeded for task [$id] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR" + fi + for pid in "${pidsArray[@]}"; do + KillChilds $pid true + if [ $? -eq 0 ]; then + Logger "Task with pid [$pid] stopped successfully." "NOTICE" + else + if [ $noErrorLogsAtAll != true ]; then + Logger "Could not stop task with pid [$pid]." "ERROR" + fi + fi + errorcount=$((errorcount+1)) + done + if [ $noTimeErrorLog != true ]; then + SendAlert true + fi + eval "HARD_MAX_EXEC_TIME_REACHED_$id=true" + if [ $functionMode == "WaitForTaskCompletion" ]; then + return $errorcount + else + return 129 + fi + fi + } + + function _ExecTasksPidsCheck { + newPidsArray=() + + if [ "$currentRunningPids" != "$(joinString " " ${pidsArray[@]})" ]; then + Logger "ExecTask running for pids [$(joinString " " ${pidsArray[@]})]." "DEBUG" + currentRunningPids="$(joinString " " ${pidsArray[@]})" + fi + + for pid in "${pidsArray[@]}"; do + if [ $(IsInteger $pid) -eq 1 ]; then + if kill -0 $pid > /dev/null 2>&1; then + # Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :) + pidState="$(eval $PROCESS_STATE_CMD)" + if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then + + # Check if pid hasn't run more than soft/hard perProcessTime + pidsTimeArray[$pid]=$((SECONDS - seconds_begin)) + if [ ${pidsTimeArray[$pid]} -gt $softPerProcessTime ]; then + if [ "$softAlert" != true ] && [ $softPerProcessTime -ne 0 ] && [ $noTimeErrorLog != true ]; then + Logger "Max soft execution time [$softPerProcessTime] exceeded for pid [$pid]." "WARN" + if [ "${commandsArrayPid[$pid]}]" != "" ]; then + Logger "Command was [${commandsArrayPid[$pid]}]]." "WARN" + fi + softAlert=true + SendAlert true + fi + fi + + + if [ ${pidsTimeArray[$pid]} -gt $hardPerProcessTime ] && [ $hardPerProcessTime -ne 0 ]; then + if [ $noTimeErrorLog != true ] && [ $noErrorLogsAtAll != true ]; then + Logger "Max hard execution time [$hardPerProcessTime] exceeded for pid [$pid]. Stopping command execution." "ERROR" + if [ "${commandsArrayPid[$pid]}]" != "" ]; then + Logger "Command was [${commandsArrayPid[$pid]}]]." "WARN" + fi + fi + KillChilds $pid true + if [ $? -eq 0 ]; then + Logger "Command with pid [$pid] stopped successfully." "NOTICE" + else + if [ $noErrorLogsAtAll != true ]; then + Logger "Could not stop command with pid [$pid]." "ERROR" + fi + fi + errorcount=$((errorcount+1)) + + if [ $noTimeErrorLog != true ]; then + SendAlert true + fi + fi + + newPidsArray+=($pid) + fi + else + # pid is dead, get its exit code from wait command + wait $pid + retval=$? + # Check for valid exit codes + if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then + if [ $noErrorLogsAtAll != true ]; then + Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "ERROR" + if [ "$functionMode" == "ParallelExec" ]; then + Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR" + fi + if [ -f "${commandsArrayOutput[$pid]}" ]; then + Logger "Truncated output:\n$(head -c16384 "${commandsArrayOutput[$pid]}")" "ERROR" + fi + fi + errorcount=$((errorcount+1)) + # Welcome to variable variable bash hell + if [ "$failedPidsList" == "" ]; then + failedPidsList="$pid:$retval" + else + failedPidsList="$failedPidsList;$pid:$retval" + fi + else + Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG" + fi + fi + fi + done + + # hasPids can be false on last iteration in ParallelExec mode + pidsArray=("${newPidsArray[@]}") + + # Trivial wait time for bash to not eat up all CPU + sleep $sleepTime + + + } + + while [ ${#pidsArray[@]} -gt 0 ] || [ $counter -lt $mainItemCount ] || [ $postponedItemCount -ne 0 ]; do + _ExecTasksTimeCheck + retval=$? + if [ $retval -ne 0 ]; then + return $retval; + fi + + # The following execution bloc is only needed in ParallelExec mode since WaitForTaskCompletion does not execute commands, but only monitors them + if [ $functionMode == "ParallelExec" ]; then + while [ ${#pidsArray[@]} -lt $numberOfProcesses ] && ([ $counter -lt $mainItemCount ] || [ $postponedItemCount -ne 0 ]); do + _ExecTasksTimeCheck + retval=$? + if [ $retval -ne 0 ]; then + return $retval; + fi + + executeCommand=false + isPostponedCommand=false + currentCommand="" + currentCommandCondition="" + needsPostponing=false + + if [ $readFromFile == true ]; then + # awk identifies first line as 1 instead of 0 so we need to increase counter + currentCommand=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$mainInput") + if [ $auxItemCount -ne 0 ]; then + currentCommandCondition=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$auxInput") + fi + + # Check if we need to fetch postponed commands + if [ "$currentCommand" == "" ]; then + currentCommand=$(awk 'NR == num_line {print; exit}' num_line=$((postponedCounter+1)) "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedMain.$id.$SCRIPT_PID.$TSTAMP") + currentCommandCondition=$(awk 'NR == num_line {print; exit}' num_line=$((postponedCounter+1)) "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedAux.$id.$SCRIPT_PID.$TSTAMP") + isPostponedCommand=true + fi + else + currentCommand="${commandsArray[$counter]}" + if [ $auxItemCount -ne 0 ]; then + currentCommandCondition="${commandsConditionArray[$counter]}" + fi + + if [ "$currentCommand" == "" ]; then + currentCommand="${postponedCommandsArray[$postponedCounter]}" + currentCommandCondition="${postponedCommandsConditionArray[$postponedCounter]}" + isPostponedCommand=true + fi + fi + + # Check if we execute postponed commands, or if we delay them + if [ $isPostponedCommand == true ]; then + # Get first value before '@' + postponedExecTime="${currentCommand%%@*}" + postponedExecTime=$((SECONDS-postponedExecTime)) + # Get everything after first '@' + temp="${currentCommand#*@}" + # Get first value before '@' + postponedRetryCount="${temp%%@*}" + # Replace currentCommand with actual filtered currentCommand + currentCommand="${temp#*@}" + + # Since we read a postponed command, we may decrase postponedItemCounter + postponedItemCount=$((postponedItemCount-1)) + #Since we read one line, we need to increase the counter + postponedCounter=$((postponedCounter+1)) + + else + postponedRetryCount=0 + postponedExecTime=0 + fi + if ([ $postponedRetryCount -lt $maxPostponeRetries ] && [ $postponedExecTime -ge $minTimeBetweenRetries ]) || [ $isPostponedCommand == false ]; then + if [ "$currentCommandCondition" != "" ]; then + Logger "Checking condition [$currentCommandCondition] for command [$currentCommand]." "DEBUG" + eval "$currentCommandCondition" & + ExecTasks $! "subConditionCheck" false 0 0 1800 3600 true $SLEEP_TIME $KEEP_LOGGING true true true + subRetval=$? + if [ $subRetval -ne 0 ]; then + # is postponing enabled ? + if [ $maxPostponeRetries -gt 0 ]; then + Logger "Condition [$currentCommandCondition] not met for command [$currentCommand]. Exit code [$subRetval]. Postponing command." "NOTICE" + postponedRetryCount=$((postponedRetryCount+1)) + if [ $postponedRetryCount -ge $maxPostponeRetries ]; then + Logger "Max retries reached for postponed command [$currentCommand]. Skipping command." "NOTICE" + else + needsPostponing=true + fi + postponedExecTime=0 + else + Logger "Condition [$currentCommandCondition] not met for command [$currentCommand]. Exit code [$subRetval]. Ignoring command." "NOTICE" + fi + else + executeCommand=true + fi + else + executeCommand=true + fi + else + needsPostponing=true + fi + + if [ $needsPostponing == true ]; then + postponedItemCount=$((postponedItemCount+1)) + if [ $readFromFile == true ]; then + echo "$((SECONDS-postponedExecTime))@$postponedRetryCount@$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedMain.$id.$SCRIPT_PID.$TSTAMP" + echo "$currentCommandCondition" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedAux.$id.$SCRIPT_PID.$TSTAMP" + else + postponedCommandsArray+=("$((SECONDS-postponedExecTime))@$postponedRetryCount@$currentCommand") + postponedCommandsConditionArray+=("$currentCommandCondition") + fi + fi + + if [ $executeCommand == true ]; then + Logger "Running command [$currentCommand]." "DEBUG" + randomOutputName=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 5) + eval "$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$pid.$randomOutputName.$SCRIPT_PID.$TSTAMP" 2>&1 & + pid=$! + pidsArray+=($pid) + commandsArrayPid[$pid]="$currentCommand" + commandsArrayOutput[$pid]="$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$pid.$randomOutputName.$SCRIPT_PID.$TSTAMP" + # Initialize pid execution time array + pidsTimeArray[$pid]=0 + else + Logger "Skipping command [$currentCommand]." "DEBUG" + fi + + if [ $isPostponedCommand == false ]; then + counter=$((counter+1)) + fi + _ExecTasksPidsCheck + done + fi + + _ExecTasksPidsCheck + done + + + # Return exit code if only one process was monitored, else return number of errors + # As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value + + eval "WAIT_FOR_TASK_COMPLETION_$id=\"$failedPidsList\"" + + if [ $mainItemCount -eq 1 ]; then + return $retval + else + return $errorcount + fi +} + +# Usage: var=$(StripSingleQuotes "$var") +function StripSingleQuotes { + local string="${1}" + + string="${string/#\'/}" # Remove singlequote if it begins string + string="${string/%\'/}" # Remove singlequote if it ends string + echo "$string" +} + +# Usage: var=$(StripDoubleQuotes "$var") +function StripDoubleQuotes { + local string="${1}" + + string="${string/#\"/}" + string="${string/%\"/}" + echo "$string" +} + +function StripQuotes { + local string="${1}" + + echo "$(StripSingleQuotes $(StripDoubleQuotes $string))" +} + +# Usage var=$(EscapeSpaces "$var") or var="$(EscapeSpaces "$var")" +function EscapeSpaces { + local string="${1}" # String on which spaces will be escaped + + echo "${string// /\\ }" +} + +# Usage var=$(EscapeDoubleQuotes "$var") or var="$(EscapeDoubleQuotes "$var")" +function EscapeDoubleQuotes { + local value="${1}" + + echo "${value//\"/\\\"}" +} + +# Usage [ $(IsNumeric $var) -eq 1 ] +function IsNumeric { + local value="${1}" + + if type expr > /dev/null 2>&1; then + expr "$value" : '^[-+]\{0,1\}[0-9]*\.\{0,1\}[0-9]\{1,\}$' > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo 1 + else + echo 0 + fi + else + if [[ $value =~ ^[-+]?[0-9]+([.][0-9]+)?$ ]]; then + echo 1 + else + echo 0 + fi + fi +} + +function IsNumericExpand { + eval "local value=\"${1}\"" # Needed eval so variable variables can be processed + + echo $(IsNumeric "$value") +} + +#### HumanToNumeric SUBSET #### +# Converts human readable sizes into integer kilobyte sizes +# Usage numericSize="$(HumanToNumeric $humanSize)" +function HumanToNumeric { + local value="${1}" + + local notation + local suffix + local suffixPresent + local multiplier + + notation=(K M G T P E) + for suffix in "${notation[@]}"; do + multiplier=$((multiplier+1)) + if [[ "$value" == *"$suffix"* ]]; then + suffixPresent=$suffix + break; + fi + done + + if [ "$suffixPresent" != "" ]; then + value=${value%$suffix*} + value=${value%.*} + # /1024 since we convert to kilobytes instead of bytes + value=$((value*(1024**multiplier/1024))) + else + value=${value%.*} + fi + + echo $value +} +#### HumanToNumeric SUBSET END #### + +#### CheckRFC822 SUBSET #### +# Checks email address validity +function CheckRFC822 { + local mail="${1}" + local rfc822="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$" + + if [[ $mail =~ $rfc822 ]]; then + echo 1 + else + echo 0 + fi +} +#### CheckRFC822 SUBSET END #### + +#### UrlEncode SUBSET #### +## Modified version of https://gist.github.com/cdown/1163649 +function UrlEncode { + local length="${#1}" + + local i + + local LANG=C + for i in $(seq 0 $((length-1))); do + local c="${1:i:1}" + case $c in + [a-zA-Z0-9.~_-]) + printf "$c" + ;; + *) + printf '%%%02X' "'$c" + ;; + esac + done +} +#### UrlEncode SUBSET END #### + +function UrlDecode { + local urlEncoded="${1//+/ }" + + printf '%b' "${urlEncoded//%/\\x}" +} + +#### ArrayContains SUBSET #### +## Modified version of http://stackoverflow.com/a/8574392 +## Usage: [ $(ArrayContains "needle" "${haystack[@]}") -eq 1 ] +function ArrayContains () { + local needle="${1}" + local haystack="${2}" + local e + + if [ "$needle" != "" ] && [ "$haystack" != "" ]; then + for e in "${@:2}"; do + if [ "$e" == "$needle" ]; then + echo 1 + return + fi + done + fi + echo 0 + return +} +#### ArrayContains SUBSET END #### + +#### GetLocalOS SUBSET #### +function GetLocalOS { + local localOsVar + local localOsName + local localOsVer + + # There is no good way to tell if currently running in BusyBox shell. Using sluggish way. + if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then + localOsVar="BusyBox" + elif set -o | grep "winxp" > /dev/null; then + localOsVar="BusyBox-w32" + else + # Detecting the special ubuntu userland in Windows 10 bash + if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then + localOsVar="Microsoft" + else + localOsVar="$(uname -spior 2>&1)" + if [ $? -ne 0 ]; then + localOsVar="$(uname -v 2>&1)" + if [ $? -ne 0 ]; then + localOsVar="$(uname)" + fi + fi + fi + fi + + case $localOsVar in + # Android uname contains both linux and android, keep it before linux entry + *"Android"*) + LOCAL_OS="Android" + ;; + *"Linux"*) + LOCAL_OS="Linux" + ;; + *"BSD"*) + LOCAL_OS="BSD" + ;; + *"MINGW32"*|*"MINGW64"*|*"MSYS"*) + LOCAL_OS="msys" + ;; + *"CYGWIN"*) + LOCAL_OS="Cygwin" + ;; + *"Microsoft"*|*"MS/Windows"*) + LOCAL_OS="WinNT10" + ;; + *"Darwin"*) + LOCAL_OS="MacOSX" + ;; + *"BusyBox"*) + LOCAL_OS="BusyBox" + ;; + *) + if [ "$IGNORE_OS_TYPE" == true ]; then + Logger "Running on unknown local OS [$localOsVar]." "WARN" + return + fi + if [ "$_OFUNCTIONS_VERSION" != "" ]; then + Logger "Running on >> $localOsVar << not supported. Please report to the author." "ERROR" + fi + exit 1 + ;; + esac + + # Get linux versions + if [ -f "/etc/os-release" ]; then + localOsName="$(GetConfFileValue "/etc/os-release" "NAME" true)" + localOsVer="$(GetConfFileValue "/etc/os-release" "VERSION" true)" + elif [ "$LOCAL_OS" == "BusyBox" ]; then + localOsVer="$(ls --help 2>&1 | head -1 | cut -f2 -d' ')" + localOsName="BusyBox" + fi + + # Get Host info for Windows + if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "WinNT10" ]; then + localOsVar="$localOsVar $(uname -a)" + if [ "$PROGRAMW6432" != "" ]; then + LOCAL_OS_BITNESS=64 + LOCAL_OS_FAMILY="Windows" + elif [ "$PROGRAMFILES" != "" ]; then + LOCAL_OS_BITNESS=32 + LOCAL_OS_FAMILY="Windows" + # Case where running on BusyBox but no program files defined + elif [ "$LOCAL_OS" == "BusyBox" ]; then + LOCAL_OS_FAMILY="Unix" + fi + # Get Host info for Unix + else + LOCAL_OS_FAMILY="Unix" + fi + + if [ "$LOCAL_OS_FAMILY" == "Unix" ]; then + if uname -m | grep '64' > /dev/null 2>&1; then + LOCAL_OS_BITNESS=64 + else + LOCAL_OS_BITNESS=32 + fi + fi + + LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer) $LOCAL_OS_BITNESS-bit $LOCAL_OS_FAMILY" + + if [ "$_OFUNCTIONS_VERSION" != "" ]; then + Logger "Local OS: [$LOCAL_OS_FULL]." "DEBUG" + fi +} +#### GetLocalOS SUBSET END #### + + +#### OFUNCTIONS MINI SUBSET END #### + +function GetRemoteOS { + + if [ "$REMOTE_OPERATION" != true ]; then + return 0 + fi + + local remoteOsVar + +$SSH_CMD env LC_ALL=C env _REMOTE_TOKEN="$_REMOTE_TOKEN" bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 +_REMOTE_TOKEN="(o_0)" + +function GetOs { + local localOsVar + local localOsName + local localOsVer + local localOsBitness + local localOsFamily + + local osInfo="/etc/os-release" + + # There is no good way to tell if currently running in BusyBox shell. Using sluggish way. + if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then + localOsVar="BusyBox" + else + # Detecting the special ubuntu userland in Windows 10 bash + if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then + localOsVar="Microsoft" + else + localOsVar="$(uname -spior 2>&1)" + if [ $? -ne 0 ]; then + localOsVar="$(uname -v 2>&1)" + if [ $? -ne 0 ]; then + localOsVar="$(uname)" + fi + fi + fi + fi + # Get linux versions + if [ -f "$osInfo" ]; then + localOsName=$(grep "^NAME=" "$osInfo") + localOsName="${localOsName##*=}" + localOsVer=$(grep "^VERSION=" "$osInfo") + localOsVer="${localOsVer##*=}" + elif [ "$localOsVar" == "BusyBox" ]; then + localOsVer=$(ls --help 2>&1 | head -1 | cut -f2 -d' ') + localOsName="BusyBox" + fi + + # Get Host info for Windows + case $localOsVar in + *"MINGW32"*|*"MINGW64"*|*"MSYS"*|*"CYGWIN*"|*"Microsoft"*|*"WinNT10*") + if [ "$PROGRAMW6432" != "" ]; then + localOsBitness=64 + localOsFamily="Windows" + elif [ "$PROGRAMFILES" != "" ]; then + localOsBitness=32 + localOsFamily="Windows" + # Case where running on BusyBox but no program files defined + elif [ "$localOsVar" == "BusyBox" ]; then + localOsFamily="Unix" + fi + ;; + *) + localOsFamily="Unix" + if uname -m | grep '64' > /dev/null 2>&1; then + localOsBitness=64 + else + localOsBitness=32 + fi + ;; + esac + + echo "$localOsVar ($localOsName $localOsVer) $localOsBitness-bit $localOsFamily" +} + +GetOs + +ENDSSH + if [ $? -ne 0 ]; then + Logger "Cannot connect to remote system [$REMOTE_HOST] port [$REMOTE_PORT] as [$REMOTE_USER]." "CRITICAL" + if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then + Logger "$(head -c16384 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "ERROR" + fi + exit 1 + fi + + + if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then + remoteOsVar="$(head -c16384 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" + case $remoteOsVar in + *"Android"*) + REMOTE_OS="Android" + ;; + *"Linux"*) + REMOTE_OS="Linux" + ;; + *"BSD"*) + REMOTE_OS="BSD" + ;; + *"MINGW32"*|*"MINGW64"*|*"MSYS"*) + REMOTE_OS="msys" + ;; + *"CYGWIN"*) + REMOTE_OS="Cygwin" + ;; + *"Microsoft"*) + REMOTE_OS="WinNT10" + ;; + *"Darwin"*) + REMOTE_OS="MacOSX" + ;; + *"BusyBox"*) + REMOTE_OS="BusyBox" + ;; + *"ssh"*|*"SSH"*) + Logger "Cannot connect to remote system." "CRITICAL" + exit 1 + ;; + *) + if [ "$IGNORE_OS_TYPE" == true ]; then #DOC: Undocumented debug only setting + Logger "Running on unknown remote OS [$remoteOsVar]." "WARN" + return + fi + Logger "Running on remote OS failed. Please report to the author if the OS is not supported." "CRITICAL" + Logger "Remote OS said:\n$remoteOsVar" "CRITICAL" + exit 1 + esac + Logger "Remote OS: [$remoteOsVar]." "DEBUG" + else + Logger "Cannot get Remote OS" "CRITICAL" + fi +} + +function RunLocalCommand { + local command="${1}" # Command to run + local hardMaxTime="${2}" # Max time to wait for command to compleet + + if [ $_DRYRUN == true ]; then + Logger "Dryrun: Local command [$command] not run." "NOTICE" + return 0 + fi + + Logger "Running command [$command] on local host." "NOTICE" + eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 & + + ExecTasks $! "${FUNCNAME[0]}" false 0 0 0 $hardMaxTime true $SLEEP_TIME $KEEP_LOGGING + retval=$? + if [ $retval -eq 0 ]; then + Logger "Command succeded." "NOTICE" + else + Logger "Command failed." "ERROR" + fi + + if [ $_LOGGER_VERBOSE == true ] || [ $retval -ne 0 ]; then + Logger "Truncated output:\n$(head -c16384 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "NOTICE" + fi + + if [ "$STOP_ON_CMD_ERROR" == true ] && [ $retval -ne 0 ]; then + Logger "Stopping on command execution error." "CRITICAL" + exit 1 + fi +} + +## Runs remote command $1 and waits for completition in $2 seconds +function RunRemoteCommand { + local command="${1}" # Command to run + local hardMaxTime="${2}" # Max time to wait for command to compleet + + + if [ "$REMOTE_OPERATION" != true ]; then + Logger "Ignoring remote command [$command] because remote host is not configured." "WARN" + return 0 + fi + + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + if [ $_DRYRUN == true ]; then + Logger "Dryrun: Local command [$command] not run." "NOTICE" + return 0 + fi + + Logger "Running command [$command] on remote host." "NOTICE" + cmd=$SSH_CMD' "env LC_ALL=C env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + ExecTasks $! "${FUNCNAME[0]}" false 0 0 0 $hardMaxTime true $SLEEP_TIME $KEEP_LOGGING + retval=$? + if [ $retval -eq 0 ]; then + Logger "Command succeded." "NOTICE" + else + Logger "Command failed." "ERROR" + fi + + if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ] && ([ $_LOGGER_VERBOSE == true ] || [ $retval -ne 0 ]) + then + Logger "Truncated output:\n$(head -c16384 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "NOTICE" + fi + + if [ "$STOP_ON_CMD_ERROR" == true ] && [ $retval -ne 0 ]; then + Logger "Stopping on command execution error." "CRITICAL" + exit 1 + fi +} + +function RunBeforeHook { + + local pids + + if [ "$LOCAL_RUN_BEFORE_CMD" != "" ]; then + RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE & + pids="$!" + fi + + if [ "$REMOTE_RUN_BEFORE_CMD" != "" ]; then + RunRemoteCommand "$REMOTE_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE & + pids="$pids;$!" + fi + if [ "$pids" != "" ]; then + ExecTasks $pids "${FUNCNAME[0]}" false 0 0 0 0 true $SLEEP_TIME $KEEP_LOGGING + fi +} + +function RunAfterHook { + + local pids + + if [ "$LOCAL_RUN_AFTER_CMD" != "" ]; then + RunLocalCommand "$LOCAL_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER & + pids="$!" + fi + + if [ "$REMOTE_RUN_AFTER_CMD" != "" ]; then + RunRemoteCommand "$REMOTE_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER & + pids="$pids;$!" + fi + if [ "$pids" != "" ]; then + ExecTasks $pids "${FUNCNAME[0]}" false 0 0 0 0 true $SLEEP_TIME $KEEP_LOGGING + fi +} + +function CheckConnectivityRemoteHost { + + local retval + + + if [ "$REMOTE_HOST_PING" != false ] && [ "$REMOTE_OPERATION" != false ]; then + eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" & + ExecTasks $! "${FUNCNAME[0]}" false 0 0 60 180 true $SLEEP_TIME $KEEP_LOGGING + retval=$? + if [ $retval -ne 0 ]; then + Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN" + return $retval + fi + fi +} + +function CheckConnectivity3rdPartyHosts { + + local remote3rdPartySuccess + local retval + local i + + + if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then + remote3rdPartySuccess=false + for i in $REMOTE_3RD_PARTY_HOSTS + do + eval "$PING_CMD $i > /dev/null 2>&1" & + ExecTasks $! "${FUNCNAME[0]}" false 0 0 60 180 true $SLEEP_TIME $KEEP_LOGGING + retval=$? + if [ $retval -ne 0 ]; then + Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE" + else + remote3rdPartySuccess=true + fi + done + + if [ $remote3rdPartySuccess == false ]; then + Logger "No remote 3rd party host responded to ping. No internet ?" "WARN" + return 1 + else + return 0 + fi + fi +} + +function RsyncPatternsAdd { + local patternType="${1}" # exclude or include + local pattern="${2}" + + local rest + + # Disable globbing so wildcards from exclusions do not get expanded + set -f + rest="$pattern" + while [ -n "$rest" ] + do + # Take the string until first occurence until $PATH_SEPARATOR_CHAR + str="${rest%%$PATH_SEPARATOR_CHAR*}" + # Handle the last case + if [ "$rest" == "${rest/$PATH_SEPARATOR_CHAR/}" ]; then + rest= + else + # Cut everything before the first occurence of $PATH_SEPARATOR_CHAR + rest="${rest#*$PATH_SEPARATOR_CHAR}" + fi + if [ "$RSYNC_PATTERNS" == "" ]; then + RSYNC_PATTERNS="--$patternType=\"$str\"" + else + RSYNC_PATTERNS="$RSYNC_PATTERNS --$patternType=\"$str\"" + fi + done + set +f +} + +function RsyncPatternsFromAdd { + local patternType="${1}" + local patternFrom="${2}" + + ## Check if the exclude list has a full path, and if not, add the config file path if there is one + if [ "$(basename "$patternFrom")" == "$patternFrom" ]; then + patternFrom="$(dirname "$CONFIG_FILE")/$patternFrom" + fi + + if [ -e "$patternFrom" ]; then + RSYNC_PATTERNS="$RSYNC_PATTERNS --$patternType-from=\"$patternFrom\"" + fi +} + +function RsyncPatterns { + + if [ "$RSYNC_PATTERN_FIRST" == "exclude" ]; then + if [ "$RSYNC_EXCLUDE_PATTERN" != "" ]; then + RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN" + fi + if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM" + fi + if [ "$RSYNC_INCLUDE_PATTERN" != "" ]; then + RsyncPatternsAdd "include" "$RSYNC_INCLUDE_PATTERN" + fi + if [ "$RSYNC_INCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM" + fi + # Use default include first for quicksync runs + elif [ "$RSYNC_PATTERN_FIRST" == "include" ] || [ "$_QUICK_SYNC" == "2" ]; then + if [ "$RSYNC_INCLUDE_PATTERN" != "" ]; then + RsyncPatternsAdd "include" "$RSYNC_INCLUDE_PATTERN" + fi + if [ "$RSYNC_INCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM" + fi + if [ "$RSYNC_EXCLUDE_PATTERN" != "" ]; then + RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN" + fi + if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM" + fi + else + # osync target-helper specific clause + if [ "$_SYNC_ON_CHANGES" != "target" ]; then + Logger "Bogus RSYNC_PATTERN_FIRST value in config file. Will not use rsync patterns." "WARN" + fi + fi +} + +function PreInit { + + local compressionString + + ## SSH compression + if [ "$SSH_COMPRESSION" != false ]; then + SSH_COMP=-C + else + SSH_COMP= + fi + + ## Ignore SSH known host verification + if [ "$SSH_IGNORE_KNOWN_HOSTS" == true ]; then + SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" + fi + + ## SSH ControlMaster Multiplexing + if [ "$SSH_CONTROLMASTER" == true ]; then + SSH_OPTS="$SSH_OPTS -o ControlMaster=auto -o ControlPersist=yes -o ControlPath=\"$RUN_DIR/$PROGRAM.ctrlm.%r@%h.$SCRIPT_PID.$TSTAMP\"" + fi + + ## Optional SSH arguments + if [ "$SSH_OPTIONAL_ARGS" != "" ]; then + SSH_OPTS="$SSH_OPTS $SSH_OPTIONAL_ARGS" + fi + + ## Support for older config files without RSYNC_EXECUTABLE option + if [ "$RSYNC_EXECUTABLE" == "" ]; then + RSYNC_EXECUTABLE=rsync + fi + + ## Sudo execution option + if [ "$SUDO_EXEC" == true ]; then + if [ "$RSYNC_REMOTE_PATH" != "" ]; then + RSYNC_PATH="sudo $RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" + else + RSYNC_PATH="sudo $RSYNC_EXECUTABLE" + fi + COMMAND_SUDO="sudo -E" + else + if [ "$RSYNC_REMOTE_PATH" != "" ]; then + RSYNC_PATH="$RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" + else + RSYNC_PATH="$RSYNC_EXECUTABLE" + fi + COMMAND_SUDO="" + fi + + ## Set compression executable and extension + if [ "$(IsInteger $COMPRESSION_LEVEL)" -eq 0 ]; then + COMPRESSION_LEVEL=3 + fi +} + +function PostInit { + + # Define remote commands + if [ -f "$SSH_RSA_PRIVATE_KEY" ]; then + SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" + SCP_CMD="$(type -p scp) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT" + RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT" + elif [ -f "$SSH_PASSWORD_FILE" ]; then + SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" + SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -q -P $REMOTE_PORT" + RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS -p $REMOTE_PORT" + else + SSH_PASSWORD="" + SSH_CMD="" + SCP_CMD="" + RSYNC_SSH_CMD="" + fi +} + +function SetCompression { + ## Busybox fix (Termux xz command does not support compression at all) + if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then + compressionString="" + if type gzip > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| gzip -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + else + COMPRESSION_PROGRAM= + COMPRESSION_EXTENSION= + fi + else + compressionString=" -$COMPRESSION_LEVEL" + + if type xz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| xz -c$compressionString" + COMPRESSION_EXTENSION=.xz + elif type lzma > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| lzma -c$compressionString" + COMPRESSION_EXTENSION=.lzma + elif type pigz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| pigz -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + COMPRESSION_OPTIONS=--rsyncable + elif type gzip > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| gzip -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + COMPRESSION_OPTIONS=--rsyncable + else + COMPRESSION_PROGRAM= + COMPRESSION_EXTENSION= + fi + fi + + if [ ".${ALERT_LOG_FILE##*.}" != "$COMPRESSION_EXTENSION" ]; then + ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" + fi +} + +function InitLocalOSDependingSettings { + + ## If running under Msys, some commands do not run the same way + ## Using mingw version of find instead of windows one + ## Getting running processes is quite different + ## Ping command is not the same + if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "Microsoft" ] || [ "$LOCAL_OS" == "WinNT10" ]; then + + # Newer bash on Win10 finally uses integrated find command instead of windows one + if [ -f "/usr/bin/find" ]; then + FIND_CMD="/usr/bin/find" + elif [ -f "/bin/find" ]; then + FIND_CMD="/bin/find" + else + FIND_CMD="$(dirname $BASH)/find" + fi + + # Newer bash on Windows 10 uses integrated ping whereas cygwin & msys use Windows version + if [ "$LOCAL_OS" == "WinNT10" ]; then + PING_CMD="ping -c 2 -i 1" + else + PING_CMD='$SYSTEMROOT\system32\ping -n 2' + fi + + # On BSD, when not root, min ping interval is 1s + elif [ "$LOCAL_OS" == "BSD" ] && [ "$LOCAL_USER" != "root" ]; then + FIND_CMD=find + PING_CMD="ping -c 2 -i 1" + else + FIND_CMD=find + PING_CMD="ping -c 2 -i .2" + fi + + if [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then + PROCESS_STATE_CMD="echo none" + DF_CMD="df" + else + PROCESS_STATE_CMD='ps -p$pid -o state= 2 > /dev/null' + # CentOS 5 needs -P for one line output + DF_CMD="df -P" + fi + + ## Stat command has different syntax on Linux and FreeBSD/MacOSX + if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then + # Tested on BSD and Mac + STAT_CMD="stat -f \"%Sm\"" + STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m" + SED_REGEX_ARG="-E" + else + # Tested on GNU stat, busybox and Cygwin + STAT_CMD="stat -c %y" + STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y" + SED_REGEX_ARG="-r" + fi + + # Set compression first time when we know what local os we have + SetCompression +} + +# Gets executed regardless of the need of remote connections. It is just that this code needs to get executed after we know if there is a remote os, and if yes, which one +function InitRemoteOSDependingSettings { + + if [ "$REMOTE_OS" == "msys" ] || [ "$REMOTE_OS" == "Cygwin" ]; then + REMOTE_FIND_CMD="$(dirname $BASH)/find" + else + REMOTE_FIND_CMD=find + fi + + ## Stat command has different syntax on Linux and FreeBSD/MacOSX + if [ "$REMOTE_OS" == "MacOSX" ] || [ "$REMOTE_OS" == "BSD" ]; then + REMOTE_STAT_CMD="stat -f \"%Sm\"" + REMOTE_STAT_CTIME_MTIME_CMD="stat -f \\\"%N;%c;%m\\\"" + else + REMOTE_STAT_CMD="stat --format %y" + REMOTE_STAT_CTIME_MTIME_CMD="stat -c \\\"%n;%Z;%Y\\\"" + fi + + ## Set rsync default arguments (complete with -r or -d depending on recursivity later) + RSYNC_DEFAULT_ARGS="-ltD -8" + if [ "$_DRYRUN" == true ]; then + RSYNC_DRY_ARG="-n" + DRY_WARNING="/!\ DRY RUN " + else + RSYNC_DRY_ARG="" + fi + + RSYNC_ATTR_ARGS="" + if [ "$PRESERVE_PERMISSIONS" != false ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -p" + fi + if [ "$PRESERVE_OWNER" != false ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -o" + fi + if [ "$PRESERVE_GROUP" != false ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -g" + fi + if [ "$PRESERVE_EXECUTABILITY" != false ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" --executability" + fi + if [ "$PRESERVE_ACL" == true ]; then + if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ] && [ "$LOCAL_OS" != "msys" ] && [ "$REMOTE_OS" != "msys" ] && [ "$LOCAL_OS" != "Cygwin" ] && [ "$REMOTE_OS" != "Cygwin" ] && [ "$LOCAL_OS" != "BusyBox" ] && [ "$REMOTE_OS" != "BusyBox" ] && [ "$LOCAL_OS" != "Android" ] && [ "$REMOTE_OS" != "Android" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A" + else + Logger "Disabling ACL synchronization on [$LOCAL_OS] due to lack of support." "NOTICE" + + fi + fi + if [ "$PRESERVE_XATTR" == true ]; then + if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ] && [ "$LOCAL_OS" != "msys" ] && [ "$REMOTE_OS" != "msys" ] && [ "$LOCAL_OS" != "Cygwin" ] && [ "$REMOTE_OS" != "Cygwin" ] && [ "$LOCAL_OS" != "BusyBox" ] && [ "$REMOTE_OS" != "BusyBox" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X" + else + Logger "Disabling extended attributes synchronization on [$LOCAL_OS] due to lack of support." "NOTICE" + fi + fi + if [ "$RSYNC_COMPRESS" == true ]; then + if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -zz --skip-compress=3fr/3g2/3gp/3gpp/7z/aac/ace/amr/apk/appx/appxbundle/arc/arj/arw/asf/avi/bz/bz2/cab/cr2/crypt[5678]/dat/dcr/deb/dmg/drc/ear/erf/flac/flv/gif/gpg/gz/iiq/jar/jp2/jpeg/jpg/h26[45]/k25/kdc/kgb/lha/lz/lzma/lzo/lzx/m4[apv]/mef/mkv/mos/mov/mp[34]/mpeg/mp[gv]/msi/nef/oga/ogg/ogv/opus/orf/pak/pef/png/qt/rar/r[0-9][0-9]/rz/rpm/rw2/rzip/s7z/sfark/sfx/sr2/srf/svgz/t[gb]z/tlz/txz/vob/wim/wma/wmv/xz/zip" + else + Logger "Disabling compression skips on synchronization on [$LOCAL_OS] due to lack of support." "NOTICE" + fi + fi + if [ "$COPY_SYMLINKS" == true ]; then + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -L" + fi + if [ "$KEEP_DIRLINKS" == true ]; then + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -K" + fi + if [ "$RSYNC_OPTIONAL_ARGS" != "" ]; then + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" "$RSYNC_OPTIONAL_ARGS + fi + if [ "$PRESERVE_HARDLINKS" == true ]; then + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -H" + fi + if [ "$CHECKSUM" == true ]; then + RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum" + fi + if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" --bwlimit=$BANDWIDTH" + fi + + if [ "$PARTIAL" == true ]; then + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" + RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\"" + fi + + if [ "$DELTA_COPIES" != false ]; then + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" --no-whole-file" + else + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" --whole-file" + fi + + # Set compression options again after we know what remote OS we are dealing with + SetCompression + + # Set recursive options + RSYNC_DEFAULT_NONRECURSIVE_ARGS="-d $RSYNC_DEFAULT_ARGS" + RSYNC_DEFAULT_ARGS="-r $RSYNC_DEFAULT_ARGS" +} + +## IFS debug function +function PrintIFS { + printf "IFS is: %q" "$IFS" +} + +# Process debugging +# Recursive function to get all parents from a pid +function ParentPid { + local pid="${1}" # Pid to analyse + local parent + + parent=$(ps -p $pid -o ppid=) + echo "$pid is a child of $parent" + if [ $parent -gt 0 ]; then + ParentPid $parent + fi +} + +#### VerComp SUBSET #### +# Neat version compare function found at http://stackoverflow.com/a/4025065/2635443 +# Returns 0 if equal, 1 if $1 > $2 and 2 if $1 < $2 +function VerComp () { + if [ "$1" == "" ] || [ "$2" == "" ]; then + Logger "Bogus Vercomp values [$1] and [$2]." "WARN" + return 1 + fi + + if [[ "$1" == "$2" ]] + then + echo 0 + return + fi + + local IFS=. + local i ver1=($1) ver2=($2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) + do + ver1[i]=0 + done + for ((i=0; i<${#ver1[@]}; i++)) + do + if [[ -z ${ver2[i]} ]] + then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})) + then + echo 1 + return + fi + if ((10#${ver1[i]} < 10#${ver2[i]})) + then + echo 2 + return + fi + done + + echo 0 + return +} +#### VerComp SUBSET END #### + +#### GetConfFileValue SUBSET #### +function GetConfFileValue () { + local file="${1}" + local name="${2}" + local noError="${3:-false}" + + local value + + value=$(grep "^$name=" "$file") + if [ $? -eq 0 ]; then + value="${value##*=}" + echo "$value" + else + if [ $noError == true ]; then + Logger "Cannot get value for [$name] in config file [$file]." "NOTICE" + else + Logger "Cannot get value for [$name] in config file [$file]." "ERROR" + fi + fi +} + +#### GetConfFileValue SUBSET END #### + +#### SetConfFileValue SUBSET #### +function SetConfFileValue () { + local file="${1}" + local name="${2}" + local value="${3}" + local separator="${4:-#}" + + if [ -f "$file" ]; then + if grep "^$name=" "$file" > /dev/null 2>&1; then + # Using -i.tmp for BSD compat + sed -i.tmp "s$separator^$name=.*$separator$name=$value$separator" "$file" + if [ $? -ne 0 ]; then + Logger "Cannot update value [$name] to [$value] in config file [$file]." "ERROR" + fi + rm -f "$file.tmp" + Logger "Set [$name] to [$value] in config file [$file]." "DEBUG" + else + echo "$name=$value" >> "$file" + if [ $? -ne 0 ]; then + Logger "Cannot create value [$name] to [$value] in config file [$file]." "ERROR" + fi + fi + else + echo "$name=$value" > "$file" + if [ $? -ne 0 ]; then + Logger "Config file [$file] does not exist. Failed to create it witn value [$name]." "ERROR" + fi + fi +} +#### SetConfFileValue SUBSET END #### + +# Function can replace [ -f /some/file* ] tests +# Modified version of http://stackoverflow.com/a/6364244/2635443 +function WildcardFileExists () { + local file="${1}" + local exists=0 + + for f in $file; do + ## Check if the glob gets expanded to existing files. + ## If not, f here will be exactly the pattern above + ## and the exists test will evaluate to false. + if [ -e "$f" ]; then + exists=1 + break + fi + done + + if [ $exists -eq 1 ]; then + echo 1 + else + echo 0 + fi +} + +# Some MacOS versions might loose file ownsership when using mv from /tmp dir (see #175) +# This is a "mv" function wrapper that helps out with macOS +function FileMove () { + local source="${1}" + local dest="${2}" + + # If file is symlink or OS is not Mac, just make a standard mv + if [ -L "$source" ] || [ "$LOCAL_OS" != "MacOSX" ]; then + mv -f "$source" "$dest" + return $? + elif [ -w "$source" ]; then + [ -f "$dest" ] && rm -f "$dest" + cp -p "$source" "$dest" && rm -f "$source" + return $? + else + return -1 + fi +} + +#### OFUNCTIONS FULL SUBSET END #### function SetLocalOSSettings { USER=root @@ -967,7 +2871,7 @@ SERVICE_FILE_OPENRC="$SERVICE_NAME-openrc" ## Generic code -trap TrapQuit TERM EXIT HUP QUIT +trap GenericTrapQuit TERM EXIT HUP QUIT if [ ! -w "$(dirname $LOG_FILE)" ]; then echo "Cannot write to log [$(dirname $LOG_FILE)]." diff --git a/osync-batch.sh b/osync-batch.sh index 43c48b1..622f388 100755 --- a/osync-batch.sh +++ b/osync-batch.sh @@ -3,7 +3,7 @@ SUBPROGRAM=osync PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones AUTHOR="(L) 2013-2020 by Orsiris de Jong" CONTACT="http://www.netpower.fr - ozy@netpower.fr" -PROGRAM_BUILD=2020031501 +PROGRAM_BUILD=2020031502 ## Runs an osync /obackup instance for every conf file found ## If an instance fails, run it again if time permits @@ -237,6 +237,1909 @@ function GenericTrapQuit { exit $exitcode } +#### TrapQuit SUBSET END #### + +#### CleanUp SUBSET #### +function CleanUp { + # Exit controlmaster before it's socket gets deleted + if [ "$SSH_CONTROLMASTER" == true ] && [ "$SSH_CMD" != "" ]; then + $SSH_CMD -O exit + fi + + if [ "$_DEBUG" != true ]; then + # Removing optional remote $RUN_DIR that goes into local $RUN_DIR + if [ -d "$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" ]; then + rm -rf "$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" + fi + # Removing all temporary run files + rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP" + # Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements) + rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp" + fi +} + +#### CleanUp SUBSET END #### + +#### OFUNCTIONS MICRO SUBSET END #### + +# osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending +function SendAlert { + local runAlert="${1:-false}" # Specifies if current message is sent while running or at the end of a run + local attachment="${2:-true}" # Should we send the log file as attachment + + + local attachmentFile + local subject + local body + + if [ "$DESTINATION_MAILS" == "" ]; then + return 0 + fi + + if [ "$_DEBUG" == true ]; then + Logger "Debug mode, no warning mail will be sent." "NOTICE" + return 0 + fi + + if [ $attachment == true ]; then + attachmentFile="$LOG_FILE" + if type "$COMPRESSION_PROGRAM" > /dev/null 2>&1; then + eval "cat \"$LOG_FILE\" \"$COMPRESSION_PROGRAM\" > \"$ALERT_LOG_FILE\"" + if [ $? -eq 0 ]; then + attachmentFile="$ALERT_LOG_FILE" + fi + fi + fi + + body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of current log"$'\n\n'"$(tail -n 1000 "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP")" + + if [ $ERROR_ALERT == true ]; then + subject="Error alert for $INSTANCE_ID" + elif [ $WARN_ALERT == true ]; then + subject="Warning alert for $INSTANCE_ID" + else + subject="Alert for $INSTANCE_ID" + fi + + if [ $runAlert == true ]; then + subject="Currently runing - $subject" + else + subject="Finished run - $subject" + fi + + SendEmail "$subject" "$body" "$DESTINATION_MAILS" "$attachmentFile" "$SENDER_MAIL" "$SMTP_SERVER" "$SMTP_PORT" "$SMTP_ENCRYPTION" "$SMTP_USER" "$SMTP_PASSWORD" + + # Delete tmp log file + if [ "$attachment" == true ]; then + if [ -f "$ALERT_LOG_FILE" ]; then + rm -f "$ALERT_LOG_FILE" + fi + fi +} + +# Generic email sending function. +# Usage (linux / BSD), attachment is optional, can be "/path/to/my.file" or "" +# SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file" +# Usage (Windows, make sure you have mailsend.exe in executable path, see http://github.com/muquit/mailsend) +# attachment is optional but must be in windows format like "c:\\some\path\\my.file", or "" +# smtp_server.domain.tld is mandatory, as is smtpPort (should be 25, 465 or 587) +# encryption can be set to tls, ssl or none +# smtpUser and smtpPassword are optional +# SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file" "senderMail@example.com" "smtpServer.domain.tld" "smtpPort" "encryption" "smtpUser" "smtpPassword" + +# If text is received as attachment ATT00001.bin or noname, consider adding the following to /etc/mail.rc +#set ttycharset=iso-8859-1 +#set sendcharsets=iso-8859-1 +#set encoding=8bit + +function SendEmail { + local subject="${1}" + local message="${2}" + local destinationMails="${3}" + local attachment="${4}" + local senderMail="${5}" + local smtpServer="${6}" + local smtpPort="${7}" + local encryption="${8}" + local smtpUser="${9}" + local smtpPassword="${10}" + + + local mail_no_attachment= + local attachment_command= + + local encryption_string= + local auth_string= + + local i + + if [ "${destinationMails}" != "" ]; then + for i in "${destinationMails[@]}"; do + if [ $(CheckRFC822 "$i") -ne 1 ]; then + Logger "Given email [$i] does not seem to be valid." "WARN" + fi + done + else + Logger "No valid email addresses given." "WARN" + return 1 + fi + + # Prior to sending an email, convert its body if needed + if [ "$MAIL_BODY_CHARSET" != "" ]; then + if type iconv > /dev/null 2>&1; then + echo "$message" | iconv -f UTF-8 -t $MAIL_BODY_CHARSET -o "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.iconv.$SCRIPT_PID.$TSTAMP" + message="$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.iconv.$SCRIPT_PID.$TSTAMP")" + else + Logger "iconv utility not installed. Will not convert email charset." "NOTICE" + fi + fi + + if [ ! -f "$attachment" ]; then + attachment_command="-a $attachment" + mail_no_attachment=1 + else + mail_no_attachment=0 + fi + + if [ "$LOCAL_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ]; then + if [ "$smtpPort" == "" ]; then + Logger "Missing smtp port, assuming 25." "WARN" + smtpPort=25 + fi + if type sendmail > /dev/null 2>&1; then + if [ "$encryption" == "tls" ]; then + echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$senderMail" -H "exec openssl s_client -quiet -tls1_2 -starttls smtp -connect $smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails" + elif [ "$encryption" == "ssl" ]; then + echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$senderMail" -H "exec openssl s_client -quiet -connect $smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails" + elif [ "$encryption" == "none" ]; then + echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$senderMail" -S "$smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails" + else + echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$senderMail" -S "$smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails" + Logger "Bogus email encryption used [$encryption]." "WARN" + fi + + if [ $? -ne 0 ]; then + Logger "Cannot send alert mail via $(type -p sendmail) !!!" "WARN" + # Do not bother try other mail systems with busybox + return 1 + else + return 0 + fi + else + Logger "Sendmail not present. Will not send any mail" "WARN" + return 1 + fi + fi + + if type mutt > /dev/null 2>&1 ; then + # We need to replace spaces with comma in order for mutt to be able to process multiple destinations + echo "$message" | $(type -p mutt) -x -s "$subject" "${destinationMails// /,}" $attachment_command + if [ $? -ne 0 ]; then + Logger "Cannot send mail via $(type -p mutt) !!!" "WARN" + else + Logger "Sent mail using mutt." "NOTICE" + return 0 + fi + fi + + if type mail > /dev/null 2>&1 ; then + # We need to detect which version of mail is installed + if ! $(type -p mail) -V > /dev/null 2>&1; then + # This may be MacOS mail program + attachment_command="" + elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then + attachment_command="-A $attachment" + elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V > /dev/null; then + attachment_command="-a$attachment" + else + attachment_command="" + fi + + echo "$message" | $(type -p mail) $attachment_command -s "$subject" "$destinationMails" + if [ $? -ne 0 ]; then + Logger "Cannot send mail via $(type -p mail) with attachments !!!" "WARN" + echo "$message" | $(type -p mail) -s "$subject" "$destinationMails" + if [ $? -ne 0 ]; then + Logger "Cannot send mail via $(type -p mail) without attachments !!!" "WARN" + else + Logger "Sent mail using mail command without attachment." "NOTICE" + return 0 + fi + else + Logger "Sent mail using mail command." "NOTICE" + return 0 + fi + fi + + if type sendmail > /dev/null 2>&1 ; then + echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) "$destinationMails" + if [ $? -ne 0 ]; then + Logger "Cannot send mail via $(type -p sendmail) !!!" "WARN" + else + Logger "Sent mail using sendmail command without attachment." "NOTICE" + return 0 + fi + fi + + # Windows specific + if type "mailsend.exe" > /dev/null 2>&1 ; then + if [ "$senderMail" == "" ]; then + Logger "Missing sender email." "ERROR" + return 1 + fi + if [ "$smtpServer" == "" ]; then + Logger "Missing smtp port." "ERROR" + return 1 + fi + if [ "$smtpPort" == "" ]; then + Logger "Missing smtp port, assuming 25." "WARN" + smtpPort=25 + fi + if [ "$encryption" != "tls" ] && [ "$encryption" != "ssl" ] && [ "$encryption" != "none" ]; then + Logger "Bogus smtp encryption, assuming none." "WARN" + encryption_string= + elif [ "$encryption" == "tls" ]; then + encryption_string=-starttls + elif [ "$encryption" == "ssl" ]:; then + encryption_string=-ssl + fi + if [ "$smtpUser" != "" ] && [ "$smtpPassword" != "" ]; then + auth_string="-auth -user \"$smtpUser\" -pass \"$smtpPassword\"" + fi + $(type mailsend.exe) -f "$senderMail" -t "$destinationMails" -sub "$subject" -M "$message" -attach "$attachment" -smtp "$smtpServer" -port "$smtpPort" $encryption_string $auth_string + if [ $? -ne 0 ]; then + Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN" + else + Logger "Sent mail using mailsend.exe command with attachment." "NOTICE" + return 0 + fi + fi + + # pfSense specific + if [ -f /usr/local/bin/mail.php ]; then + echo "$message" | /usr/local/bin/mail.php -s="$subject" + if [ $? -ne 0 ]; then + Logger "Cannot send mail via /usr/local/bin/mail.php (pfsense) !!!" "WARN" + else + Logger "Sent mail using pfSense mail.php." "NOTICE" + return 0 + fi + fi + + # If function has not returned 0 yet, assume it is critical that no alert can be sent + Logger "Cannot send mail (neither mutt, mail, sendmail, sendemail, mailsend (windows) or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue +} + +#### TrapError SUBSET #### +function TrapError { + local job="$0" + local line="$1" + local code="${2:-1}" + + if [ $_LOGGER_SILENT == false ]; then + (>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m") + fi +} +#### TrapError SUBSET END #### + +function LoadConfigFile { + local configFile="${1}" + local revisionRequired="${2}" + + + local revisionPresent + + if [ ! -f "$configFile" ]; then + Logger "Cannot load configuration file [$configFile]. Cannot start." "CRITICAL" + exit 1 + elif [[ "$configFile" != *".conf" ]]; then + Logger "Wrong configuration file supplied [$configFile]. Cannot start." "CRITICAL" + exit 1 + else + revisionPresent="$(GetConfFileValue "$configFile" "CONFIG_FILE_REVISION" true)" + if [ "$(IsNumeric "${revisionPresent%%.*}")" -eq 0 ]; then + Logger "Missing CONFIG_FILE_REVISION. Please provide a valid config file, or run the config update script." "WARN" + Logger "CONFIG_FILE_REVISION does not seem numeric [$revisionPresent]." "DEBUG" + elif [ "$revisionRequired" != "" ]; then + if [ $(VerComp "$revisionPresent" "$revisionRequired") -eq 2 ]; then + Logger "Configuration file seems out of date. Required version [$revisionRequired]. Actual version [$revisionPresent]." "CRITICAL" + exit 1 + fi + fi + # Remove everything that is not a variable assignation + grep '^[^ ]*=[^;&]*' "$configFile" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + source "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi + + CONFIG_FILE="$configFile" +} + +# Quick and dirty performance logger only used for debugging + +_OFUNCTIONS_SPINNER="|/-\\" +function Spinner { + if [ $_LOGGER_SILENT == true ] || [ "$_LOGGER_ERR_ONLY" == true ] || [ "$_SYNC_ON_CHANGES" == "initiator" ] || [ "$_SYNC_ON_CHANGES" == "target" ] ; then + return 0 + else + printf " [%c] \b\b\b\b\b\b" "$_OFUNCTIONS_SPINNER" + _OFUNCTIONS_SPINNER=${_OFUNCTIONS_SPINNER#?}${_OFUNCTIONS_SPINNER%%???} + return 0 + fi +} + +# WaitForTaskCompletion function emulation, now uses ExecTasks +function WaitForTaskCompletion { + local pids="${1}" + local softMaxTime="${2:-0}" + local hardMaxTime="${3:-0}" + local sleepTime="${4:-.05}" + local keepLogging="${5:-0}" + local counting="${6:-true}" + local spinner="${7:-true}" + local noErrorLog="${8:-false}" + local id="${9-base}" + + ExecTasks "$pids" "$id" false 0 0 "$softMaxTime" "$hardMaxTime" "$counting" "$sleepTime" "$keepLogging" "$spinner" "$noErrorlog" +} + +# ParallelExec function emulation, now uses ExecTasks +function ParallelExec { + local numberOfProcesses="${1}" + local commandsArg="${2}" + local readFromFile="${3:-false}" + local softMaxTime="${4:-0}" + local hardMaxTime="${5:-0}" + local sleepTime="${6:-.05}" + local keepLogging="${7:-0}" + local counting="${8:-true}" + local spinner="${9:-false}" + local noErrorLog="${10:-false}" + + if [ $readFromFile == true ]; then + ExecTasks "$commandsArg" "base" $readFromFile 0 0 "$softMaxTime" "$hardMaxTime" "$counting" "$sleepTime" "$keepLogging" "$spinner" "$noErrorLog" false "$numberOfProcesses" + else + ExecTasks "$commandsArg" "base" $readFromFile 0 0 "$softMaxTime" "$hardMaxTime" "$counting" "$sleepTime" "$keepLogging" "$spinner" "$noErrorLog" false "$numberOfProcesses" + fi +} + +## Main asynchronous execution function +## Function can work in: +## WaitForTaskCompletion mode: monitors given pid in background, and stops them if max execution time is reached. Suitable for multiple synchronous pids to monitor and wait for +## ParallExec mode: takes list of commands to execute in parallel per batch, and stops them if max execution time is reahed. + +## Example of improved wait $! +## ExecTasks $! "some_identifier" false 0 0 0 0 true 1 1800 false +## Example: monitor two sleep processes, warn if execution time is higher than 10 seconds, stop after 20 seconds +## sleep 15 & +## pid=$! +## sleep 20 & +## pid2=$! +## ExecTasks "some_identifier" 0 0 10 20 1 1800 true true false false 1 "$pid;$pid2" + +## Example of parallel execution of four commands, only if directories exist. Warn if execution takes more than 300 seconds. Stop if takes longer than 900 seconds. Exeute max 3 commands in parallel. +## commands="du -csh /var;du -csh /etc;du -csh /home;du -csh /usr" +## conditions="[ -d /var ];[ -d /etc ];[ -d /home];[ -d /usr]" +## ExecTasks "$commands" "some_identifier" false 0 0 300 900 true 1 1800 true false false 3 "$conditions" + +## Bear in mind that given commands and conditions need to be quoted + +## ExecTasks has the following ofunctions subfunction requirements: +## Spinner +## Logger +## JoinString +## KillChilds + +## Full call +##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes + +function ExecTasks { + # Mandatory arguments + local mainInput="${1}" # Contains list of pids / commands separated by semicolons or filepath to list of pids / commands + + # Optional arguments + local id="${2:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id + local readFromFile="${3:-false}" # Is mainInput / auxInput a semicolon separated list (true) or a filepath (false) + local softPerProcessTime="${4:-0}" # Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0 + local hardPerProcessTime="${5:-0}" # Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0 + local softMaxTime="${6:-0}" # Max time (in seconds) for the whole function to run before a warning is logged, unless set to 0 + local hardMaxTime="${7:-0}" # Max time (in seconds) for the whole function to run before all pids / commands given are stopped, unless set to 0 + local counting="${8:-true}" # Should softMaxTime and hardMaxTime be accounted since function begin (true) or since script begin (false) + local sleepTime="${9:-.5}" # Seconds between each state check. The shorter the value, the snappier ExecTasks will be, but as a tradeoff, more cpu power will be used (good values are between .05 and 1) + local keepLogging="${10:-1800}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging + local spinner="${11:-true}" # Show spinner (true) or do not show anything (false) while running + local noTimeErrorLog="${12:-false}" # Log errors when reaching soft / hard execution times (false) or do not log errors on those triggers (true) + local noErrorLogsAtAll="${13:-false}" # Do not log any errros at all (useful for recursive ExecTasks checks) + + # Parallelism specific arguments + local numberOfProcesses="${14:-0}" # Number of simulanteous commands to run, given as mainInput. Set to 0 by default (WaitForTaskCompletion mode). Setting this value enables ParallelExec mode. + local auxInput="${15}" # Contains list of commands separated by semicolons or filepath fo list of commands. Exit code of those commands decide whether main commands will be executed or not + local maxPostponeRetries="${16:-3}" # If a conditional command fails, how many times shall we try to postpone the associated main command. Set this to 0 to disable postponing + local minTimeBetweenRetries="${17:-300}" # Time (in seconds) between postponed command retries + local validExitCodes="${18:-0}" # Semi colon separated list of valid main command exit codes which will not trigger errors + + + local i + + + # Since ExecTasks takes up to 17 arguments, do a quick preflight check in DEBUG mode + if [ "$_DEBUG" == true ]; then + declare -a booleans=(readFromFile counting spinner noTimeErrorLog noErrorLogsAtAll) + for i in "${booleans[@]}"; do + test="if [ \$$i != false ] && [ \$$i != true ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi" + eval "$test" + done + declare -a integers=(softPerProcessTime hardPerProcessTime softMaxTime hardMaxTime keepLogging numberOfProcesses maxPostponeRetries minTimeBetweenRetries) + for i in "${integers[@]}"; do + test="if [ $(IsNumericExpand \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi" + eval "$test" + done + fi + + # Expand validExitCodes into array + IFS=';' read -r -a validExitCodes <<< "$validExitCodes" + + # ParallelExec specific variables + local auxItemCount=0 # Number of conditional commands + local commandsArray=() # Array containing commands + local commandsConditionArray=() # Array containing conditional commands + local currentCommand # Variable containing currently processed command + local currentCommandCondition # Variable containing currently processed conditional command + local commandsArrayPid=() # Array containing commands indexed by pids + local commandsArrayOutput=() # Array containing command results indexed by pids + local postponedRetryCount=0 # Number of current postponed commands retries + local postponedItemCount=0 # Number of commands that have been postponed (keep at least one in order to check once) + local postponedCounter=0 + local isPostponedCommand=false # Is the current command from a postponed file ? + local postponedExecTime=0 # How much time has passed since last postponed condition was checked + local needsPostponing # Does currentCommand need to be postponed + local temp + + # Common variables + local pid # Current pid working on + local pidState # State of the process + local mainItemCount=0 # number of given items (pids or commands) + local readFromFile # Should we read pids / commands from a file (true) + local counter=0 + local log_ttime=0 # local time instance for comparaison + + local seconds_begin=$SECONDS # Seconds since the beginning of the script + local exec_time=0 # Seconds since the beginning of this function + + local retval=0 # return value of monitored pid process + local subRetval=0 # return value of condition commands + local errorcount=0 # Number of pids that finished with errors + local pidsArray # Array of currently running pids + local newPidsArray # New array of currently running pids for next iteration + local pidsTimeArray # Array containing execution begin time of pids + local executeCommand # Boolean to check if currentCommand can be executed given a condition + local functionMode + local softAlert=false # Does a soft alert need to be triggered, if yes, send an alert once + local failedPidsList # List containing failed pids with exit code separated by semicolons (eg : 2355:1;4534:2;2354:3) + local randomOutputName # Random filename for command outputs + local currentRunningPids # String of pids running, used for debugging purposes only + + # Initialise global variable + eval "WAIT_FOR_TASK_COMPLETION_$id=\"\"" + eval "HARD_MAX_EXEC_TIME_REACHED_$id=false" + + # Init function variables depending on mode + + if [ $numberOfProcesses -gt 0 ]; then + functionMode=ParallelExec + else + functionMode=WaitForTaskCompletion + fi + + if [ $readFromFile == false ]; then + if [ $functionMode == "WaitForTaskCompletion" ]; then + IFS=';' read -r -a pidsArray <<< "$mainInput" + mainItemCount="${#pidsArray[@]}" + else + IFS=';' read -r -a commandsArray <<< "$mainInput" + mainItemCount="${#commandsArray[@]}" + IFS=';' read -r -a commandsConditionArray <<< "$auxInput" + auxItemCount="${#commandsConditionArray[@]}" + fi + else + if [ -f "$mainInput" ]; then + mainItemCount=$(wc -l < "$mainInput") + readFromFile=true + else + Logger "Cannot read main file [$mainInput]." "WARN" + fi + if [ "$auxInput" != "" ]; then + if [ -f "$auxInput" ]; then + auxItemCount=$(wc -l < "$auxInput") + else + Logger "Cannot read aux file [$auxInput]." "WARN" + fi + fi + fi + + if [ $functionMode == "WaitForTaskCompletion" ]; then + # Force first while loop condition to be true because we don't deal with counters but pids in WaitForTaskCompletion mode + counter=$mainItemCount + fi + + + # soft / hard execution time checks that needs to be a subfunction since it is called both from main loop and from parallelExec sub loop + function _ExecTasksTimeCheck { + if [ $spinner == true ]; then + Spinner + fi + if [ $counting == true ]; then + exec_time=$((SECONDS - seconds_begin)) + else + exec_time=$SECONDS + fi + + if [ $keepLogging -ne 0 ]; then + # This log solely exists for readability purposes before having next set of logs + if [ ${#pidsArray[@]} -eq $numberOfProcesses ] && [ $log_ttime -eq 0 ]; then + log_ttime=$exec_time + Logger "There are $((mainItemCount-counter+postponedItemCount)) / $mainItemCount tasks in the queue of which $postponedItemCount are postponed. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE" + fi + if [ $(((exec_time + 1) % keepLogging)) -eq 0 ]; then + if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1 second + log_ttime=$exec_time + if [ $functionMode == "WaitForTaskCompletion" ]; then + Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE" + elif [ $functionMode == "ParallelExec" ]; then + Logger "There are $((mainItemCount-counter+postponedItemCount)) / $mainItemCount tasks in the queue of which $postponedItemCount are postponed. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE" + fi + fi + fi + fi + + if [ $exec_time -gt $softMaxTime ]; then + if [ "$softAlert" != true ] && [ $softMaxTime -ne 0 ] && [ $noTimeErrorLog != true ]; then + Logger "Max soft execution time [$softMaxTime] exceeded for task [$id] with pids [$(joinString , ${pidsArray[@]})]." "WARN" + softAlert=true + SendAlert true + fi + fi + + if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then + if [ $noTimeErrorLog != true ]; then + Logger "Max hard execution time [$hardMaxTime] exceeded for task [$id] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR" + fi + for pid in "${pidsArray[@]}"; do + KillChilds $pid true + if [ $? -eq 0 ]; then + Logger "Task with pid [$pid] stopped successfully." "NOTICE" + else + if [ $noErrorLogsAtAll != true ]; then + Logger "Could not stop task with pid [$pid]." "ERROR" + fi + fi + errorcount=$((errorcount+1)) + done + if [ $noTimeErrorLog != true ]; then + SendAlert true + fi + eval "HARD_MAX_EXEC_TIME_REACHED_$id=true" + if [ $functionMode == "WaitForTaskCompletion" ]; then + return $errorcount + else + return 129 + fi + fi + } + + function _ExecTasksPidsCheck { + newPidsArray=() + + if [ "$currentRunningPids" != "$(joinString " " ${pidsArray[@]})" ]; then + Logger "ExecTask running for pids [$(joinString " " ${pidsArray[@]})]." "DEBUG" + currentRunningPids="$(joinString " " ${pidsArray[@]})" + fi + + for pid in "${pidsArray[@]}"; do + if [ $(IsInteger $pid) -eq 1 ]; then + if kill -0 $pid > /dev/null 2>&1; then + # Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :) + pidState="$(eval $PROCESS_STATE_CMD)" + if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then + + # Check if pid hasn't run more than soft/hard perProcessTime + pidsTimeArray[$pid]=$((SECONDS - seconds_begin)) + if [ ${pidsTimeArray[$pid]} -gt $softPerProcessTime ]; then + if [ "$softAlert" != true ] && [ $softPerProcessTime -ne 0 ] && [ $noTimeErrorLog != true ]; then + Logger "Max soft execution time [$softPerProcessTime] exceeded for pid [$pid]." "WARN" + if [ "${commandsArrayPid[$pid]}]" != "" ]; then + Logger "Command was [${commandsArrayPid[$pid]}]]." "WARN" + fi + softAlert=true + SendAlert true + fi + fi + + + if [ ${pidsTimeArray[$pid]} -gt $hardPerProcessTime ] && [ $hardPerProcessTime -ne 0 ]; then + if [ $noTimeErrorLog != true ] && [ $noErrorLogsAtAll != true ]; then + Logger "Max hard execution time [$hardPerProcessTime] exceeded for pid [$pid]. Stopping command execution." "ERROR" + if [ "${commandsArrayPid[$pid]}]" != "" ]; then + Logger "Command was [${commandsArrayPid[$pid]}]]." "WARN" + fi + fi + KillChilds $pid true + if [ $? -eq 0 ]; then + Logger "Command with pid [$pid] stopped successfully." "NOTICE" + else + if [ $noErrorLogsAtAll != true ]; then + Logger "Could not stop command with pid [$pid]." "ERROR" + fi + fi + errorcount=$((errorcount+1)) + + if [ $noTimeErrorLog != true ]; then + SendAlert true + fi + fi + + newPidsArray+=($pid) + fi + else + # pid is dead, get its exit code from wait command + wait $pid + retval=$? + # Check for valid exit codes + if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then + if [ $noErrorLogsAtAll != true ]; then + Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "ERROR" + if [ "$functionMode" == "ParallelExec" ]; then + Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR" + fi + if [ -f "${commandsArrayOutput[$pid]}" ]; then + Logger "Truncated output:\n$(head -c16384 "${commandsArrayOutput[$pid]}")" "ERROR" + fi + fi + errorcount=$((errorcount+1)) + # Welcome to variable variable bash hell + if [ "$failedPidsList" == "" ]; then + failedPidsList="$pid:$retval" + else + failedPidsList="$failedPidsList;$pid:$retval" + fi + else + Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG" + fi + fi + fi + done + + # hasPids can be false on last iteration in ParallelExec mode + pidsArray=("${newPidsArray[@]}") + + # Trivial wait time for bash to not eat up all CPU + sleep $sleepTime + + + } + + while [ ${#pidsArray[@]} -gt 0 ] || [ $counter -lt $mainItemCount ] || [ $postponedItemCount -ne 0 ]; do + _ExecTasksTimeCheck + retval=$? + if [ $retval -ne 0 ]; then + return $retval; + fi + + # The following execution bloc is only needed in ParallelExec mode since WaitForTaskCompletion does not execute commands, but only monitors them + if [ $functionMode == "ParallelExec" ]; then + while [ ${#pidsArray[@]} -lt $numberOfProcesses ] && ([ $counter -lt $mainItemCount ] || [ $postponedItemCount -ne 0 ]); do + _ExecTasksTimeCheck + retval=$? + if [ $retval -ne 0 ]; then + return $retval; + fi + + executeCommand=false + isPostponedCommand=false + currentCommand="" + currentCommandCondition="" + needsPostponing=false + + if [ $readFromFile == true ]; then + # awk identifies first line as 1 instead of 0 so we need to increase counter + currentCommand=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$mainInput") + if [ $auxItemCount -ne 0 ]; then + currentCommandCondition=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$auxInput") + fi + + # Check if we need to fetch postponed commands + if [ "$currentCommand" == "" ]; then + currentCommand=$(awk 'NR == num_line {print; exit}' num_line=$((postponedCounter+1)) "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedMain.$id.$SCRIPT_PID.$TSTAMP") + currentCommandCondition=$(awk 'NR == num_line {print; exit}' num_line=$((postponedCounter+1)) "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedAux.$id.$SCRIPT_PID.$TSTAMP") + isPostponedCommand=true + fi + else + currentCommand="${commandsArray[$counter]}" + if [ $auxItemCount -ne 0 ]; then + currentCommandCondition="${commandsConditionArray[$counter]}" + fi + + if [ "$currentCommand" == "" ]; then + currentCommand="${postponedCommandsArray[$postponedCounter]}" + currentCommandCondition="${postponedCommandsConditionArray[$postponedCounter]}" + isPostponedCommand=true + fi + fi + + # Check if we execute postponed commands, or if we delay them + if [ $isPostponedCommand == true ]; then + # Get first value before '@' + postponedExecTime="${currentCommand%%@*}" + postponedExecTime=$((SECONDS-postponedExecTime)) + # Get everything after first '@' + temp="${currentCommand#*@}" + # Get first value before '@' + postponedRetryCount="${temp%%@*}" + # Replace currentCommand with actual filtered currentCommand + currentCommand="${temp#*@}" + + # Since we read a postponed command, we may decrase postponedItemCounter + postponedItemCount=$((postponedItemCount-1)) + #Since we read one line, we need to increase the counter + postponedCounter=$((postponedCounter+1)) + + else + postponedRetryCount=0 + postponedExecTime=0 + fi + if ([ $postponedRetryCount -lt $maxPostponeRetries ] && [ $postponedExecTime -ge $minTimeBetweenRetries ]) || [ $isPostponedCommand == false ]; then + if [ "$currentCommandCondition" != "" ]; then + Logger "Checking condition [$currentCommandCondition] for command [$currentCommand]." "DEBUG" + eval "$currentCommandCondition" & + ExecTasks $! "subConditionCheck" false 0 0 1800 3600 true $SLEEP_TIME $KEEP_LOGGING true true true + subRetval=$? + if [ $subRetval -ne 0 ]; then + # is postponing enabled ? + if [ $maxPostponeRetries -gt 0 ]; then + Logger "Condition [$currentCommandCondition] not met for command [$currentCommand]. Exit code [$subRetval]. Postponing command." "NOTICE" + postponedRetryCount=$((postponedRetryCount+1)) + if [ $postponedRetryCount -ge $maxPostponeRetries ]; then + Logger "Max retries reached for postponed command [$currentCommand]. Skipping command." "NOTICE" + else + needsPostponing=true + fi + postponedExecTime=0 + else + Logger "Condition [$currentCommandCondition] not met for command [$currentCommand]. Exit code [$subRetval]. Ignoring command." "NOTICE" + fi + else + executeCommand=true + fi + else + executeCommand=true + fi + else + needsPostponing=true + fi + + if [ $needsPostponing == true ]; then + postponedItemCount=$((postponedItemCount+1)) + if [ $readFromFile == true ]; then + echo "$((SECONDS-postponedExecTime))@$postponedRetryCount@$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedMain.$id.$SCRIPT_PID.$TSTAMP" + echo "$currentCommandCondition" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedAux.$id.$SCRIPT_PID.$TSTAMP" + else + postponedCommandsArray+=("$((SECONDS-postponedExecTime))@$postponedRetryCount@$currentCommand") + postponedCommandsConditionArray+=("$currentCommandCondition") + fi + fi + + if [ $executeCommand == true ]; then + Logger "Running command [$currentCommand]." "DEBUG" + randomOutputName=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 5) + eval "$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$pid.$randomOutputName.$SCRIPT_PID.$TSTAMP" 2>&1 & + pid=$! + pidsArray+=($pid) + commandsArrayPid[$pid]="$currentCommand" + commandsArrayOutput[$pid]="$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$pid.$randomOutputName.$SCRIPT_PID.$TSTAMP" + # Initialize pid execution time array + pidsTimeArray[$pid]=0 + else + Logger "Skipping command [$currentCommand]." "DEBUG" + fi + + if [ $isPostponedCommand == false ]; then + counter=$((counter+1)) + fi + _ExecTasksPidsCheck + done + fi + + _ExecTasksPidsCheck + done + + + # Return exit code if only one process was monitored, else return number of errors + # As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value + + eval "WAIT_FOR_TASK_COMPLETION_$id=\"$failedPidsList\"" + + if [ $mainItemCount -eq 1 ]; then + return $retval + else + return $errorcount + fi +} + +# Usage: var=$(StripSingleQuotes "$var") +function StripSingleQuotes { + local string="${1}" + + string="${string/#\'/}" # Remove singlequote if it begins string + string="${string/%\'/}" # Remove singlequote if it ends string + echo "$string" +} + +# Usage: var=$(StripDoubleQuotes "$var") +function StripDoubleQuotes { + local string="${1}" + + string="${string/#\"/}" + string="${string/%\"/}" + echo "$string" +} + +function StripQuotes { + local string="${1}" + + echo "$(StripSingleQuotes $(StripDoubleQuotes $string))" +} + +# Usage var=$(EscapeSpaces "$var") or var="$(EscapeSpaces "$var")" +function EscapeSpaces { + local string="${1}" # String on which spaces will be escaped + + echo "${string// /\\ }" +} + +# Usage var=$(EscapeDoubleQuotes "$var") or var="$(EscapeDoubleQuotes "$var")" +function EscapeDoubleQuotes { + local value="${1}" + + echo "${value//\"/\\\"}" +} + +# Usage [ $(IsNumeric $var) -eq 1 ] +function IsNumeric { + local value="${1}" + + if type expr > /dev/null 2>&1; then + expr "$value" : '^[-+]\{0,1\}[0-9]*\.\{0,1\}[0-9]\{1,\}$' > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo 1 + else + echo 0 + fi + else + if [[ $value =~ ^[-+]?[0-9]+([.][0-9]+)?$ ]]; then + echo 1 + else + echo 0 + fi + fi +} + +function IsNumericExpand { + eval "local value=\"${1}\"" # Needed eval so variable variables can be processed + + echo $(IsNumeric "$value") +} + +#### HumanToNumeric SUBSET #### +# Converts human readable sizes into integer kilobyte sizes +# Usage numericSize="$(HumanToNumeric $humanSize)" +function HumanToNumeric { + local value="${1}" + + local notation + local suffix + local suffixPresent + local multiplier + + notation=(K M G T P E) + for suffix in "${notation[@]}"; do + multiplier=$((multiplier+1)) + if [[ "$value" == *"$suffix"* ]]; then + suffixPresent=$suffix + break; + fi + done + + if [ "$suffixPresent" != "" ]; then + value=${value%$suffix*} + value=${value%.*} + # /1024 since we convert to kilobytes instead of bytes + value=$((value*(1024**multiplier/1024))) + else + value=${value%.*} + fi + + echo $value +} +#### HumanToNumeric SUBSET END #### + +#### CheckRFC822 SUBSET #### +# Checks email address validity +function CheckRFC822 { + local mail="${1}" + local rfc822="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$" + + if [[ $mail =~ $rfc822 ]]; then + echo 1 + else + echo 0 + fi +} +#### CheckRFC822 SUBSET END #### + +#### UrlEncode SUBSET #### +## Modified version of https://gist.github.com/cdown/1163649 +function UrlEncode { + local length="${#1}" + + local i + + local LANG=C + for i in $(seq 0 $((length-1))); do + local c="${1:i:1}" + case $c in + [a-zA-Z0-9.~_-]) + printf "$c" + ;; + *) + printf '%%%02X' "'$c" + ;; + esac + done +} +#### UrlEncode SUBSET END #### + +function UrlDecode { + local urlEncoded="${1//+/ }" + + printf '%b' "${urlEncoded//%/\\x}" +} + +#### ArrayContains SUBSET #### +## Modified version of http://stackoverflow.com/a/8574392 +## Usage: [ $(ArrayContains "needle" "${haystack[@]}") -eq 1 ] +function ArrayContains () { + local needle="${1}" + local haystack="${2}" + local e + + if [ "$needle" != "" ] && [ "$haystack" != "" ]; then + for e in "${@:2}"; do + if [ "$e" == "$needle" ]; then + echo 1 + return + fi + done + fi + echo 0 + return +} +#### ArrayContains SUBSET END #### + +#### GetLocalOS SUBSET #### +function GetLocalOS { + local localOsVar + local localOsName + local localOsVer + + # There is no good way to tell if currently running in BusyBox shell. Using sluggish way. + if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then + localOsVar="BusyBox" + elif set -o | grep "winxp" > /dev/null; then + localOsVar="BusyBox-w32" + else + # Detecting the special ubuntu userland in Windows 10 bash + if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then + localOsVar="Microsoft" + else + localOsVar="$(uname -spior 2>&1)" + if [ $? -ne 0 ]; then + localOsVar="$(uname -v 2>&1)" + if [ $? -ne 0 ]; then + localOsVar="$(uname)" + fi + fi + fi + fi + + case $localOsVar in + # Android uname contains both linux and android, keep it before linux entry + *"Android"*) + LOCAL_OS="Android" + ;; + *"Linux"*) + LOCAL_OS="Linux" + ;; + *"BSD"*) + LOCAL_OS="BSD" + ;; + *"MINGW32"*|*"MINGW64"*|*"MSYS"*) + LOCAL_OS="msys" + ;; + *"CYGWIN"*) + LOCAL_OS="Cygwin" + ;; + *"Microsoft"*|*"MS/Windows"*) + LOCAL_OS="WinNT10" + ;; + *"Darwin"*) + LOCAL_OS="MacOSX" + ;; + *"BusyBox"*) + LOCAL_OS="BusyBox" + ;; + *) + if [ "$IGNORE_OS_TYPE" == true ]; then + Logger "Running on unknown local OS [$localOsVar]." "WARN" + return + fi + if [ "$_OFUNCTIONS_VERSION" != "" ]; then + Logger "Running on >> $localOsVar << not supported. Please report to the author." "ERROR" + fi + exit 1 + ;; + esac + + # Get linux versions + if [ -f "/etc/os-release" ]; then + localOsName="$(GetConfFileValue "/etc/os-release" "NAME" true)" + localOsVer="$(GetConfFileValue "/etc/os-release" "VERSION" true)" + elif [ "$LOCAL_OS" == "BusyBox" ]; then + localOsVer="$(ls --help 2>&1 | head -1 | cut -f2 -d' ')" + localOsName="BusyBox" + fi + + # Get Host info for Windows + if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "WinNT10" ]; then + localOsVar="$localOsVar $(uname -a)" + if [ "$PROGRAMW6432" != "" ]; then + LOCAL_OS_BITNESS=64 + LOCAL_OS_FAMILY="Windows" + elif [ "$PROGRAMFILES" != "" ]; then + LOCAL_OS_BITNESS=32 + LOCAL_OS_FAMILY="Windows" + # Case where running on BusyBox but no program files defined + elif [ "$LOCAL_OS" == "BusyBox" ]; then + LOCAL_OS_FAMILY="Unix" + fi + # Get Host info for Unix + else + LOCAL_OS_FAMILY="Unix" + fi + + if [ "$LOCAL_OS_FAMILY" == "Unix" ]; then + if uname -m | grep '64' > /dev/null 2>&1; then + LOCAL_OS_BITNESS=64 + else + LOCAL_OS_BITNESS=32 + fi + fi + + LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer) $LOCAL_OS_BITNESS-bit $LOCAL_OS_FAMILY" + + if [ "$_OFUNCTIONS_VERSION" != "" ]; then + Logger "Local OS: [$LOCAL_OS_FULL]." "DEBUG" + fi +} +#### GetLocalOS SUBSET END #### + + +#### OFUNCTIONS MINI SUBSET END #### + +function GetRemoteOS { + + if [ "$REMOTE_OPERATION" != true ]; then + return 0 + fi + + local remoteOsVar + +$SSH_CMD env LC_ALL=C env _REMOTE_TOKEN="$_REMOTE_TOKEN" bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 +_REMOTE_TOKEN="(o_0)" + +function GetOs { + local localOsVar + local localOsName + local localOsVer + local localOsBitness + local localOsFamily + + local osInfo="/etc/os-release" + + # There is no good way to tell if currently running in BusyBox shell. Using sluggish way. + if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then + localOsVar="BusyBox" + else + # Detecting the special ubuntu userland in Windows 10 bash + if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then + localOsVar="Microsoft" + else + localOsVar="$(uname -spior 2>&1)" + if [ $? -ne 0 ]; then + localOsVar="$(uname -v 2>&1)" + if [ $? -ne 0 ]; then + localOsVar="$(uname)" + fi + fi + fi + fi + # Get linux versions + if [ -f "$osInfo" ]; then + localOsName=$(grep "^NAME=" "$osInfo") + localOsName="${localOsName##*=}" + localOsVer=$(grep "^VERSION=" "$osInfo") + localOsVer="${localOsVer##*=}" + elif [ "$localOsVar" == "BusyBox" ]; then + localOsVer=$(ls --help 2>&1 | head -1 | cut -f2 -d' ') + localOsName="BusyBox" + fi + + # Get Host info for Windows + case $localOsVar in + *"MINGW32"*|*"MINGW64"*|*"MSYS"*|*"CYGWIN*"|*"Microsoft"*|*"WinNT10*") + if [ "$PROGRAMW6432" != "" ]; then + localOsBitness=64 + localOsFamily="Windows" + elif [ "$PROGRAMFILES" != "" ]; then + localOsBitness=32 + localOsFamily="Windows" + # Case where running on BusyBox but no program files defined + elif [ "$localOsVar" == "BusyBox" ]; then + localOsFamily="Unix" + fi + ;; + *) + localOsFamily="Unix" + if uname -m | grep '64' > /dev/null 2>&1; then + localOsBitness=64 + else + localOsBitness=32 + fi + ;; + esac + + echo "$localOsVar ($localOsName $localOsVer) $localOsBitness-bit $localOsFamily" +} + +GetOs + +ENDSSH + if [ $? -ne 0 ]; then + Logger "Cannot connect to remote system [$REMOTE_HOST] port [$REMOTE_PORT] as [$REMOTE_USER]." "CRITICAL" + if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then + Logger "$(head -c16384 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "ERROR" + fi + exit 1 + fi + + + if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then + remoteOsVar="$(head -c16384 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" + case $remoteOsVar in + *"Android"*) + REMOTE_OS="Android" + ;; + *"Linux"*) + REMOTE_OS="Linux" + ;; + *"BSD"*) + REMOTE_OS="BSD" + ;; + *"MINGW32"*|*"MINGW64"*|*"MSYS"*) + REMOTE_OS="msys" + ;; + *"CYGWIN"*) + REMOTE_OS="Cygwin" + ;; + *"Microsoft"*) + REMOTE_OS="WinNT10" + ;; + *"Darwin"*) + REMOTE_OS="MacOSX" + ;; + *"BusyBox"*) + REMOTE_OS="BusyBox" + ;; + *"ssh"*|*"SSH"*) + Logger "Cannot connect to remote system." "CRITICAL" + exit 1 + ;; + *) + if [ "$IGNORE_OS_TYPE" == true ]; then #DOC: Undocumented debug only setting + Logger "Running on unknown remote OS [$remoteOsVar]." "WARN" + return + fi + Logger "Running on remote OS failed. Please report to the author if the OS is not supported." "CRITICAL" + Logger "Remote OS said:\n$remoteOsVar" "CRITICAL" + exit 1 + esac + Logger "Remote OS: [$remoteOsVar]." "DEBUG" + else + Logger "Cannot get Remote OS" "CRITICAL" + fi +} + +function RunLocalCommand { + local command="${1}" # Command to run + local hardMaxTime="${2}" # Max time to wait for command to compleet + + if [ $_DRYRUN == true ]; then + Logger "Dryrun: Local command [$command] not run." "NOTICE" + return 0 + fi + + Logger "Running command [$command] on local host." "NOTICE" + eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 & + + ExecTasks $! "${FUNCNAME[0]}" false 0 0 0 $hardMaxTime true $SLEEP_TIME $KEEP_LOGGING + retval=$? + if [ $retval -eq 0 ]; then + Logger "Command succeded." "NOTICE" + else + Logger "Command failed." "ERROR" + fi + + if [ $_LOGGER_VERBOSE == true ] || [ $retval -ne 0 ]; then + Logger "Truncated output:\n$(head -c16384 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "NOTICE" + fi + + if [ "$STOP_ON_CMD_ERROR" == true ] && [ $retval -ne 0 ]; then + Logger "Stopping on command execution error." "CRITICAL" + exit 1 + fi +} + +## Runs remote command $1 and waits for completition in $2 seconds +function RunRemoteCommand { + local command="${1}" # Command to run + local hardMaxTime="${2}" # Max time to wait for command to compleet + + + if [ "$REMOTE_OPERATION" != true ]; then + Logger "Ignoring remote command [$command] because remote host is not configured." "WARN" + return 0 + fi + + CheckConnectivity3rdPartyHosts + CheckConnectivityRemoteHost + if [ $_DRYRUN == true ]; then + Logger "Dryrun: Local command [$command] not run." "NOTICE" + return 0 + fi + + Logger "Running command [$command] on remote host." "NOTICE" + cmd=$SSH_CMD' "env LC_ALL=C env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1' + Logger "cmd: $cmd" "DEBUG" + eval "$cmd" & + ExecTasks $! "${FUNCNAME[0]}" false 0 0 0 $hardMaxTime true $SLEEP_TIME $KEEP_LOGGING + retval=$? + if [ $retval -eq 0 ]; then + Logger "Command succeded." "NOTICE" + else + Logger "Command failed." "ERROR" + fi + + if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ] && ([ $_LOGGER_VERBOSE == true ] || [ $retval -ne 0 ]) + then + Logger "Truncated output:\n$(head -c16384 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "NOTICE" + fi + + if [ "$STOP_ON_CMD_ERROR" == true ] && [ $retval -ne 0 ]; then + Logger "Stopping on command execution error." "CRITICAL" + exit 1 + fi +} + +function RunBeforeHook { + + local pids + + if [ "$LOCAL_RUN_BEFORE_CMD" != "" ]; then + RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE & + pids="$!" + fi + + if [ "$REMOTE_RUN_BEFORE_CMD" != "" ]; then + RunRemoteCommand "$REMOTE_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE & + pids="$pids;$!" + fi + if [ "$pids" != "" ]; then + ExecTasks $pids "${FUNCNAME[0]}" false 0 0 0 0 true $SLEEP_TIME $KEEP_LOGGING + fi +} + +function RunAfterHook { + + local pids + + if [ "$LOCAL_RUN_AFTER_CMD" != "" ]; then + RunLocalCommand "$LOCAL_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER & + pids="$!" + fi + + if [ "$REMOTE_RUN_AFTER_CMD" != "" ]; then + RunRemoteCommand "$REMOTE_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER & + pids="$pids;$!" + fi + if [ "$pids" != "" ]; then + ExecTasks $pids "${FUNCNAME[0]}" false 0 0 0 0 true $SLEEP_TIME $KEEP_LOGGING + fi +} + +function CheckConnectivityRemoteHost { + + local retval + + + if [ "$REMOTE_HOST_PING" != false ] && [ "$REMOTE_OPERATION" != false ]; then + eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" & + ExecTasks $! "${FUNCNAME[0]}" false 0 0 60 180 true $SLEEP_TIME $KEEP_LOGGING + retval=$? + if [ $retval -ne 0 ]; then + Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN" + return $retval + fi + fi +} + +function CheckConnectivity3rdPartyHosts { + + local remote3rdPartySuccess + local retval + local i + + + if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then + remote3rdPartySuccess=false + for i in $REMOTE_3RD_PARTY_HOSTS + do + eval "$PING_CMD $i > /dev/null 2>&1" & + ExecTasks $! "${FUNCNAME[0]}" false 0 0 60 180 true $SLEEP_TIME $KEEP_LOGGING + retval=$? + if [ $retval -ne 0 ]; then + Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE" + else + remote3rdPartySuccess=true + fi + done + + if [ $remote3rdPartySuccess == false ]; then + Logger "No remote 3rd party host responded to ping. No internet ?" "WARN" + return 1 + else + return 0 + fi + fi +} + +function RsyncPatternsAdd { + local patternType="${1}" # exclude or include + local pattern="${2}" + + local rest + + # Disable globbing so wildcards from exclusions do not get expanded + set -f + rest="$pattern" + while [ -n "$rest" ] + do + # Take the string until first occurence until $PATH_SEPARATOR_CHAR + str="${rest%%$PATH_SEPARATOR_CHAR*}" + # Handle the last case + if [ "$rest" == "${rest/$PATH_SEPARATOR_CHAR/}" ]; then + rest= + else + # Cut everything before the first occurence of $PATH_SEPARATOR_CHAR + rest="${rest#*$PATH_SEPARATOR_CHAR}" + fi + if [ "$RSYNC_PATTERNS" == "" ]; then + RSYNC_PATTERNS="--$patternType=\"$str\"" + else + RSYNC_PATTERNS="$RSYNC_PATTERNS --$patternType=\"$str\"" + fi + done + set +f +} + +function RsyncPatternsFromAdd { + local patternType="${1}" + local patternFrom="${2}" + + ## Check if the exclude list has a full path, and if not, add the config file path if there is one + if [ "$(basename "$patternFrom")" == "$patternFrom" ]; then + patternFrom="$(dirname "$CONFIG_FILE")/$patternFrom" + fi + + if [ -e "$patternFrom" ]; then + RSYNC_PATTERNS="$RSYNC_PATTERNS --$patternType-from=\"$patternFrom\"" + fi +} + +function RsyncPatterns { + + if [ "$RSYNC_PATTERN_FIRST" == "exclude" ]; then + if [ "$RSYNC_EXCLUDE_PATTERN" != "" ]; then + RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN" + fi + if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM" + fi + if [ "$RSYNC_INCLUDE_PATTERN" != "" ]; then + RsyncPatternsAdd "include" "$RSYNC_INCLUDE_PATTERN" + fi + if [ "$RSYNC_INCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM" + fi + # Use default include first for quicksync runs + elif [ "$RSYNC_PATTERN_FIRST" == "include" ] || [ "$_QUICK_SYNC" == "2" ]; then + if [ "$RSYNC_INCLUDE_PATTERN" != "" ]; then + RsyncPatternsAdd "include" "$RSYNC_INCLUDE_PATTERN" + fi + if [ "$RSYNC_INCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "include" "$RSYNC_INCLUDE_FROM" + fi + if [ "$RSYNC_EXCLUDE_PATTERN" != "" ]; then + RsyncPatternsAdd "exclude" "$RSYNC_EXCLUDE_PATTERN" + fi + if [ "$RSYNC_EXCLUDE_FROM" != "" ]; then + RsyncPatternsFromAdd "exclude" "$RSYNC_EXCLUDE_FROM" + fi + else + # osync target-helper specific clause + if [ "$_SYNC_ON_CHANGES" != "target" ]; then + Logger "Bogus RSYNC_PATTERN_FIRST value in config file. Will not use rsync patterns." "WARN" + fi + fi +} + +function PreInit { + + local compressionString + + ## SSH compression + if [ "$SSH_COMPRESSION" != false ]; then + SSH_COMP=-C + else + SSH_COMP= + fi + + ## Ignore SSH known host verification + if [ "$SSH_IGNORE_KNOWN_HOSTS" == true ]; then + SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" + fi + + ## SSH ControlMaster Multiplexing + if [ "$SSH_CONTROLMASTER" == true ]; then + SSH_OPTS="$SSH_OPTS -o ControlMaster=auto -o ControlPersist=yes -o ControlPath=\"$RUN_DIR/$PROGRAM.ctrlm.%r@%h.$SCRIPT_PID.$TSTAMP\"" + fi + + ## Optional SSH arguments + if [ "$SSH_OPTIONAL_ARGS" != "" ]; then + SSH_OPTS="$SSH_OPTS $SSH_OPTIONAL_ARGS" + fi + + ## Support for older config files without RSYNC_EXECUTABLE option + if [ "$RSYNC_EXECUTABLE" == "" ]; then + RSYNC_EXECUTABLE=rsync + fi + + ## Sudo execution option + if [ "$SUDO_EXEC" == true ]; then + if [ "$RSYNC_REMOTE_PATH" != "" ]; then + RSYNC_PATH="sudo $RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" + else + RSYNC_PATH="sudo $RSYNC_EXECUTABLE" + fi + COMMAND_SUDO="sudo -E" + else + if [ "$RSYNC_REMOTE_PATH" != "" ]; then + RSYNC_PATH="$RSYNC_REMOTE_PATH/$RSYNC_EXECUTABLE" + else + RSYNC_PATH="$RSYNC_EXECUTABLE" + fi + COMMAND_SUDO="" + fi + + ## Set compression executable and extension + if [ "$(IsInteger $COMPRESSION_LEVEL)" -eq 0 ]; then + COMPRESSION_LEVEL=3 + fi +} + +function PostInit { + + # Define remote commands + if [ -f "$SSH_RSA_PRIVATE_KEY" ]; then + SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" + SCP_CMD="$(type -p scp) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT" + RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT" + elif [ -f "$SSH_PASSWORD_FILE" ]; then + SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" + SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -q -P $REMOTE_PORT" + RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS -p $REMOTE_PORT" + else + SSH_PASSWORD="" + SSH_CMD="" + SCP_CMD="" + RSYNC_SSH_CMD="" + fi +} + +function SetCompression { + ## Busybox fix (Termux xz command does not support compression at all) + if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then + compressionString="" + if type gzip > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| gzip -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + else + COMPRESSION_PROGRAM= + COMPRESSION_EXTENSION= + fi + else + compressionString=" -$COMPRESSION_LEVEL" + + if type xz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| xz -c$compressionString" + COMPRESSION_EXTENSION=.xz + elif type lzma > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| lzma -c$compressionString" + COMPRESSION_EXTENSION=.lzma + elif type pigz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| pigz -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + COMPRESSION_OPTIONS=--rsyncable + elif type gzip > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| gzip -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + COMPRESSION_OPTIONS=--rsyncable + else + COMPRESSION_PROGRAM= + COMPRESSION_EXTENSION= + fi + fi + + if [ ".${ALERT_LOG_FILE##*.}" != "$COMPRESSION_EXTENSION" ]; then + ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" + fi +} + +function InitLocalOSDependingSettings { + + ## If running under Msys, some commands do not run the same way + ## Using mingw version of find instead of windows one + ## Getting running processes is quite different + ## Ping command is not the same + if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "Microsoft" ] || [ "$LOCAL_OS" == "WinNT10" ]; then + + # Newer bash on Win10 finally uses integrated find command instead of windows one + if [ -f "/usr/bin/find" ]; then + FIND_CMD="/usr/bin/find" + elif [ -f "/bin/find" ]; then + FIND_CMD="/bin/find" + else + FIND_CMD="$(dirname $BASH)/find" + fi + + # Newer bash on Windows 10 uses integrated ping whereas cygwin & msys use Windows version + if [ "$LOCAL_OS" == "WinNT10" ]; then + PING_CMD="ping -c 2 -i 1" + else + PING_CMD='$SYSTEMROOT\system32\ping -n 2' + fi + + # On BSD, when not root, min ping interval is 1s + elif [ "$LOCAL_OS" == "BSD" ] && [ "$LOCAL_USER" != "root" ]; then + FIND_CMD=find + PING_CMD="ping -c 2 -i 1" + else + FIND_CMD=find + PING_CMD="ping -c 2 -i .2" + fi + + if [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then + PROCESS_STATE_CMD="echo none" + DF_CMD="df" + else + PROCESS_STATE_CMD='ps -p$pid -o state= 2 > /dev/null' + # CentOS 5 needs -P for one line output + DF_CMD="df -P" + fi + + ## Stat command has different syntax on Linux and FreeBSD/MacOSX + if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then + # Tested on BSD and Mac + STAT_CMD="stat -f \"%Sm\"" + STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m" + SED_REGEX_ARG="-E" + else + # Tested on GNU stat, busybox and Cygwin + STAT_CMD="stat -c %y" + STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y" + SED_REGEX_ARG="-r" + fi + + # Set compression first time when we know what local os we have + SetCompression +} + +# Gets executed regardless of the need of remote connections. It is just that this code needs to get executed after we know if there is a remote os, and if yes, which one +function InitRemoteOSDependingSettings { + + if [ "$REMOTE_OS" == "msys" ] || [ "$REMOTE_OS" == "Cygwin" ]; then + REMOTE_FIND_CMD="$(dirname $BASH)/find" + else + REMOTE_FIND_CMD=find + fi + + ## Stat command has different syntax on Linux and FreeBSD/MacOSX + if [ "$REMOTE_OS" == "MacOSX" ] || [ "$REMOTE_OS" == "BSD" ]; then + REMOTE_STAT_CMD="stat -f \"%Sm\"" + REMOTE_STAT_CTIME_MTIME_CMD="stat -f \\\"%N;%c;%m\\\"" + else + REMOTE_STAT_CMD="stat --format %y" + REMOTE_STAT_CTIME_MTIME_CMD="stat -c \\\"%n;%Z;%Y\\\"" + fi + + ## Set rsync default arguments (complete with -r or -d depending on recursivity later) + RSYNC_DEFAULT_ARGS="-ltD -8" + if [ "$_DRYRUN" == true ]; then + RSYNC_DRY_ARG="-n" + DRY_WARNING="/!\ DRY RUN " + else + RSYNC_DRY_ARG="" + fi + + RSYNC_ATTR_ARGS="" + if [ "$PRESERVE_PERMISSIONS" != false ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -p" + fi + if [ "$PRESERVE_OWNER" != false ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -o" + fi + if [ "$PRESERVE_GROUP" != false ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -g" + fi + if [ "$PRESERVE_EXECUTABILITY" != false ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" --executability" + fi + if [ "$PRESERVE_ACL" == true ]; then + if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ] && [ "$LOCAL_OS" != "msys" ] && [ "$REMOTE_OS" != "msys" ] && [ "$LOCAL_OS" != "Cygwin" ] && [ "$REMOTE_OS" != "Cygwin" ] && [ "$LOCAL_OS" != "BusyBox" ] && [ "$REMOTE_OS" != "BusyBox" ] && [ "$LOCAL_OS" != "Android" ] && [ "$REMOTE_OS" != "Android" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A" + else + Logger "Disabling ACL synchronization on [$LOCAL_OS] due to lack of support." "NOTICE" + + fi + fi + if [ "$PRESERVE_XATTR" == true ]; then + if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ] && [ "$LOCAL_OS" != "msys" ] && [ "$REMOTE_OS" != "msys" ] && [ "$LOCAL_OS" != "Cygwin" ] && [ "$REMOTE_OS" != "Cygwin" ] && [ "$LOCAL_OS" != "BusyBox" ] && [ "$REMOTE_OS" != "BusyBox" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X" + else + Logger "Disabling extended attributes synchronization on [$LOCAL_OS] due to lack of support." "NOTICE" + fi + fi + if [ "$RSYNC_COMPRESS" == true ]; then + if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -zz --skip-compress=3fr/3g2/3gp/3gpp/7z/aac/ace/amr/apk/appx/appxbundle/arc/arj/arw/asf/avi/bz/bz2/cab/cr2/crypt[5678]/dat/dcr/deb/dmg/drc/ear/erf/flac/flv/gif/gpg/gz/iiq/jar/jp2/jpeg/jpg/h26[45]/k25/kdc/kgb/lha/lz/lzma/lzo/lzx/m4[apv]/mef/mkv/mos/mov/mp[34]/mpeg/mp[gv]/msi/nef/oga/ogg/ogv/opus/orf/pak/pef/png/qt/rar/r[0-9][0-9]/rz/rpm/rw2/rzip/s7z/sfark/sfx/sr2/srf/svgz/t[gb]z/tlz/txz/vob/wim/wma/wmv/xz/zip" + else + Logger "Disabling compression skips on synchronization on [$LOCAL_OS] due to lack of support." "NOTICE" + fi + fi + if [ "$COPY_SYMLINKS" == true ]; then + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -L" + fi + if [ "$KEEP_DIRLINKS" == true ]; then + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -K" + fi + if [ "$RSYNC_OPTIONAL_ARGS" != "" ]; then + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" "$RSYNC_OPTIONAL_ARGS + fi + if [ "$PRESERVE_HARDLINKS" == true ]; then + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -H" + fi + if [ "$CHECKSUM" == true ]; then + RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum" + fi + if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" --bwlimit=$BANDWIDTH" + fi + + if [ "$PARTIAL" == true ]; then + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" + RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\"" + fi + + if [ "$DELTA_COPIES" != false ]; then + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" --no-whole-file" + else + RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" --whole-file" + fi + + # Set compression options again after we know what remote OS we are dealing with + SetCompression + + # Set recursive options + RSYNC_DEFAULT_NONRECURSIVE_ARGS="-d $RSYNC_DEFAULT_ARGS" + RSYNC_DEFAULT_ARGS="-r $RSYNC_DEFAULT_ARGS" +} + +## IFS debug function +function PrintIFS { + printf "IFS is: %q" "$IFS" +} + +# Process debugging +# Recursive function to get all parents from a pid +function ParentPid { + local pid="${1}" # Pid to analyse + local parent + + parent=$(ps -p $pid -o ppid=) + echo "$pid is a child of $parent" + if [ $parent -gt 0 ]; then + ParentPid $parent + fi +} + +#### VerComp SUBSET #### +# Neat version compare function found at http://stackoverflow.com/a/4025065/2635443 +# Returns 0 if equal, 1 if $1 > $2 and 2 if $1 < $2 +function VerComp () { + if [ "$1" == "" ] || [ "$2" == "" ]; then + Logger "Bogus Vercomp values [$1] and [$2]." "WARN" + return 1 + fi + + if [[ "$1" == "$2" ]] + then + echo 0 + return + fi + + local IFS=. + local i ver1=($1) ver2=($2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) + do + ver1[i]=0 + done + for ((i=0; i<${#ver1[@]}; i++)) + do + if [[ -z ${ver2[i]} ]] + then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})) + then + echo 1 + return + fi + if ((10#${ver1[i]} < 10#${ver2[i]})) + then + echo 2 + return + fi + done + + echo 0 + return +} +#### VerComp SUBSET END #### + +#### GetConfFileValue SUBSET #### +function GetConfFileValue () { + local file="${1}" + local name="${2}" + local noError="${3:-false}" + + local value + + value=$(grep "^$name=" "$file") + if [ $? -eq 0 ]; then + value="${value##*=}" + echo "$value" + else + if [ $noError == true ]; then + Logger "Cannot get value for [$name] in config file [$file]." "NOTICE" + else + Logger "Cannot get value for [$name] in config file [$file]." "ERROR" + fi + fi +} + +#### GetConfFileValue SUBSET END #### + +#### SetConfFileValue SUBSET #### +function SetConfFileValue () { + local file="${1}" + local name="${2}" + local value="${3}" + local separator="${4:-#}" + + if [ -f "$file" ]; then + if grep "^$name=" "$file" > /dev/null 2>&1; then + # Using -i.tmp for BSD compat + sed -i.tmp "s$separator^$name=.*$separator$name=$value$separator" "$file" + if [ $? -ne 0 ]; then + Logger "Cannot update value [$name] to [$value] in config file [$file]." "ERROR" + fi + rm -f "$file.tmp" + Logger "Set [$name] to [$value] in config file [$file]." "DEBUG" + else + echo "$name=$value" >> "$file" + if [ $? -ne 0 ]; then + Logger "Cannot create value [$name] to [$value] in config file [$file]." "ERROR" + fi + fi + else + echo "$name=$value" > "$file" + if [ $? -ne 0 ]; then + Logger "Config file [$file] does not exist. Failed to create it witn value [$name]." "ERROR" + fi + fi +} +#### SetConfFileValue SUBSET END #### + +# Function can replace [ -f /some/file* ] tests +# Modified version of http://stackoverflow.com/a/6364244/2635443 +function WildcardFileExists () { + local file="${1}" + local exists=0 + + for f in $file; do + ## Check if the glob gets expanded to existing files. + ## If not, f here will be exactly the pattern above + ## and the exists test will evaluate to false. + if [ -e "$f" ]; then + exists=1 + break + fi + done + + if [ $exists -eq 1 ]; then + echo 1 + else + echo 0 + fi +} + +# Some MacOS versions might loose file ownsership when using mv from /tmp dir (see #175) +# This is a "mv" function wrapper that helps out with macOS +function FileMove () { + local source="${1}" + local dest="${2}" + + # If file is symlink or OS is not Mac, just make a standard mv + if [ -L "$source" ] || [ "$LOCAL_OS" != "MacOSX" ]; then + mv -f "$source" "$dest" + return $? + elif [ -w "$source" ]; then + [ -f "$dest" ] && rm -f "$dest" + cp -p "$source" "$dest" && rm -f "$source" + return $? + else + return -1 + fi +} + +#### OFUNCTIONS FULL SUBSET END #### function CheckEnvironment { ## osync / obackup executable full path can be set here if it cannot be found on the system @@ -326,7 +2229,7 @@ function Usage { exit 128 } -trap TrapQuit TERM EXIT HUP QUIT +trap GenericTrapQuit TERM EXIT HUP QUIT opts="" for i in "$@" diff --git a/osync.sh b/osync.sh index 8c6668f..55983ea 100755 --- a/osync.sh +++ b/osync.sh @@ -14,7 +14,7 @@ CONFIG_FILE_REVISION_REQUIRED=1.3.0 _OFUNCTIONS_VERSION=2.3.0-RC4 -_OFUNCTIONS_BUILD=2020031501 +_OFUNCTIONS_BUILD=2020031502 _OFUNCTIONS_BOOTSTRAP=true if ! type "$BASH" > /dev/null; then @@ -391,6 +391,7 @@ function GenericTrapQuit { exit $exitcode } +#### TrapQuit SUBSET END #### function CleanUp { # Exit controlmaster before it's socket gets deleted