From 4e44020619cb5a438cd1c023b2e2c604261a8dcf Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Thu, 7 Jul 2022 10:58:16 +0200 Subject: [PATCH] Update ofunctions from pmocr --- dev/ofunctions.sh | 203 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 179 insertions(+), 24 deletions(-) diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index cba78ee..835b9b9 100644 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash -## Generic and highly portable bash functions written in 2013-2020 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr +## Generic and highly portable bash functions written in 2013-2022 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr #TODO: ExecTasks postponed arrays / files grow a lot. Consider having them "rolling" (cleaned at numberOfEvents) -#TODO: command line arguments don't take -AaqV for example +#TODO: command line arguments do not take merged args (-AaqV for example) #################################################################################################################################################################### ## To use in a program, define the following variables: @@ -30,8 +30,8 @@ #### OFUNCTIONS FULL SUBSET #### #### OFUNCTIONS MINI SUBSET #### #### OFUNCTIONS MICRO SUBSET #### -_OFUNCTIONS_VERSION=2.3.2 -_OFUNCTIONS_BUILD=2021051801 +_OFUNCTIONS_VERSION=2.4.3 +_OFUNCTIONS_BUILD=2022050801 #### _OFUNCTIONS_BOOTSTRAP SUBSET #### _OFUNCTIONS_BOOTSTRAP=true #### _OFUNCTIONS_BOOTSTRAP SUBSET END #### @@ -94,7 +94,7 @@ function PoorMansRandomGenerator { local digits="${1}" # The number of digits to generate local number - # Some read bytes can't be used, se we read twice the number of required bytes + # Some read bytes cannot be used, se we read twice the number of required bytes dd if=/dev/urandom bs=$digits count=2 2> /dev/null | while read -r -n1 char; do number=$number$(printf "%d" "'$char") if [ ${#number} -ge $digits ]; then @@ -138,7 +138,7 @@ else fi ## Special note when remote target is on the same host as initiator (happens for unit tests): we'll have to differentiate RUN_DIR so remote CleanUp won't affect initiator. -## If the same program gets remotely executed, add _REMOTE_EXECUTION=true to it's environment so it knows it has to write into a separate directory +## If the same program gets remotely executed, add _REMOTE_EXECUTION=true to environment so it knows it has to write into a separate directory ## This will thus not affect local $RUN_DIR variables if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" @@ -206,29 +206,29 @@ function RemoteLogger { if [ "$level" == "CRITICAL" ]; then _Logger "" "$prefix\e[1;33;41m$value\e[0m" true - if [ $_DEBUG == true ]; then + if [ "$_DEBUG" == true ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi return elif [ "$level" == "ERROR" ]; then _Logger "" "$prefix\e[31m$value\e[0m" true - if [ $_DEBUG == true ]; then + if [ "$_DEBUG" == true ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi return elif [ "$level" == "WARN" ]; then _Logger "" "$prefix\e[33m$value\e[0m" true - if [ $_DEBUG == true ]; then + if [ "$_DEBUG" == true ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi return elif [ "$level" == "NOTICE" ]; then - if [ $_LOGGER_ERR_ONLY != true ]; then + if [ "$_LOGGER_ERR_ONLY" != true ]; then _Logger "" "$prefix$value" fi return elif [ "$level" == "VERBOSE" ]; then - if [ $_LOGGER_VERBOSE == true ]; then + if [ "$_LOGGER_VERBOSE" == true ]; then _Logger "" "$prefix$value" fi return @@ -308,7 +308,7 @@ function Logger { fi return elif [ "$level" == "VERBOSE" ]; then - if [ $_LOGGER_VERBOSE == true ]; then + if [ "$_LOGGER_VERBOSE" == true ]; then _Logger "$prefix($level):$value" "$prefix$value" fi return @@ -443,7 +443,7 @@ function GenericTrapQuit { #### CleanUp SUBSET #### function CleanUp { - # Exit controlmaster before it's socket gets deleted + # Exit controlmaster before the socket gets deleted if [ "$SSH_CONTROLMASTER" == true ] && [ "$SSH_CMD" != "" ]; then $SSH_CMD -O exit fi @@ -738,7 +738,7 @@ function LoadConfigFile { Logger "Cannot load configuration file [$configFile]. Cannot start." "CRITICAL" exit 1 elif [[ "$configFile" != *".conf" ]]; then - Logger "Wrong configuration file supplied [$configFile]. Cannot start." "CRITICAL" + Logger "Wrong configuration file supplied [$configFile], file extension is not .conf, Cannot start." "CRITICAL" exit 1 else revisionPresent="$(GetConfFileValue "$configFile" "CONFIG_FILE_REVISION" true)" @@ -835,7 +835,7 @@ function ParallelExec { ## pid=$! ## sleep 20 & ## pid2=$! -## ExecTasks "some_identifier" 0 0 10 20 1 1800 true true false false 1 "$pid;$pid2" +## ExecTasks "$pid;$pid2" "some_identifier" 0 0 10 20 1 1800 true true false false 1 ## Example of parallel execution of four commands, only if directories exist. Warn if execution takes more than 300 seconds. Stop if takes longer than 900 seconds. Exeute max 3 commands in parallel. ## commands="du -csh /var;du -csh /etc;du -csh /home;du -csh /usr" @@ -858,7 +858,7 @@ function ExecTasks { local mainInput="${1}" # Contains list of pids / commands separated by semicolons or filepath to list of pids / commands # Optional arguments - local id="${2:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id + local id="${2:-(undisclosed)}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id local readFromFile="${3:-false}" # Is mainInput / auxInput a semicolon separated list (true) or a filepath (false) local softPerProcessTime="${4:-0}" # Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0 local hardPerProcessTime="${5:-0}" # Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0 @@ -981,7 +981,7 @@ function ExecTasks { fi if [ $functionMode == "WaitForTaskCompletion" ]; then - # Force first while loop condition to be true because we don't deal with counters but pids in WaitForTaskCompletion mode + # Force first while loop condition to be true because we do not deal with counters but pids in WaitForTaskCompletion mode counter=$mainItemCount fi @@ -1008,7 +1008,7 @@ function ExecTasks { if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1 second log_ttime=$exec_time if [ $functionMode == "WaitForTaskCompletion" ]; then - Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE" + Logger "Current tasks ID=$id still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE" elif [ $functionMode == "ParallelExec" ]; then Logger "There are $((mainItemCount-counter+postponedItemCount)) / $mainItemCount tasks in the queue of which $postponedItemCount are postponed. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE" fi @@ -1066,7 +1066,7 @@ function ExecTasks { pidState="$(eval $PROCESS_STATE_CMD)" if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then - # Check if pid hasn't run more than soft/hard perProcessTime + # Check if pid has not run more than soft/hard perProcessTime pidsTimeArray[$pid]=$((SECONDS - seconds_begin)) if [ ${pidsTimeArray[$pid]} -gt $softPerProcessTime ]; then if [ "$softAlert" != true ] && [ $softPerProcessTime -ne 0 ] && [ $noTimeErrorLog != true ]; then @@ -1110,7 +1110,7 @@ function ExecTasks { retval=$? # Check for valid exit codes if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then - if [ $noErrorLogsAtAll != true ]; then + if [ "$noErrorLogsAtAll" != true ]; then Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "ERROR" if [ "$functionMode" == "ParallelExec" ]; then Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR" @@ -1126,6 +1126,11 @@ function ExecTasks { else failedPidsList="$failedPidsList;$pid:$retval" fi + elif [ "$_DEBUG" == true ]; then + if [ -f "${commandsArrayOutput[$pid]}" ]; then + Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG" + Logger "Truncated output:\n$(head -c16384 "${commandsArrayOutput[$pid]}")" "DEBUG" + fi else Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG" fi @@ -1262,11 +1267,11 @@ function ExecTasks { if [ $executeCommand == true ]; then Logger "Running command [$currentCommand]." "DEBUG" randomOutputName=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 5) - eval "$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$pid.$randomOutputName.$SCRIPT_PID.$TSTAMP" 2>&1 & + eval "$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$randomOutputName.$SCRIPT_PID.$TSTAMP" 2>&1 & pid=$! pidsArray+=($pid) commandsArrayPid[$pid]="$currentCommand" - commandsArrayOutput[$pid]="$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$pid.$randomOutputName.$SCRIPT_PID.$TSTAMP" + commandsArrayOutput[$pid]="$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$randomOutputName.$SCRIPT_PID.$TSTAMP" # Initialize pid execution time array pidsTimeArray[$pid]=0 else @@ -1290,6 +1295,13 @@ function ExecTasks { eval "WAIT_FOR_TASK_COMPLETION_$id=\"$failedPidsList\"" + # CleanUp ExecTasks temp files + if [ "$_DEBUG" != true ]; then + [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$randomOutputName.$SCRIPT_PID.$TSTAMP" ] && rm -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$randomOutputName.$SCRIPT_PID.$TSTAMP" > /dev/null 2>&1 + [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedMain.$id.$SCRIPT_PID.$TSTAMP" ] && rm -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedMain.$id.$SCRIPT_PID.$TSTAMP" > /dev/null 2>&1 + [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedAux.$id.$SCRIPT_PID.$TSTAMP" ] && rm -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedAux.$id.$SCRIPT_PID.$TSTAMP" > /dev/null 2>&1 + fi + if [ $mainItemCount -eq 1 ]; then return $retval else @@ -1670,7 +1682,7 @@ function GetOs { localOsVer=$(grep "^VERSION=" "$osInfo") localOsVer="${localOsVer##*=}" elif [ "$localOsVar" == "BusyBox" ]; then - localOsVer=$(ls --help 2>&1 | head -1 | cut -f2 -d' ') + localOsVer="$(ls --help 2>&1 | head -1 | cut -f2 -d' ')" localOsName="BusyBox" fi @@ -2292,7 +2304,7 @@ function InitRemoteOSDependingSettings { fi fi if [ "$RSYNC_COMPRESS" == true ]; then - SKIP_COMPRESS_EXTENSIONS="--skip-compress=3fr/3g2/3gp/3gpp/7z/aac/ace/amr/apk/appx/appxbundle/arc/arj/arw/asf/avi/bz/bz2/cab/cr2/crypt[5678]/dat/dcr/deb/dmg/drc/ear/erf/flac/flv/gif/gpg/gz/iiq/jar/jp2/jpeg/jpg/h26[45]/k25/kdc/kgb/lha/lz/lzma/lzo/lzx/m4[apv]/mef/mkv/mos/mov/mp[34]/mpeg/mp[gv]/msi/nef/oga/ogg/ogv/opus/orf/pak/pef/png/qt/rar/r[0-9][0-9]/rz/rpm/rw2/rzip/s7z/sfark/sfx/sr2/srf/svgz/t[gb]z/tlz/txz/vob/wim/wma/wmv/xz/zip" + SKIP_COMPRESS_EXTENSIONS="--skip-compress=3fr/3g2/3gp/3gpp/7z/aac/ace/amr/apk/appx/appxbundle/arc/arj/arw/asf/avi/bz/bz2/cab/cr2/crypt[5678]/dat/dcr/deb/dmg/drc/ear/erf/flac/flv/gif/gpg/gz/iiq/jar/jp2/jpeg/jpg/h26[45]/k25/kdc/kgb/lha/lz/lzma/lzo/lzx/m4[apv]/mef/mkv/mos/mov/mp[34]/mpeg/mp[gv]/msi/nef/oga/ogg/ogv/opus/orf/pak/pef/png/qt/rar/r[0-9][0-9]/rz/rpm/rw2/rzip/s7z/sfark/sfx/sr2/srf/svgz/t[gb]z/tlz/txz/vob/wim/wma/wmv/xz/zip/zst" if [ "$LOCAL_OS" == "Qnap" ] || [ "$REMOTE_OS" == "Qnap" ]; then RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -z $SKIP_COMPRESS_EXTENSIONS" elif [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then @@ -2501,4 +2513,147 @@ function FileMove () { } #### FileMove SUBSET END #### +#### InotifyWaitPoller SUBSET #### + +# Checks directories every second for file changes using a static list +# Orignal code Copyright (c) 2020 Francesco Bianco +# Improved version https://github.com/javanile/inotifywait-polling/pull/1 by adding timeouts, optional recursion, excludes and a couple of fixes +print_event () { + [[ -z "${events}" || "${events}" == *"$2"* ]] && echo "$1 $2 $3" + case "$2" in + CREATE) + [[ -z "${events}" || "${events}" == *"OPEN"* ]] && echo "$1 OPEN $3" + [[ -z "${events}" || "${events}" == *"MODIFY"* ]] && echo "$1 MODIFY $3" + [[ -z "${events}" || "${events}" == *"CLOSE"* ]] && echo "$1 CLOSE_WRITE,CLOSE $3" + ;; + esac + return 0 +} + +function _InotifyWaitPoller () { + local path="${1}" + local includes="${2}" + local excludes="${3}" + local recursive="${4:-true}" + local monitor="${5:-false}" + local event_log_file="${6}" + local events="${7}" + local interval="${8}" + + local include + local exclude + local find_results_file + local find_recursion + local find_includes= + local find_excludes= + local stop_loop=false + local find_cmd + local notable_event=false + + find_results_file="$RUN_DIR/$PROGRAM._InotifyWaitPoller.${path//\//_}.$SCRIPT_PID.$TSTAMP" + + IFS=';' read -r -a includes <<< "$includes" + for include in "${includes[@]}"; do + find_includes="$find_includes -iregex \"$include\"" + done + + IFS=';' read -r -a excludes <<< "$excludes" + for exclude in "${excludes[@]}"; do + find_excludes="$find_excludes ! -iregex \"$exclude\"" + done + + if [ $recursive != true ]; then + find_recursion=" -maxdepth 1" + fi + + #[ "$event_log_file" != "" ] && [ -f "$event_log_file" ] && rm -f "$event_log_file" >/dev/null 2>&1 + + find_cmd="find \"$path\" ! -regex \"$RUN_DIR/$PROGRAM.*.$SCRIPT_PID.$TSTAMP\" $find_includes $find_excludes $find_recursion -printf \"%s %y %p\\n\" | sort -k3 - > \"$find_results_file\"" + Logger "COMMAND: $find_cmd" "VERBOSE" + eval "$find_cmd" + + while [ $stop_loop == false ]; do + sleep "$interval" + sign= + last_run_results=$(cat "$find_results_file") + eval "$find_cmd" + differences=$(diff <(echo "$last_run_results") <(cat "$find_results_file")) && true + if [ "$monitor" != true ] && [ "$notable_event" == true ]; then + stop_loop=true + fi + + [ -z "$differences" ] && continue + while IFS= read line || [[ -n "${line}" ]]; do + if [[ "${line}" == "." ]]; then + while IFS=';' read -r item; do + event="$(echo "${item}" | cut -s -d':' -f1)" + focus="$(echo "${item}" | cut -s -d':' -f2)" + dir="$(dirname "${focus}")/" + file="$(basename "${focus}" | cut -d';' -f1)" + if [[ "${events}" == *"${event}"* ]]; then + [ "$event_log_file" != "" ] && printf "${dir}${file}\0" >> "$event_log_file" || print_event "${dir}" "${event}" "${file}" + notable_event=true + fi + done <<< "${sign}" + break + fi + flag="$(echo "${line}" | cut -s -d' ' -f1)" + file="$(echo "${line}" | cut -s -d' ' -f4-)" + [[ -n "${file}" ]] || continue + [[ "${file}" != "$find_results_file" ]] || continue + case "${flag}" in + "<") + event=DELETE + ;; + ">") + event=CREATE + if [[ "${sign}" == *"DELETE:${file};"* ]]; then + event=MODIFY + sign="$(echo "${sign}" | sed "s#DELETE:${file};##g")" + elif [[ "${sign}" == *"DELETE:"* ]]; then + event=MOVED_TO + sign="$(echo "${sign}" | sed "s#DELETE:.*;##g")" + fi + ;; + esac + sign+="${event}:${file};" + done <<< "$(echo -e "${differences}\n.")" + done + [ -f "$find_results_file" ] && rm -f "$find_results_file" + exit 0 +} +# InotifyWaitPoller dirs exclusions recursive monitor_mode timeout opt_regex event_log events +function InotifyWaitPoller () { + local paths="${1}" # Comma separated list of paths + local includes="${2}" comma separated list of excludes in find regex format + local excludes="${3}" # Comma separated list of excludes in find regex format + local recursive="${4:-true}" + local monitor="${5:-false}" + local event_log_file="${6}" # optional event log file + local events="${7:-CREATE,MODIFY,MOVED_TO}" # possible events: CREATE,OPEN,MODIFY,CLOSE,CLOSE_WRITE,DELETE,MOVED_TO + local timeout="${8:-0}" # seconds + local interval="${9:-5}" # polling interval + + local pids= + local path + local alive + local count=0 + + IFS=';' read -r -a paths <<< "$paths" + for path in "${paths[@]}"; do + if ! [ -e "$path" ]; then + Logger "Cannot watch [$path]. No such file or directory." "CRITICAL" + else + _InotifyWaitPoller "$path" "$includes" "$excludes" "$recursive" "$monitor" "$event_log_file" "$events" "$interval" & + pids="$pids;$!" + fi + done + if [ "$pids" != "" ]; then + ExecTasks $pids "InotifyWaitPoller" false 0 0 0 $timeout + else + Logger "No directories available to watch." "CRITICAL" + fi +} +#### InotifyWaitPoller SUBSET END #### + #### OFUNCTIONS FULL SUBSET END ####