Archives
/
osync
mirror of https://github.com/deajan/osync
2
0
Fork 0
Code Issues Releases Wiki Activity

Update ofunctions from pmocr

Browse Source
pull/254/head
Orsiris de Jong 2 years ago
parent f2761d6a42
commit 4e44020619
1 changed files with 179 additions and 24 deletions
Show Stats Download Patch File Download Diff File

203
dev/ofunctions.sh
Unescape Escape View File

@ -1,8 +1,8 @@
#!/usr/bin/env bash
## Generic and highly portable bash functions written in 2013-2020 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## Generic and highly portable bash functions written in 2013-2022 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
#TODO: ExecTasks postponed arrays / files grow a lot. Consider having them "rolling" (cleaned at numberOfEvents)
#TODO: command line arguments don't take -AaqV for example
#TODO: command line arguments do not take merged args (-AaqV for example)
####################################################################################################################################################################
## To use in a program, define the following variables:
@ -30,8 +30,8 @@
#### OFUNCTIONS FULL SUBSET ####
#### OFUNCTIONS MINI SUBSET ####
#### OFUNCTIONS MICRO SUBSET ####
_OFUNCTIONS_VERSION=2.3.2
_OFUNCTIONS_BUILD=2021051801
_OFUNCTIONS_VERSION=2.4.3
_OFUNCTIONS_BUILD=2022050801
#### _OFUNCTIONS_BOOTSTRAP SUBSET ####
_OFUNCTIONS_BOOTSTRAP=true
#### _OFUNCTIONS_BOOTSTRAP SUBSET END ####
@ -94,7 +94,7 @@ function PoorMansRandomGenerator {
local digits="${1}" # The number of digits to generate
local number
# Some read bytes can't be used, se we read twice the number of required bytes
# Some read bytes cannot be used, se we read twice the number of required bytes
dd if=/dev/urandom bs=$digits count=2 2> /dev/null | while read -r -n1 char; do
number=$number$(printf "%d" "'$char")
if [ ${#number} -ge $digits ]; then
@ -138,7 +138,7 @@ else
fi
## Special note when remote target is on the same host as initiator (happens for unit tests): we'll have to differentiate RUN_DIR so remote CleanUp won't affect initiator.
## If the same program gets remotely executed, add _REMOTE_EXECUTION=true to it's environment so it knows it has to write into a separate directory
## If the same program gets remotely executed, add _REMOTE_EXECUTION=true to environment so it knows it has to write into a separate directory
## This will thus not affect local $RUN_DIR variables
if [ "$_REMOTE_EXECUTION" == true ]; then
mkdir -p "$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP"
@ -206,29 +206,29 @@ function RemoteLogger {
if [ "$level" == "CRITICAL" ]; then
_Logger "" "$prefix\e[1;33;41m$value\e[0m" true
if [ $_DEBUG == true ]; then
if [ "$_DEBUG" == true ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi
return
elif [ "$level" == "ERROR" ]; then
_Logger "" "$prefix\e[31m$value\e[0m" true
if [ $_DEBUG == true ]; then
if [ "$_DEBUG" == true ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi
return
elif [ "$level" == "WARN" ]; then
_Logger "" "$prefix\e[33m$value\e[0m" true
if [ $_DEBUG == true ]; then
if [ "$_DEBUG" == true ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi
return
elif [ "$level" == "NOTICE" ]; then
if [ $_LOGGER_ERR_ONLY != true ]; then
if [ "$_LOGGER_ERR_ONLY" != true ]; then
_Logger "" "$prefix$value"
fi
return
elif [ "$level" == "VERBOSE" ]; then
if [ $_LOGGER_VERBOSE == true ]; then
if [ "$_LOGGER_VERBOSE" == true ]; then
_Logger "" "$prefix$value"
fi
return
@ -308,7 +308,7 @@ function Logger {
fi
return
elif [ "$level" == "VERBOSE" ]; then
if [ $_LOGGER_VERBOSE == true ]; then
if [ "$_LOGGER_VERBOSE" == true ]; then
_Logger "$prefix($level):$value" "$prefix$value"
fi
return
@ -443,7 +443,7 @@ function GenericTrapQuit {
#### CleanUp SUBSET ####
function CleanUp {
# Exit controlmaster before it's socket gets deleted
# Exit controlmaster before the socket gets deleted
if [ "$SSH_CONTROLMASTER" == true ] && [ "$SSH_CMD" != "" ]; then
$SSH_CMD -O exit
fi
@ -738,7 +738,7 @@ function LoadConfigFile {
Logger "Cannot load configuration file [$configFile]. Cannot start." "CRITICAL"
exit 1
elif [[ "$configFile" != *".conf" ]]; then
Logger "Wrong configuration file supplied [$configFile]. Cannot start." "CRITICAL"
Logger "Wrong configuration file supplied [$configFile], file extension is not .conf, Cannot start." "CRITICAL"
exit 1
else
revisionPresent="$(GetConfFileValue "$configFile" "CONFIG_FILE_REVISION" true)"
@ -835,7 +835,7 @@ function ParallelExec {
## pid=$!
## sleep 20 &
## pid2=$!
## ExecTasks "some_identifier" 0 0 10 20 1 1800 true true false false 1 "$pid;$pid2"
## ExecTasks "$pid;$pid2" "some_identifier" 0 0 10 20 1 1800 true true false false 1
## Example of parallel execution of four commands, only if directories exist. Warn if execution takes more than 300 seconds. Stop if takes longer than 900 seconds. Exeute max 3 commands in parallel.
## commands="du -csh /var;du -csh /etc;du -csh /home;du -csh /usr"
@ -858,7 +858,7 @@ function ExecTasks {
local mainInput="${1}" # Contains list of pids / commands separated by semicolons or filepath to list of pids / commands
# Optional arguments
local id="${2:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id
local id="${2:-(undisclosed)}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id
local readFromFile="${3:-false}" # Is mainInput / auxInput a semicolon separated list (true) or a filepath (false)
local softPerProcessTime="${4:-0}" # Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0
local hardPerProcessTime="${5:-0}" # Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0
@ -981,7 +981,7 @@ function ExecTasks {
fi
if [ $functionMode == "WaitForTaskCompletion" ]; then
# Force first while loop condition to be true because we don't deal with counters but pids in WaitForTaskCompletion mode
# Force first while loop condition to be true because we do not deal with counters but pids in WaitForTaskCompletion mode
counter=$mainItemCount
fi
@ -1008,7 +1008,7 @@ function ExecTasks {
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1 second
log_ttime=$exec_time
if [ $functionMode == "WaitForTaskCompletion" ]; then
Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
Logger "Current tasks ID=$id still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
elif [ $functionMode == "ParallelExec" ]; then
Logger "There are $((mainItemCount-counter+postponedItemCount)) / $mainItemCount tasks in the queue of which $postponedItemCount are postponed. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
fi
@ -1066,7 +1066,7 @@ function ExecTasks {
pidState="$(eval $PROCESS_STATE_CMD)"
if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
# Check if pid hasn't run more than soft/hard perProcessTime
# Check if pid has not run more than soft/hard perProcessTime
pidsTimeArray[$pid]=$((SECONDS - seconds_begin))
if [ ${pidsTimeArray[$pid]} -gt $softPerProcessTime ]; then
if [ "$softAlert" != true ] && [ $softPerProcessTime -ne 0 ] && [ $noTimeErrorLog != true ]; then
@ -1110,7 +1110,7 @@ function ExecTasks {
retval=$?
# Check for valid exit codes
if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then
if [ $noErrorLogsAtAll != true ]; then
if [ "$noErrorLogsAtAll" != true ]; then
Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "ERROR"
if [ "$functionMode" == "ParallelExec" ]; then
Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR"
@ -1126,6 +1126,11 @@ function ExecTasks {
else
failedPidsList="$failedPidsList;$pid:$retval"
fi
elif [ "$_DEBUG" == true ]; then
if [ -f "${commandsArrayOutput[$pid]}" ]; then
Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG"
Logger "Truncated output:\n$(head -c16384 "${commandsArrayOutput[$pid]}")" "DEBUG"
fi
else
Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG"
fi
@ -1262,11 +1267,11 @@ function ExecTasks {
if [ $executeCommand == true ]; then
Logger "Running command [$currentCommand]." "DEBUG"
randomOutputName=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 5)
eval "$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$pid.$randomOutputName.$SCRIPT_PID.$TSTAMP" 2>&1 &
eval "$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$randomOutputName.$SCRIPT_PID.$TSTAMP" 2>&1 &
pid=$!
pidsArray+=($pid)
commandsArrayPid[$pid]="$currentCommand"
commandsArrayOutput[$pid]="$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$pid.$randomOutputName.$SCRIPT_PID.$TSTAMP"
commandsArrayOutput[$pid]="$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$randomOutputName.$SCRIPT_PID.$TSTAMP"
# Initialize pid execution time array
pidsTimeArray[$pid]=0
else
@ -1290,6 +1295,13 @@ function ExecTasks {
eval "WAIT_FOR_TASK_COMPLETION_$id=\"$failedPidsList\""
# CleanUp ExecTasks temp files
if [ "$_DEBUG" != true ]; then
[ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$randomOutputName.$SCRIPT_PID.$TSTAMP" ] && rm -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$randomOutputName.$SCRIPT_PID.$TSTAMP" > /dev/null 2>&1
[ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedMain.$id.$SCRIPT_PID.$TSTAMP" ] && rm -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedMain.$id.$SCRIPT_PID.$TSTAMP" > /dev/null 2>&1
[ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedAux.$id.$SCRIPT_PID.$TSTAMP" ] && rm -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedAux.$id.$SCRIPT_PID.$TSTAMP" > /dev/null 2>&1
fi
if [ $mainItemCount -eq 1 ]; then
return $retval
else
@ -1670,7 +1682,7 @@ function GetOs {
localOsVer=$(grep "^VERSION=" "$osInfo")
localOsVer="${localOsVer##*=}"
elif [ "$localOsVar" == "BusyBox" ]; then
localOsVer=$(ls --help 2>&1 | head -1 | cut -f2 -d' ')
localOsVer="$(ls --help 2>&1 | head -1 | cut -f2 -d' ')"
localOsName="BusyBox"
fi
@ -2292,7 +2304,7 @@ function InitRemoteOSDependingSettings {
fi
fi
if [ "$RSYNC_COMPRESS" == true ]; then
SKIP_COMPRESS_EXTENSIONS="--skip-compress=3fr/3g2/3gp/3gpp/7z/aac/ace/amr/apk/appx/appxbundle/arc/arj/arw/asf/avi/bz/bz2/cab/cr2/crypt[5678]/dat/dcr/deb/dmg/drc/ear/erf/flac/flv/gif/gpg/gz/iiq/jar/jp2/jpeg/jpg/h26[45]/k25/kdc/kgb/lha/lz/lzma/lzo/lzx/m4[apv]/mef/mkv/mos/mov/mp[34]/mpeg/mp[gv]/msi/nef/oga/ogg/ogv/opus/orf/pak/pef/png/qt/rar/r[0-9][0-9]/rz/rpm/rw2/rzip/s7z/sfark/sfx/sr2/srf/svgz/t[gb]z/tlz/txz/vob/wim/wma/wmv/xz/zip"
SKIP_COMPRESS_EXTENSIONS="--skip-compress=3fr/3g2/3gp/3gpp/7z/aac/ace/amr/apk/appx/appxbundle/arc/arj/arw/asf/avi/bz/bz2/cab/cr2/crypt[5678]/dat/dcr/deb/dmg/drc/ear/erf/flac/flv/gif/gpg/gz/iiq/jar/jp2/jpeg/jpg/h26[45]/k25/kdc/kgb/lha/lz/lzma/lzo/lzx/m4[apv]/mef/mkv/mos/mov/mp[34]/mpeg/mp[gv]/msi/nef/oga/ogg/ogv/opus/orf/pak/pef/png/qt/rar/r[0-9][0-9]/rz/rpm/rw2/rzip/s7z/sfark/sfx/sr2/srf/svgz/t[gb]z/tlz/txz/vob/wim/wma/wmv/xz/zip/zst"
if [ "$LOCAL_OS" == "Qnap" ] || [ "$REMOTE_OS" == "Qnap" ]; then
RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -z $SKIP_COMPRESS_EXTENSIONS"
elif [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then
@ -2501,4 +2513,147 @@ function FileMove () {
}
#### FileMove SUBSET END ####
#### InotifyWaitPoller SUBSET ####
# Checks directories every second for file changes using a static list
# Orignal code Copyright (c) 2020 Francesco Bianco <bianco@javanile.org>
# Improved version https://github.com/javanile/inotifywait-polling/pull/1 by adding timeouts, optional recursion, excludes and a couple of fixes
print_event () {
[[ -z "${events}" || "${events}" == *"$2"* ]] && echo "$1 $2 $3"
case "$2" in
CREATE)
[[ -z "${events}" || "${events}" == *"OPEN"* ]] && echo "$1 OPEN $3"
[[ -z "${events}" || "${events}" == *"MODIFY"* ]] && echo "$1 MODIFY $3"
[[ -z "${events}" || "${events}" == *"CLOSE"* ]] && echo "$1 CLOSE_WRITE,CLOSE $3"
;;
esac
return 0
}
function _InotifyWaitPoller () {
local path="${1}"
local includes="${2}"
local excludes="${3}"
local recursive="${4:-true}"
local monitor="${5:-false}"
local event_log_file="${6}"
local events="${7}"
local interval="${8}"
local include
local exclude
local find_results_file
local find_recursion
local find_includes=
local find_excludes=
local stop_loop=false
local find_cmd
local notable_event=false
find_results_file="$RUN_DIR/$PROGRAM._InotifyWaitPoller.${path//\//_}.$SCRIPT_PID.$TSTAMP"
IFS=';' read -r -a includes <<< "$includes"
for include in "${includes[@]}"; do
find_includes="$find_includes -iregex \"$include\""
done
IFS=';' read -r -a excludes <<< "$excludes"
for exclude in "${excludes[@]}"; do
find_excludes="$find_excludes ! -iregex \"$exclude\""
done
if [ $recursive != true ]; then
find_recursion=" -maxdepth 1"
fi
#[ "$event_log_file" != "" ] && [ -f "$event_log_file" ] && rm -f "$event_log_file" >/dev/null 2>&1
find_cmd="find \"$path\" ! -regex \"$RUN_DIR/$PROGRAM.*.$SCRIPT_PID.$TSTAMP\" $find_includes $find_excludes $find_recursion -printf \"%s %y %p\\n\" | sort -k3 - > \"$find_results_file\""
Logger "COMMAND: $find_cmd" "VERBOSE"
eval "$find_cmd"
while [ $stop_loop == false ]; do
sleep "$interval"
sign=
last_run_results=$(cat "$find_results_file")
eval "$find_cmd"
differences=$(diff <(echo "$last_run_results") <(cat "$find_results_file")) && true
if [ "$monitor" != true ] && [ "$notable_event" == true ]; then
stop_loop=true
fi
[ -z "$differences" ] && continue
while IFS= read line || [[ -n "${line}" ]]; do
if [[ "${line}" == "." ]]; then
while IFS=';' read -r item; do
event="$(echo "${item}" | cut -s -d':' -f1)"
focus="$(echo "${item}" | cut -s -d':' -f2)"
dir="$(dirname "${focus}")/"
file="$(basename "${focus}" | cut -d';' -f1)"
if [[ "${events}" == *"${event}"* ]]; then
[ "$event_log_file" != "" ] && printf "${dir}${file}\0" >> "$event_log_file" || print_event "${dir}" "${event}" "${file}"
notable_event=true
fi
done <<< "${sign}"
break
fi
flag="$(echo "${line}" | cut -s -d' ' -f1)"
file="$(echo "${line}" | cut -s -d' ' -f4-)"
[[ -n "${file}" ]] || continue
[[ "${file}" != "$find_results_file" ]] || continue
case "${flag}" in
"<")
event=DELETE
;;
">")
event=CREATE
if [[ "${sign}" == *"DELETE:${file};"* ]]; then
event=MODIFY
sign="$(echo "${sign}" | sed "s#DELETE:${file};##g")"
elif [[ "${sign}" == *"DELETE:"* ]]; then
event=MOVED_TO
sign="$(echo "${sign}" | sed "s#DELETE:.*;##g")"
fi
;;
esac
sign+="${event}:${file};"
done <<< "$(echo -e "${differences}\n.")"
done
[ -f "$find_results_file" ] && rm -f "$find_results_file"
exit 0
}
# InotifyWaitPoller dirs exclusions recursive monitor_mode timeout opt_regex event_log events
function InotifyWaitPoller () {
local paths="${1}" # Comma separated list of paths
local includes="${2}" comma separated list of excludes in find regex format
local excludes="${3}" # Comma separated list of excludes in find regex format
local recursive="${4:-true}"
local monitor="${5:-false}"
local event_log_file="${6}" # optional event log file
local events="${7:-CREATE,MODIFY,MOVED_TO}" # possible events: CREATE,OPEN,MODIFY,CLOSE,CLOSE_WRITE,DELETE,MOVED_TO
local timeout="${8:-0}" # seconds
local interval="${9:-5}" # polling interval
local pids=
local path
local alive
local count=0
IFS=';' read -r -a paths <<< "$paths"
for path in "${paths[@]}"; do
if ! [ -e "$path" ]; then
Logger "Cannot watch [$path]. No such file or directory." "CRITICAL"
else
_InotifyWaitPoller "$path" "$includes" "$excludes" "$recursive" "$monitor" "$event_log_file" "$events" "$interval" &
pids="$pids;$!"
fi
done
if [ "$pids" != "" ]; then
ExecTasks $pids "InotifyWaitPoller" false 0 0 0 $timeout
else
Logger "No directories available to watch." "CRITICAL"
fi
}
#### InotifyWaitPoller SUBSET END ####
#### OFUNCTIONS FULL SUBSET END ####

Write Preview
Loading…
Cancel
Save