Archives
/
osync
mirror of https://github.com/deajan/osync
2
0
Fork 0
Code Issues Releases Wiki Activity

Rebuilt target

Browse Source
pull/60/head
deajan 8 years ago
parent 4418b28846
commit 2275efa3cc
1 changed files with 167 additions and 266 deletions
Show Stats Download Patch File Download Diff File

433
osync.sh
Unescape Escape View File

@ -4,10 +4,14 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.2-dev-parallel-unstable
PROGRAM_BUILD=2016080206
PROGRAM_BUILD=2016080401
IS_STABLE=no
## FUNC_BUILD=2016080306
#### MINIMAL-FUNCTION-SET BEGIN ####
## FUNC_BUILD=2016080402
## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
#TODO: set _LOGGER_PREFIX in other apps, specially for osync daemon mode
@ -19,8 +23,6 @@ if ! type "$BASH" > /dev/null; then
exit 127
fi
#### obackup & osync specific code BEGIN ####
## Log a state message every $KEEP_LOGGING seconds. Should not be equal to soft or hard execution time so your log will not be unnecessary big.
KEEP_LOGGING=1801
@ -30,13 +32,12 @@ export LC_ALL=C
# Standard alert mail body
MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
#### obackup & osync specific code END ####
#### MINIMAL-FUNCTION-SET BEGIN ####
# Environment variables
# Environment variables that can be overriden by programs
_DRYRUN=0
_SILENT=0
_LOGGER_PREFIX="date"
_LOGGER_STDERR=0
# Initial error status, logging 'WARN', 'ERROR' or 'CRITICAL' will enable alerts flags
ERROR_ALERT=0
@ -87,7 +88,8 @@ ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.last.log"
function Dummy {
sleep .1
sleep $SLEEP_TIME
}
# Sub function of Logger
@ -95,6 +97,7 @@ function _Logger {
local svalue="${1}" # What to log to stdout
local lvalue="${2:-$svalue}" # What to log to logfile, defaults to screen value
local evalue="${3}" # What to log to stderr
echo -e "$lvalue" >> "$LOG_FILE"
if [ "$_LOGGER_STDERR" -eq 1 ]; then
@ -109,6 +112,7 @@ function Logger {
local value="${1}" # Sentence to log (in double quotes)
local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, NOTICE, WARN, ERROR, CRITIAL
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
@ -148,6 +152,7 @@ function _QuickLogger {
local value="${1}"
local destination="${2}" # Destination: stdout, log, both
if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then
echo -e "$(date) - $value" >> "$LOG_FILE"
elif ([ "$destination" == "stdout" ] || [ "$destination" == "both" ]); then
@ -159,6 +164,7 @@ function _QuickLogger {
function QuickLogger {
local value="${1}"
if [ "$_SILENT" -eq 1 ]; then
_QuickLogger "$value" "log"
else
@ -200,6 +206,7 @@ function KillChilds {
function KillAllChilds {
local pids="${1}" # List of parent pids to kill separated by semi-colon
local errorcount=0
IFS=';' read -a pidsArray <<< "$pids"
@ -518,8 +525,6 @@ function LoadConfigFile {
CONFIG_FILE="$config_file"
}
#### MINIMAL-FUNCTION-SET END ####
function Spinner {
if [ $_SILENT -eq 1 ]; then
return 0
@ -553,6 +558,133 @@ function Spinner {
esac
}
function WaitForTaskCompletion {
local pids="${1}" # pids to wait for, separated by semi-colon
local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0.
local caller_name="${4}" # Who called this function
local exit_on_error="${5:-false}" # Should the function exit on subprocess errors
local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once
local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function
local retval=0 # return value of monitored pid process
local errorcount=0 # Number of pids that finished with errors
local pidCount # number of given pids
IFS=';' read -a pidsArray <<< "$pids"
pidCount=${#pidsArray[@]}
while [ ${#pidsArray[@]} -gt 0 ]; do
newPidsArray=()
for pid in "${pidsArray[@]}"; do
if kill -0 $pid > /dev/null 2>&1; then
newPidsArray+=($pid)
else
wait $pid
result=$?
if [ $result -ne 0 ]; then
errorcount=$((errorcount+1))
Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$result]." "DEBUG"
fi
fi
done
Spinner
exec_time=$(($SECONDS - $seconds_begin))
if [ $((($exec_time + 1) % $KEEP_LOGGING)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then
log_ttime=$exec_time
Logger "Current tasks still running with pids [${pidsArray[@]}]." "NOTICE"
fi
fi
if [ $exec_time -gt $soft_max_time ]; then
if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then
Logger "Max soft execution time exceeded for task [$caller_name] with pids [${pidsArray[@]}]." "WARN"
soft_alert=1
SendAlert
fi
if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then
Logger "Max hard execution time exceeded for task [$caller_name] with pids [${pidsArray[@]}]. Stopping task execution." "ERROR"
KillChilds $pid
if [ $? == 0 ]; then
Logger "Task stopped successfully" "NOTICE"
#return 0
else
errrorcount=$((errorcount+1))
#return 1
fi
fi
fi
pidsArray=("${newPidsArray[@]}")
sleep $SLEEP_TIME
done
if [ $exit_on_error == true ] && [ $errorcount -gt 0 ]; then
Logger "Stopping execution." "CRITICAL"
exit 1337
else
return $errorcount
fi
}
function WaitForCompletion {
local pid="${1}" # pid to wait for
local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0.
local caller_name="${4}" # Who called this function
local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once
local log_time=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function
local retval=0 # return value of monitored pid process
while eval "$PROCESS_TEST_CMD" > /dev/null
do
Spinner
if [ $((($SECONDS + 1) % $KEEP_LOGGING)) -eq 0 ]; then
if [ $log_time -ne $SECONDS ]; then
log_time=$SECONDS
Logger "Current task still running." "NOTICE"
fi
fi
if [ $SECONDS -gt $soft_max_time ]; then
if [ $soft_alert -eq 0 ] && [ $soft_max_time != 0 ]; then
Logger "Max soft execution time exceeded for script." "WARN"
soft_alert=1
SendAlert
fi
if [ $SECONDS -gt $hard_max_time ] && [ $hard_max_time != 0 ]; then
Logger "Max hard execution time exceeded for script in [$caller_name]. Stopping current task execution." "ERROR"
KillChilds $pid
if [ $? == 0 ]; then
Logger "Task stopped successfully" "NOTICE"
return 0
else
return 1
fi
fi
fi
sleep $SLEEP_TIME
done
wait $pid
retval=$?
return $retval
}
#### MINIMAL-FUNCTION-SET END ####
# obsolete, use StripQuotes
function SedStripQuotes {
echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g")
@ -730,232 +862,6 @@ function GetRemoteOS {
fi
}
function WaitForTaskCompletion {
local pids="${1}" # pids to wait for, separated by semi-colon
local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0.
local caller_name="${4}" # Who called this function
local exit_on_error="${5:-false}" # Should the function exit on subprocess errors
local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once
local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function
local retval=0 # return value of monitored pid process
local errorcount=0 # Number of pids that finished with errors
local pidCount # number of given pids
IFS=';' read -a pidsArray <<< "$pids"
pidCount=${#pidsArray[@]}
while [ ${#pidsArray[@]} -gt 0 ]; do
newPidsArray=()
for pid in "${pidsArray[@]}"; do
if kill -0 $pid > /dev/null 2>&1; then
newPidsArray+=($pid)
else
wait $pid
result=$?
if [ $result -ne 0 ]; then
errorcount=$((errorcount+1))
Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$result]." "DEBUG"
fi
fi
done
Spinner
exec_time=$(($SECONDS - $seconds_begin))
if [ $((($exec_time + 1) % $KEEP_LOGGING)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then
log_ttime=$exec_time
Logger "Current tasks still running with pids [${pidsArray[@]}]." "NOTICE"
fi
fi
if [ $exec_time -gt $soft_max_time ]; then
if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then
Logger "Max soft execution time exceeded for task [$caller_name] with pids [${pidsArray[@]}]." "WARN"
soft_alert=1
SendAlert
fi
if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then
Logger "Max hard execution time exceeded for task [$caller_name] with pids [${pidsArray[@]}]. Stopping task execution." "ERROR"
KillChilds $pid
if [ $? == 0 ]; then
Logger "Task stopped successfully" "NOTICE"
#return 0
else
errrorcount=$((errorcount+1))
#return 1
fi
fi
fi
pidsArray=("${newPidsArray[@]}")
sleep $SLEEP_TIME
done
if [ $exit_on_error == true ] && [ $errorcount -gt 0 ]; then
Logger "Stopping execution." "CRITICAL"
exit 1337
else
return $errorcount
fi
}
function WaitForOldTaskCompletion {
local pid="${1}" # pid to wait for
local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0.
local caller_name="${4}" # Who called this function
local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once
local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function
local retval=0 # return value of monitored pid process
while kill -0 $pid > /dev/null 2>&1
do
Spinner
exec_time=$(($SECONDS - $seconds_begin))
if [ $((($exec_time + 1) % $KEEP_LOGGING)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then
log_ttime=$exec_time
Logger "Current task still running with pid [$pid]." "NOTICE"
fi
fi
if [ $exec_time -gt $soft_max_time ]; then
if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then
Logger "Max soft execution time exceeded for task [$caller_name] with pid [$pid]." "WARN"
soft_alert=1
SendAlert
fi
if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then
Logger "Max hard execution time exceeded for task [$caller_name] with pid [$pid]. Stopping task execution." "ERROR"
KillChilds $pid
if [ $? == 0 ]; then
Logger "Task stopped successfully" "NOTICE"
return 0
else
return 1
fi
fi
fi
sleep $SLEEP_TIME
done
wait $pid
retval=$?
return $retval
}
function WaitForXTaskCompletion {
local pids="${1}" # pids to wait for, separated by semi-colon
local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0.
local caller_name="${4}" # Who called this function
local exit_on_error="${5}" # Should the function exit on subprocess errors
local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once
local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function
local retval=0 # return value of monitored pid process
while kill -0 $pids > /dev/null 2>&1
do
Spinner
exec_time=$(($SECONDS - $seconds_begin))
if [ $((($exec_time + 1) % $KEEP_LOGGING)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then
log_ttime=$exec_time
Logger "Current task still running with pid [$pid]." "NOTICE"
fi
fi
if [ $exec_time -gt $soft_max_time ]; then
if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then
Logger "Max soft execution time exceeded for task [$caller_name] with pid [$pid]." "WARN"
soft_alert=1
SendAlert
fi
if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then
Logger "Max hard execution time exceeded for task [$caller_name] with pid [$pid]. Stopping task execution." "ERROR"
KillChilds $pid
if [ $? == 0 ]; then
Logger "Task stopped successfully" "NOTICE"
return 0
else
return 1
fi
fi
fi
sleep $SLEEP_TIME
done
wait $pid
retval=$?
return $retval
}
function WaitForCompletion {
local pid="${1}" # pid to wait for
local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0.
local caller_name="${4}" # Who called this function
local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once
local log_time=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function
local retval=0 # return value of monitored pid process
while eval "$PROCESS_TEST_CMD" > /dev/null
do
Spinner
if [ $((($SECONDS + 1) % $KEEP_LOGGING)) -eq 0 ]; then
if [ $log_time -ne $SECONDS ]; then
log_time=$SECONDS
Logger "Current task still running." "NOTICE"
fi
fi
if [ $SECONDS -gt $soft_max_time ]; then
if [ $soft_alert -eq 0 ] && [ $soft_max_time != 0 ]; then
Logger "Max soft execution time exceeded for script." "WARN"
soft_alert=1
SendAlert
fi
if [ $SECONDS -gt $hard_max_time ] && [ $hard_max_time != 0 ]; then
Logger "Max hard execution time exceeded for script in [$caller_name]. Stopping current task execution." "ERROR"
KillChilds $pid
if [ $? == 0 ]; then
Logger "Task stopped successfully" "NOTICE"
return 0
else
return 1
fi
fi
fi
sleep $SLEEP_TIME
done
wait $pid
retval=$?
return $retval
}
function RunLocalCommand {
local command="${1}" # Command to run
local hard_max_time="${2}" # Max time to wait for command to compleet
@ -1344,17 +1250,11 @@ function InitRemoteOSSettings {
}
## END Generic functions
_LOGGER_PREFIX="time"
## Working directory. This directory exists in any replica and contains state files, backups, soft deleted files etc
OSYNC_DIR=".osync_workdir"
_LOGGER_PREFIX="time"
_LOGGER_STDERR=0
function TrapStop {
if [ $SOFT_STOP -eq 0 ]; then
Logger " /!\ WARNING: Manual exit of osync is really not recommended. Sync will be in inconsistent state." "WARN"
@ -1371,7 +1271,7 @@ function TrapStop {
}
function TrapQuit {
local exitcode=
local exitcode
if [ $ERROR_ALERT -ne 0 ]; then
UnlockReplicas
@ -1623,7 +1523,7 @@ function _CreateStateDirsLocal {
function _CreateStateDirsRemote {
local replica_state_dir="${1}"
local cmd=
local cmd
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
@ -1872,8 +1772,8 @@ function tree_list {
local replica_type="${2}" # replica type: initiator, target
local tree_filename="${3}" # filename to output tree (will be prefixed with $replica_type)
local escaped_replica_path=
local rsync_cmd=
local escaped_replica_path
local rsync_cmd
escaped_replica_path=$(EscapeSpaces "$replica_path")
@ -1910,7 +1810,7 @@ function delete_list {
local deleted_list_file="${4}" # file containing deleted file list, will be prefixed with replica type
local deleted_failed_list_file="${5}" # file containing files that could not be deleted on last run, will be prefixed with replica type
local cmd=
local cmd
Logger "Creating $replica_type replica deleted file list." "NOTICE"
if [ -f "${INITIATOR[1]}${INITIATOR[3]}/$replica_type$TREE_AFTER_FILENAME_NO_SUFFIX" ]; then
@ -1953,7 +1853,8 @@ function _get_file_ctime_mtime_remote {
local replica_type="${2}"
local file_list="${3}"
local cmd=
local cmd
cmd='cat "'$file_list'" | '$SSH_CMD' "while read file; do '$REMOTE_STAT_CTIME_MTIME_CMD' \"'$replica_path'\$file\"; done | sort" > "'$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID'"'
Logger "CMD: $cmd" "DEBUG"
eval $cmd
@ -1974,8 +1875,8 @@ function sync_attrs {
local target_replica="${2}"
local delete_list_filename="$DELETED_LIST_FILENAME" # Contains deleted list filename, will be prefixed with replica type
local rsync_cmd=
local retval=
local rsync_cmd
local retval
Logger "Getting list of files that need updates." "NOTICE"
@ -2088,8 +1989,8 @@ function sync_update {
local destination_replica="${2}" # Contains replica type of destination: initiator, target
local delete_list_filename="${3}" # Contains deleted list filename, will be prefixed with replica type
local rsync_cmd=
local retval=
local rsync_cmd
local retval
Logger "Updating $destination_replica replica." "NOTICE"
if [ "$source_replica" == "${INITIATOR[0]}" ]; then
@ -2144,7 +2045,7 @@ function _delete_local {
local deletion_dir="${3}" # deletion dir in format .[workdir]/deleted
local deleted_failed_list_file="${4}" # file containing files that could not be deleted on last run, will be prefixed with replica type
local parentdir=
local parentdir
## On every run, check wheter the next item is already deleted because it is included in a directory already deleted
local previous_file=""
@ -2205,8 +2106,8 @@ function _delete_remote {
local deletion_dir="${3}" # deletion dir in format .[workdir]/deleted
local deleted_failed_list_file="${4}" # file containing files that could not be deleted on last run, will be prefixed with replica type
local esc_dest_dir=
local rsync_cmd=
local esc_dest_dir
local rsync_cmd
## This is a special coded function. Need to redelcare local functions on remote host, passing all needed variables as escaped arguments to ssh command.
## Anything beetween << ENDSSH and ENDSSH will be executed remotely
@ -2339,8 +2240,8 @@ function deletion_propagation {
local deleted_list_file="${2}" # file containing deleted file list, will be prefixed with replica type
local deleted_failed_list_file="${3}" # file containing files that could not be deleted on last run, will be prefixed with replica type
local replica_dir=
local delete_dir=
local replica_dir
local delete_dir
Logger "Propagating deletions to $replica_type replica." "NOTICE"
@ -2565,7 +2466,7 @@ function _SoftDeleteLocal {
local replica_deletion_path="${2}" # Contains the full path to softdelete / backup directory without ending slash
local change_time="${3}"
local retval=
local retval
if [ -d "$replica_deletion_path" ]; then
if [ $_DRYRUN -eq 1 ]; then
@ -2603,7 +2504,7 @@ function _SoftDeleteRemote {
local replica_deletion_path="${2}" # Contains the full path to softdelete / backup directory without ending slash
local change_time="${3}"
local retval=
local retval
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
@ -2657,8 +2558,8 @@ function SoftDelete {
_SoftDeleteRemote "${TARGET[0]}" "${TARGET[1]}${TARGET[4]}" $CONFLICT_BACKUP_DAYS &
pids="$pids;$!"
fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false
fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false
if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]; then
Logger "Running soft deletion cleanup." "NOTICE"
@ -2672,8 +2573,8 @@ function SoftDelete {
_SoftDeleteRemote "${TARGET[0]}" "${TARGET[1]}${TARGET[5]}" $SOFT_DELETE_DAYS &
pids="$pids;$!"
fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false
fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false
}
function Init {
@ -2884,8 +2785,8 @@ function Usage {
function SyncOnChanges {
local cmd=
local retval=
local cmd
local retval
if ! type inotifywait > /dev/null 2>&1 ; then
Logger "No inotifywait command found. Cannot monitor changes." "CRITICAL"

Write Preview
Loading…
Cancel
Save