Archives
/
osync
mirror of https://github.com/deajan/osync
2
0
Fork 0
Code Issues Releases Wiki Activity

Bug fix for in tree_list error log failing for WaitForCompletion

Browse Source
pull/48/merge
deajan 8 years ago
parent d0da1398d2
commit 7a485bd784
5 changed files with 373 additions and 326 deletions
Show Stats Download Patch File Download Diff File

11
CHANGELOG.md
Unescape Escape View File

@ -9,12 +9,17 @@ KNOWN ISSUES
- Cannot finish sync if one replica contains a directory and the other replica contains a file named the same way (Unix doesn't allow this)
- Soft deletion does not honor exclusion lists
- Permissions aren't set right when setfacl / chmod only is used and the file is not modified (WIP for v1.1)
RECENT CHANGES
--------------
! XX Dec 2015: osync v1.1 released
! XX Mar 2016: osync v1.1 released
- Fixed a nasty bug with log writing and tree_list function
- Improved mail fallback
- Improved more logging
- Fixed conflict prevalance is target in quicksync mode
- Fixed file attributes aren't updated in a right manner when file mtime is not altered (Big thanks to vstefanoxx)
- Better upgrade script (adding missing new config values)
- More fixes for GNU / non-GNU versions of mail command
- Added bogus config file checks & environment checks
@ -36,7 +41,7 @@ RECENT CHANGES
- Improved Logging
- Updated osync to be fully compliant with coding style
- Uploaded coding style manifest
- Integrated new realpath emulation from https://github.com/mkropat/sh-realpath
!- Integrated new realpath emulation from https://github.com/mkropat/sh-realpath
v0-v1.0x - Jun 2013 - Sep 2015
- Added LSB info to init script for Debian based distros

230
dev/debug_osync.sh
Unescape Escape View File

@ -8,10 +8,10 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(L) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.1-dev
PROGRAM_BUILD=2016030101
PROGRAM_BUILD=2016030801
IS_STABLE=no
## FUNC_BUILD=2016021803
## FUNC_BUILD=2016030801
## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
@ -44,6 +44,8 @@ else
_VERBOSE=1
fi
#### MINIMAL-FUNCTION-SET BEGIN ####
SCRIPT_PID=$$
LOCAL_USER=$(whoami)
@ -110,15 +112,15 @@ function Logger {
# </OSYNC SPECIFIC>
if [ "$level" == "CRITICAL" ]; then
_Logger "$prefix\e[41m$value\e[0m" "$prefix$value"
_Logger "$prefix\e[41m$value\e[0m" "$prefix$level:$value"
ERROR_ALERT=1
return
elif [ "$level" == "ERROR" ]; then
_Logger "$prefix\e[91m$value\e[0m" "$prefix$value"
_Logger "$prefix\e[91m$value\e[0m" "$prefix$level:$value"
ERROR_ALERT=1
return
elif [ "$level" == "WARN" ]; then
_Logger "$prefix\e[93m$value\e[0m" "$prefix$value"
_Logger "$prefix\e[93m$value\e[0m" "$prefix$level:$value"
WARN_ALERT=1
return
elif [ "$level" == "NOTICE" ]; then
@ -155,95 +157,7 @@ function KillChilds {
if [ "$self" == true ]; then
kill -s SIGTERM "$pid" || (sleep 30 && kill -9 "$pid" &)
fi
}
function TrapError {
local job="$0"
local line="$1"
local code="${2:-1}"
if [ $_SILENT -eq 0 ]; then
echo -e " /!\ ERROR in ${job}: Near line ${line}, exit code ${code}"
fi
}
function Spinner {
if [ $_SILENT -eq 1 ]; then
return 0
fi
case $toggle
in
1)
echo -n " \ "
echo -ne "\r"
toggle="2"
;;
2)
echo -n " | "
echo -ne "\r"
toggle="3"
;;
3)
echo -n " / "
echo -ne "\r"
toggle="4"
;;
*)
echo -n " - "
echo -ne "\r"
toggle="1"
;;
esac
}
function SedStripQuotes {
echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g")
}
function StripSingleQuotes {
local string="${1}"
string="${string/#\'/}" # Remove singlequote if it begins string
string="${string/%\'/}" # Remove singlequote if it ends string
echo "$string"
}
function StripDoubleQuotes {
local string="${1}"
string="${string/#\"/}"
string="${string/%\"/}"
echo "$string"
}
function StripQuotes {
local string="${1}"
echo "$(StripSingleQuotes $(StripDoubleQuotes $string))"
}
function EscapeSpaces {
local string="${1}" # String on which spaces will be escaped
echo "${string// /\ }"
}
function IsNumeric {
eval "local value=\"${1}\"" # Needed so variable variables can be processed
local re="^-?[0-9]+([.][0-9]+)?$"
if [[ $value =~ $re ]]; then
echo 1
else
echo 0
fi
}
function CleanUp {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
if [ "$_DEBUG" != "yes" ]; then
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID"
fi
# sleep 30 needs to wait before killing itself
}
function SendAlert {
@ -252,6 +166,10 @@ function SendAlert {
local mail_no_attachment=
local attachment_command=
if [ "$DESTINATION_MAILS" == "" ]; then
return 0
fi
if [ "$_DEBUG" == "yes" ]; then
Logger "Debug mode, no warning email will be sent." "NOTICE"
return 0
@ -284,9 +202,10 @@ function SendAlert {
attachment_command="-a $ALERT_LOG_FILE"
fi
if type mutt > /dev/null 2>&1 ; then
echo "$MAIL_ALERT_MSG" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS $attachment_command
cmd="echo \"$MAIL_ALERT_MSG\" | $(type -p mutt) -x -s \"$subject\" $DESTINATION_MAILS $attachment_command"
eval $cmd
if [ $? != 0 ]; then
Logger "WARNING: Cannot send alert email via $(type -p mutt) !!!" "WARN"
Logger "Cannot send alert email via $(type -p mutt) !!!" "WARN"
else
Logger "Sent alert mail using mutt." "NOTICE"
return 0
@ -301,12 +220,14 @@ function SendAlert {
else
attachment_command=""
fi
echo "$MAIL_ALERT_MSG" | $(type -p mail) $attachment_command -s "$subject" $DESTINATION_MAILS
cmd="echo \"$MAIL_ALERT_MSG\" | $(type -p mail) $attachment_command -s \"$subject\" $DESTINATION_MAILS"
eval $cmd
if [ $? != 0 ]; then
Logger "WARNING: Cannot send alert email via $(type -p mail) with attachments !!!" "WARN"
echo "$MAIL_ALERT_MSG" | $(type -p mail) -s "$subject" $DESTINATION_MAILS
Logger "Cannot send alert email via $(type -p mail) with attachments !!!" "WARN"
cmd="echo \"$MAIL_ALERT_MSG\" | $(type -p mail) -s \"$subject\" $DESTINATION_MAILS"
eval $cmd
if [ $? != 0 ]; then
Logger "WARNING: Cannot send alert email via $(type -p mail) without attachments !!!" "WARN"
Logger "Cannot send alert email via $(type -p mail) without attachments !!!" "WARN"
else
Logger "Sent alert mail using mail command without attachment." "NOTICE"
return 0
@ -318,9 +239,10 @@ function SendAlert {
fi
if type sendmail > /dev/null 2>&1 ; then
echo -e "Subject:$subject\r\n$MAIL_ALERT_MSG" | $(type -p sendmail) $DESTINATION_MAILS
cmd="echo -e \"Subject:$subject\r\n$MAIL_ALERT_MSG\" | $(type -p sendmail) $DESTINATION_MAILS"
eval $cmd
if [ $? != 0 ]; then
Logger "WARNING: Cannot send alert email via $(type -p sendmail) !!!" "WARN"
Logger "Cannot send alert email via $(type -p sendmail) !!!" "WARN"
else
Logger "Sent alert mail using sendmail command without attachment." "NOTICE"
return 0
@ -335,7 +257,7 @@ function SendAlert {
fi
$(type -p sendemail) -f $SENDER_MAIL -t $DESTINATION_MAILS -u "$subject" -m "$MAIL_ALERT_MSG" -s $SMTP_SERVER $SMTP_OPTIONS > /dev/null 2>&1
if [ $? != 0 ]; then
Logger "WARNING: Cannot send alert email via $(type -p sendemail) !!!" "WARN"
Logger "Cannot send alert email via $(type -p sendemail) !!!" "WARN"
else
Logger "Sent alert mail using sendemail command without attachment." "NOTICE"
return 0
@ -343,7 +265,7 @@ function SendAlert {
fi
# If function has not returned 0 yet, assume it's critical that no alert can be sent
Logger "/!\ CRITICAL: Cannot send alert (neither mutt, mail, sendmail nor sendemail found)." "ERROR" # Is not marked critical because execution must continue
Logger "Cannot send alert (neither mutt, mail, sendmail nor sendemail found)." "ERROR" # Is not marked critical because execution must continue
# Delete tmp log file
if [ -f "$ALERT_LOG_FILE" ]; then
@ -351,6 +273,97 @@ function SendAlert {
fi
}
#### MINIMAL-FUNCTION-SET END ####
function TrapError {
local job="$0"
local line="$1"
local code="${2:-1}"
if [ $_SILENT -eq 0 ]; then
echo -e " /!\ ERROR in ${job}: Near line ${line}, exit code ${code}"
fi
}
function Spinner {
if [ $_SILENT -eq 1 ]; then
return 0
fi
case $toggle
in
1)
echo -n " \ "
echo -ne "\r"
toggle="2"
;;
2)
echo -n " | "
echo -ne "\r"
toggle="3"
;;
3)
echo -n " / "
echo -ne "\r"
toggle="4"
;;
*)
echo -n " - "
echo -ne "\r"
toggle="1"
;;
esac
}
function SedStripQuotes {
echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g")
}
function StripSingleQuotes {
local string="${1}"
string="${string/#\'/}" # Remove singlequote if it begins string
string="${string/%\'/}" # Remove singlequote if it ends string
echo "$string"
}
function StripDoubleQuotes {
local string="${1}"
string="${string/#\"/}"
string="${string/%\"/}"
echo "$string"
}
function StripQuotes {
local string="${1}"
echo "$(StripSingleQuotes $(StripDoubleQuotes $string))"
}
function EscapeSpaces {
local string="${1}" # String on which spaces will be escaped
echo "${string// /\ }"
}
function IsNumeric {
eval "local value=\"${1}\"" # Needed so variable variables can be processed
local re="^-?[0-9]+([.][0-9]+)?$"
if [[ $value =~ $re ]]; then
echo 1
else
echo 0
fi
}
function CleanUp {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
if [ "$_DEBUG" != "yes" ]; then
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID"
fi
}
function LoadConfigFile {
local config_file="${1}"
__CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
@ -493,13 +506,13 @@ function WaitForTaskCompletion {
fi
if [ $exec_time -gt $soft_max_time ]; then
if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then
Logger "Max soft execution time exceeded for task." "WARN"
Logger "Max soft execution time exceeded for task [$caller_name]." "WARN"
soft_alert=1
SendAlert
fi
if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then
Logger "Max hard execution time exceeded for task. Stopping task execution." "ERROR"
Logger "Max hard execution time exceeded for task [$caller_name]. Stopping task execution." "ERROR"
kill -s SIGTERM $pid
if [ $? == 0 ]; then
Logger "Task stopped succesfully" "NOTICE"
@ -1482,8 +1495,9 @@ function tree_list {
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
## Redirect commands stderr here to get rsync stderr output in logfile
eval "$rsync_cmd" 2>> "$LOG_FILE"
## Redirect commands stderr here to get rsync stderr output in logfile with eval "$rsync_cmd" 2>> "$LOG_FILE"
## When log writing fails, $! is empty and WaitForCompletion fails. Removing the 2>> log
eval "$rsync_cmd"
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
retval=$?
## Retval 24 = some files vanished while creating list

7
dev/n_osync.sh
Unescape Escape View File

@ -8,7 +8,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(L) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.1-dev
PROGRAM_BUILD=2016030101
PROGRAM_BUILD=2016030801
IS_STABLE=no
source "./ofunctions.sh"
@ -513,8 +513,9 @@ function tree_list {
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
## Redirect commands stderr here to get rsync stderr output in logfile
eval "$rsync_cmd" 2>> "$LOG_FILE"
## Redirect commands stderr here to get rsync stderr output in logfile with eval "$rsync_cmd" 2>> "$LOG_FILE"
## When log writing fails, $! is empty and WaitForCompletion fails. Removing the 2>> log
eval "$rsync_cmd"
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
retval=$?
## Retval 24 = some files vanished while creating list

223
dev/ofunctions.sh
Unescape Escape View File

@ -1,4 +1,4 @@
## FUNC_BUILD=2016021803
## FUNC_BUILD=2016030801
## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
@ -31,6 +31,8 @@ else
_VERBOSE=1
fi
#### MINIMAL-FUNCTION-SET BEGIN ####
SCRIPT_PID=$$
LOCAL_USER=$(whoami)
@ -97,15 +99,15 @@ function Logger {
# </OSYNC SPECIFIC>
if [ "$level" == "CRITICAL" ]; then
_Logger "$prefix\e[41m$value\e[0m" "$prefix$value"
_Logger "$prefix\e[41m$value\e[0m" "$prefix$level:$value"
ERROR_ALERT=1
return
elif [ "$level" == "ERROR" ]; then
_Logger "$prefix\e[91m$value\e[0m" "$prefix$value"
_Logger "$prefix\e[91m$value\e[0m" "$prefix$level:$value"
ERROR_ALERT=1
return
elif [ "$level" == "WARN" ]; then
_Logger "$prefix\e[93m$value\e[0m" "$prefix$value"
_Logger "$prefix\e[93m$value\e[0m" "$prefix$level:$value"
WARN_ALERT=1
return
elif [ "$level" == "NOTICE" ]; then
@ -142,95 +144,7 @@ function KillChilds {
if [ "$self" == true ]; then
kill -s SIGTERM "$pid" || (sleep 30 && kill -9 "$pid" &)
fi
}
function TrapError {
local job="$0"
local line="$1"
local code="${2:-1}"
if [ $_SILENT -eq 0 ]; then
echo -e " /!\ ERROR in ${job}: Near line ${line}, exit code ${code}"
fi
}
function Spinner {
if [ $_SILENT -eq 1 ]; then
return 0
fi
case $toggle
in
1)
echo -n " \ "
echo -ne "\r"
toggle="2"
;;
2)
echo -n " | "
echo -ne "\r"
toggle="3"
;;
3)
echo -n " / "
echo -ne "\r"
toggle="4"
;;
*)
echo -n " - "
echo -ne "\r"
toggle="1"
;;
esac
}
function SedStripQuotes {
echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g")
}
function StripSingleQuotes {
local string="${1}"
string="${string/#\'/}" # Remove singlequote if it begins string
string="${string/%\'/}" # Remove singlequote if it ends string
echo "$string"
}
function StripDoubleQuotes {
local string="${1}"
string="${string/#\"/}"
string="${string/%\"/}"
echo "$string"
}
function StripQuotes {
local string="${1}"
echo "$(StripSingleQuotes $(StripDoubleQuotes $string))"
}
function EscapeSpaces {
local string="${1}" # String on which spaces will be escaped
echo "${string// /\ }"
}
function IsNumeric {
eval "local value=\"${1}\"" # Needed so variable variables can be processed
local re="^-?[0-9]+([.][0-9]+)?$"
if [[ $value =~ $re ]]; then
echo 1
else
echo 0
fi
}
function CleanUp {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
if [ "$_DEBUG" != "yes" ]; then
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID"
fi
# sleep 30 needs to wait before killing itself
}
function SendAlert {
@ -239,6 +153,10 @@ function SendAlert {
local mail_no_attachment=
local attachment_command=
if [ "$DESTINATION_MAILS" == "" ]; then
return 0
fi
if [ "$_DEBUG" == "yes" ]; then
Logger "Debug mode, no warning email will be sent." "NOTICE"
return 0
@ -271,9 +189,10 @@ function SendAlert {
attachment_command="-a $ALERT_LOG_FILE"
fi
if type mutt > /dev/null 2>&1 ; then
echo "$MAIL_ALERT_MSG" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS $attachment_command
cmd="echo \"$MAIL_ALERT_MSG\" | $(type -p mutt) -x -s \"$subject\" $DESTINATION_MAILS $attachment_command"
eval $cmd
if [ $? != 0 ]; then
Logger "WARNING: Cannot send alert email via $(type -p mutt) !!!" "WARN"
Logger "Cannot send alert email via $(type -p mutt) !!!" "WARN"
else
Logger "Sent alert mail using mutt." "NOTICE"
return 0
@ -288,12 +207,14 @@ function SendAlert {
else
attachment_command=""
fi
echo "$MAIL_ALERT_MSG" | $(type -p mail) $attachment_command -s "$subject" $DESTINATION_MAILS
cmd="echo \"$MAIL_ALERT_MSG\" | $(type -p mail) $attachment_command -s \"$subject\" $DESTINATION_MAILS"
eval $cmd
if [ $? != 0 ]; then
Logger "WARNING: Cannot send alert email via $(type -p mail) with attachments !!!" "WARN"
echo "$MAIL_ALERT_MSG" | $(type -p mail) -s "$subject" $DESTINATION_MAILS
Logger "Cannot send alert email via $(type -p mail) with attachments !!!" "WARN"
cmd="echo \"$MAIL_ALERT_MSG\" | $(type -p mail) -s \"$subject\" $DESTINATION_MAILS"
eval $cmd
if [ $? != 0 ]; then
Logger "WARNING: Cannot send alert email via $(type -p mail) without attachments !!!" "WARN"
Logger "Cannot send alert email via $(type -p mail) without attachments !!!" "WARN"
else
Logger "Sent alert mail using mail command without attachment." "NOTICE"
return 0
@ -305,9 +226,10 @@ function SendAlert {
fi
if type sendmail > /dev/null 2>&1 ; then
echo -e "Subject:$subject\r\n$MAIL_ALERT_MSG" | $(type -p sendmail) $DESTINATION_MAILS
cmd="echo -e \"Subject:$subject\r\n$MAIL_ALERT_MSG\" | $(type -p sendmail) $DESTINATION_MAILS"
eval $cmd
if [ $? != 0 ]; then
Logger "WARNING: Cannot send alert email via $(type -p sendmail) !!!" "WARN"
Logger "Cannot send alert email via $(type -p sendmail) !!!" "WARN"
else
Logger "Sent alert mail using sendmail command without attachment." "NOTICE"
return 0
@ -322,7 +244,7 @@ function SendAlert {
fi
$(type -p sendemail) -f $SENDER_MAIL -t $DESTINATION_MAILS -u "$subject" -m "$MAIL_ALERT_MSG" -s $SMTP_SERVER $SMTP_OPTIONS > /dev/null 2>&1
if [ $? != 0 ]; then
Logger "WARNING: Cannot send alert email via $(type -p sendemail) !!!" "WARN"
Logger "Cannot send alert email via $(type -p sendemail) !!!" "WARN"
else
Logger "Sent alert mail using sendemail command without attachment." "NOTICE"
return 0
@ -330,7 +252,7 @@ function SendAlert {
fi
# If function has not returned 0 yet, assume it's critical that no alert can be sent
Logger "/!\ CRITICAL: Cannot send alert (neither mutt, mail, sendmail nor sendemail found)." "ERROR" # Is not marked critical because execution must continue
Logger "Cannot send alert (neither mutt, mail, sendmail nor sendemail found)." "ERROR" # Is not marked critical because execution must continue
# Delete tmp log file
if [ -f "$ALERT_LOG_FILE" ]; then
@ -338,6 +260,97 @@ function SendAlert {
fi
}
#### MINIMAL-FUNCTION-SET END ####
function TrapError {
local job="$0"
local line="$1"
local code="${2:-1}"
if [ $_SILENT -eq 0 ]; then
echo -e " /!\ ERROR in ${job}: Near line ${line}, exit code ${code}"
fi
}
function Spinner {
if [ $_SILENT -eq 1 ]; then
return 0
fi
case $toggle
in
1)
echo -n " \ "
echo -ne "\r"
toggle="2"
;;
2)
echo -n " | "
echo -ne "\r"
toggle="3"
;;
3)
echo -n " / "
echo -ne "\r"
toggle="4"
;;
*)
echo -n " - "
echo -ne "\r"
toggle="1"
;;
esac
}
function SedStripQuotes {
echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g")
}
function StripSingleQuotes {
local string="${1}"
string="${string/#\'/}" # Remove singlequote if it begins string
string="${string/%\'/}" # Remove singlequote if it ends string
echo "$string"
}
function StripDoubleQuotes {
local string="${1}"
string="${string/#\"/}"
string="${string/%\"/}"
echo "$string"
}
function StripQuotes {
local string="${1}"
echo "$(StripSingleQuotes $(StripDoubleQuotes $string))"
}
function EscapeSpaces {
local string="${1}" # String on which spaces will be escaped
echo "${string// /\ }"
}
function IsNumeric {
eval "local value=\"${1}\"" # Needed so variable variables can be processed
local re="^-?[0-9]+([.][0-9]+)?$"
if [[ $value =~ $re ]]; then
echo 1
else
echo 0
fi
}
function CleanUp {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
if [ "$_DEBUG" != "yes" ]; then
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID"
fi
}
function LoadConfigFile {
local config_file="${1}"
__CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
@ -480,13 +493,13 @@ function WaitForTaskCompletion {
fi
if [ $exec_time -gt $soft_max_time ]; then
if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then
Logger "Max soft execution time exceeded for task." "WARN"
Logger "Max soft execution time exceeded for task [$caller_name]." "WARN"
soft_alert=1
SendAlert
fi
if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then
Logger "Max hard execution time exceeded for task. Stopping task execution." "ERROR"
Logger "Max hard execution time exceeded for task [$caller_name]. Stopping task execution." "ERROR"
kill -s SIGTERM $pid
if [ $? == 0 ]; then
Logger "Task stopped succesfully" "NOTICE"

228
osync.sh
Unescape Escape View File

@ -8,10 +8,10 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(L) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.1-dev
PROGRAM_BUILD=2016030101
PROGRAM_BUILD=2016030801
IS_STABLE=no
## FUNC_BUILD=2016021803
## FUNC_BUILD=2016030801
## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
@ -40,6 +40,8 @@ else
_VERBOSE=1
fi
#### MINIMAL-FUNCTION-SET BEGIN ####
SCRIPT_PID=$$
LOCAL_USER=$(whoami)
@ -105,15 +107,15 @@ function Logger {
# </OSYNC SPECIFIC>
if [ "$level" == "CRITICAL" ]; then
_Logger "$prefix\e[41m$value\e[0m" "$prefix$value"
_Logger "$prefix\e[41m$value\e[0m" "$prefix$level:$value"
ERROR_ALERT=1
return
elif [ "$level" == "ERROR" ]; then
_Logger "$prefix\e[91m$value\e[0m" "$prefix$value"
_Logger "$prefix\e[91m$value\e[0m" "$prefix$level:$value"
ERROR_ALERT=1
return
elif [ "$level" == "WARN" ]; then
_Logger "$prefix\e[93m$value\e[0m" "$prefix$value"
_Logger "$prefix\e[93m$value\e[0m" "$prefix$level:$value"
WARN_ALERT=1
return
elif [ "$level" == "NOTICE" ]; then
@ -145,94 +147,7 @@ function KillChilds {
if [ "$self" == true ]; then
kill -s SIGTERM "$pid" || (sleep 30 && kill -9 "$pid" &)
fi
}
function TrapError {
local job="$0"
local line="$1"
local code="${2:-1}"
if [ $_SILENT -eq 0 ]; then
echo -e " /!\ ERROR in ${job}: Near line ${line}, exit code ${code}"
fi
}
function Spinner {
if [ $_SILENT -eq 1 ]; then
return 0
fi
case $toggle
in
1)
echo -n " \ "
echo -ne "\r"
toggle="2"
;;
2)
echo -n " | "
echo -ne "\r"
toggle="3"
;;
3)
echo -n " / "
echo -ne "\r"
toggle="4"
;;
*)
echo -n " - "
echo -ne "\r"
toggle="1"
;;
esac
}
function SedStripQuotes {
echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g")
}
function StripSingleQuotes {
local string="${1}"
string="${string/#\'/}" # Remove singlequote if it begins string
string="${string/%\'/}" # Remove singlequote if it ends string
echo "$string"
}
function StripDoubleQuotes {
local string="${1}"
string="${string/#\"/}"
string="${string/%\"/}"
echo "$string"
}
function StripQuotes {
local string="${1}"
echo "$(StripSingleQuotes $(StripDoubleQuotes $string))"
}
function EscapeSpaces {
local string="${1}" # String on which spaces will be escaped
echo "${string// /\ }"
}
function IsNumeric {
eval "local value=\"${1}\"" # Needed so variable variables can be processed
local re="^-?[0-9]+([.][0-9]+)?$"
if [[ $value =~ $re ]]; then
echo 1
else
echo 0
fi
}
function CleanUp {
if [ "$_DEBUG" != "yes" ]; then
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID"
fi
# sleep 30 needs to wait before killing itself
}
function SendAlert {
@ -240,6 +155,10 @@ function SendAlert {
local mail_no_attachment=
local attachment_command=
if [ "$DESTINATION_MAILS" == "" ]; then
return 0
fi
if [ "$_DEBUG" == "yes" ]; then
Logger "Debug mode, no warning email will be sent." "NOTICE"
return 0
@ -272,9 +191,10 @@ function SendAlert {
attachment_command="-a $ALERT_LOG_FILE"
fi
if type mutt > /dev/null 2>&1 ; then
echo "$MAIL_ALERT_MSG" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS $attachment_command
cmd="echo \"$MAIL_ALERT_MSG\" | $(type -p mutt) -x -s \"$subject\" $DESTINATION_MAILS $attachment_command"
eval $cmd
if [ $? != 0 ]; then
Logger "WARNING: Cannot send alert email via $(type -p mutt) !!!" "WARN"
Logger "Cannot send alert email via $(type -p mutt) !!!" "WARN"
else
Logger "Sent alert mail using mutt." "NOTICE"
return 0
@ -289,12 +209,14 @@ function SendAlert {
else
attachment_command=""
fi
echo "$MAIL_ALERT_MSG" | $(type -p mail) $attachment_command -s "$subject" $DESTINATION_MAILS
cmd="echo \"$MAIL_ALERT_MSG\" | $(type -p mail) $attachment_command -s \"$subject\" $DESTINATION_MAILS"
eval $cmd
if [ $? != 0 ]; then
Logger "WARNING: Cannot send alert email via $(type -p mail) with attachments !!!" "WARN"
echo "$MAIL_ALERT_MSG" | $(type -p mail) -s "$subject" $DESTINATION_MAILS
Logger "Cannot send alert email via $(type -p mail) with attachments !!!" "WARN"
cmd="echo \"$MAIL_ALERT_MSG\" | $(type -p mail) -s \"$subject\" $DESTINATION_MAILS"
eval $cmd
if [ $? != 0 ]; then
Logger "WARNING: Cannot send alert email via $(type -p mail) without attachments !!!" "WARN"
Logger "Cannot send alert email via $(type -p mail) without attachments !!!" "WARN"
else
Logger "Sent alert mail using mail command without attachment." "NOTICE"
return 0
@ -306,9 +228,10 @@ function SendAlert {
fi
if type sendmail > /dev/null 2>&1 ; then
echo -e "Subject:$subject\r\n$MAIL_ALERT_MSG" | $(type -p sendmail) $DESTINATION_MAILS
cmd="echo -e \"Subject:$subject\r\n$MAIL_ALERT_MSG\" | $(type -p sendmail) $DESTINATION_MAILS"
eval $cmd
if [ $? != 0 ]; then
Logger "WARNING: Cannot send alert email via $(type -p sendmail) !!!" "WARN"
Logger "Cannot send alert email via $(type -p sendmail) !!!" "WARN"
else
Logger "Sent alert mail using sendmail command without attachment." "NOTICE"
return 0
@ -323,7 +246,7 @@ function SendAlert {
fi
$(type -p sendemail) -f $SENDER_MAIL -t $DESTINATION_MAILS -u "$subject" -m "$MAIL_ALERT_MSG" -s $SMTP_SERVER $SMTP_OPTIONS > /dev/null 2>&1
if [ $? != 0 ]; then
Logger "WARNING: Cannot send alert email via $(type -p sendemail) !!!" "WARN"
Logger "Cannot send alert email via $(type -p sendemail) !!!" "WARN"
else
Logger "Sent alert mail using sendemail command without attachment." "NOTICE"
return 0
@ -331,7 +254,7 @@ function SendAlert {
fi
# If function has not returned 0 yet, assume it's critical that no alert can be sent
Logger "/!\ CRITICAL: Cannot send alert (neither mutt, mail, sendmail nor sendemail found)." "ERROR" # Is not marked critical because execution must continue
Logger "Cannot send alert (neither mutt, mail, sendmail nor sendemail found)." "ERROR" # Is not marked critical because execution must continue
# Delete tmp log file
if [ -f "$ALERT_LOG_FILE" ]; then
@ -339,6 +262,96 @@ function SendAlert {
fi
}
#### MINIMAL-FUNCTION-SET END ####
function TrapError {
local job="$0"
local line="$1"
local code="${2:-1}"
if [ $_SILENT -eq 0 ]; then
echo -e " /!\ ERROR in ${job}: Near line ${line}, exit code ${code}"
fi
}
function Spinner {
if [ $_SILENT -eq 1 ]; then
return 0
fi
case $toggle
in
1)
echo -n " \ "
echo -ne "\r"
toggle="2"
;;
2)
echo -n " | "
echo -ne "\r"
toggle="3"
;;
3)
echo -n " / "
echo -ne "\r"
toggle="4"
;;
*)
echo -n " - "
echo -ne "\r"
toggle="1"
;;
esac
}
function SedStripQuotes {
echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g")
}
function StripSingleQuotes {
local string="${1}"
string="${string/#\'/}" # Remove singlequote if it begins string
string="${string/%\'/}" # Remove singlequote if it ends string
echo "$string"
}
function StripDoubleQuotes {
local string="${1}"
string="${string/#\"/}"
string="${string/%\"/}"
echo "$string"
}
function StripQuotes {
local string="${1}"
echo "$(StripSingleQuotes $(StripDoubleQuotes $string))"
}
function EscapeSpaces {
local string="${1}" # String on which spaces will be escaped
echo "${string// /\ }"
}
function IsNumeric {
eval "local value=\"${1}\"" # Needed so variable variables can be processed
local re="^-?[0-9]+([.][0-9]+)?$"
if [[ $value =~ $re ]]; then
echo 1
else
echo 0
fi
}
function CleanUp {
if [ "$_DEBUG" != "yes" ]; then
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID"
fi
}
function LoadConfigFile {
local config_file="${1}"
@ -476,13 +489,13 @@ function WaitForTaskCompletion {
fi
if [ $exec_time -gt $soft_max_time ]; then
if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then
Logger "Max soft execution time exceeded for task." "WARN"
Logger "Max soft execution time exceeded for task [$caller_name]." "WARN"
soft_alert=1
SendAlert
fi
if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then
Logger "Max hard execution time exceeded for task. Stopping task execution." "ERROR"
Logger "Max hard execution time exceeded for task [$caller_name]. Stopping task execution." "ERROR"
kill -s SIGTERM $pid
if [ $? == 0 ]; then
Logger "Task stopped succesfully" "NOTICE"
@ -1389,8 +1402,9 @@ function tree_list {
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replica_path/\" | grep \"^-\|^d\" | awk '{\$1=\$2=\$3=\$4=\"\" ;print}' | awk '{\$1=\$1 ;print}' | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.$replica_type.$SCRIPT_PID\" &"
fi
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
## Redirect commands stderr here to get rsync stderr output in logfile
eval "$rsync_cmd" 2>> "$LOG_FILE"
## Redirect commands stderr here to get rsync stderr output in logfile with eval "$rsync_cmd" 2>> "$LOG_FILE"
## When log writing fails, $! is empty and WaitForCompletion fails. Removing the 2>> log
eval "$rsync_cmd"
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
retval=$?
## Retval 24 = some files vanished while creating list

Write Preview
Loading…
Cancel
Save