Rebuilt targets

pull/163/head
deajan 6 years ago
parent 25070032c4
commit bbfa4c9d3b

@ -2,12 +2,14 @@
#TODO treeList, deleteList, _getFileCtimeMtime, conflictList should be called without having statedir informed. Just give the full path ?
#Check dryruns with nosuffix mode for timestampList
#WIP: currently TRAVIS_RUN debug lines in n_osync and run_tests for conflictLog
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2018 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.3.0-beta1
PROGRAM_BUILD=2018100105
PROGRAM_BUILD=2018100106
IS_STABLE=no
##### Execution order #__WITH_PARANOIA_DEBUG
@ -40,22 +42,10 @@ IS_STABLE=no
# UnlockReplicas yes #__WITH_PARANOIA_DEBUG
# CleanUp no #__WITH_PARANOIA_DEBUG
_OFUNCTIONS_VERSION=2.3.0-RC1
_OFUNCTIONS_BUILD=2018100103
_OFUNCTIONS_VERSION=2.3.0-RC2
_OFUNCTIONS_BUILD=2018100204
_OFUNCTIONS_BOOTSTRAP=true
## To use in a program, define the following variables:
## PROGRAM=program-name
## INSTANCE_ID=program-instance-name
## _DEBUG=yes/no
## _LOGGER_SILENT=true/false
## _LOGGER_VERBOSE=true/false
## _LOGGER_ERR_ONLY=true/false
## _LOGGER_PREFIX="date"/"time"/""
## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel
## When called from subprocesses, variable of main process cannot be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID.$TSTAMP
if ! type "$BASH" > /dev/null; then
echo "Please run this script only with bash shell. Tested on bash >= 3.2"
exit 127
@ -131,7 +121,6 @@ else
RUN_DIR=.
fi
#### PoorMansRandomGenerator SUBSET ####
# Get a random number on Windows BusyBox alike, also works on most Unixes
function PoorMansRandomGenerator {
local digits="${1}" # The number of digits to generate
@ -146,7 +135,6 @@ function PoorMansRandomGenerator {
fi
done
}
#### PoorMansRandomGenerator SUBSET END ####
# Initial TSTMAP value before function declaration
TSTAMP=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 4)
@ -159,13 +147,6 @@ set -o pipefail
set -o errtrace
function Dummy {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
sleep $SLEEP_TIME
}
# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array
# usage: joinString separaratorChar Array
function joinString {
@ -411,6 +392,35 @@ function KillAllChilds {
return $errorcount
}
function CleanUp {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
if [ "$_DEBUG" != "yes" ]; then
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP"
# Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements)
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp"
fi
}
function GenericTrapQuit {
local exitcode=0
# Get ERROR / WARN alert flags from subprocesses that call Logger
if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then
WARN_ALERT=true
exitcode=2
fi
if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then
ERROR_ALERT=true
exitcode=1
fi
CleanUp
exit $exitcode
}
# osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending
function SendAlert {
local runAlert="${1:-false}" # Specifies if current message is sent while running or at the end of a run
@ -1216,16 +1226,6 @@ function ExecTasks {
fi
}
function CleanUp {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
if [ "$_DEBUG" != "yes" ]; then
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP"
# Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements)
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp"
fi
}
# Usage: var=$(StripSingleQuotes "$var")
function StripSingleQuotes {
local string="${1}"
@ -1264,27 +1264,37 @@ function EscapeDoubleQuotes {
echo "${value//\"/\\\"}"
}
function IsNumericExpand {
eval "local value=\"${1}\"" # Needed eval so variable variables can be processed
if [[ $value =~ ^-?[0-9]+([.][0-9]+)?$ ]]; then
echo 1
else
echo 0
fi
}
# Usage [ $(IsNumeric $var) -eq 1 ]
function IsNumeric {
local value="${1}"
if [[ $value =~ ^[0-9]+([.][0-9]+)?$ ]]; then
echo 1
if type expr > /dev/null 2>&1; then
expr "$value" : "^[-+]\?[0-9]*\.\?[0-9]\+$" > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo 1
else
echo 0
fi
else
echo 0
if [[ $value =~ ^[-+]?[0-9]+([.][0-9]+)?$ ]]; then
echo 1
else
echo 0
fi
fi
}
function IsNumericExpand {
eval "local value=\"${1}\"" # Needed eval so variable variables can be processed
echo $(IsNumeric "$value")
# if [[ $value =~ ^-?[0-9]+([.][0-9]+)?$ ]]; then
# echo 1
#else
# echo 0
#fi
}
# Function is busybox compatible since busybox ash does not understand direct regex, we use expr
function IsInteger {
local value="${1}"
@ -1874,9 +1884,9 @@ function RsyncPatternsAdd {
rest="${rest#*$PATH_SEPARATOR_CHAR}"
fi
if [ "$RSYNC_PATTERNS" == "" ]; then
RSYNC_PATTERNS="--"$patternType"=\"$str\""
RSYNC_PATTERNS="--$patternType=\"$str\""
else
RSYNC_PATTERNS="$RSYNC_PATTERNS --"$patternType"=\"$str\""
RSYNC_PATTERNS="$RSYNC_PATTERNS --$patternType=\"$str\""
fi
done
set +f
@ -1893,7 +1903,7 @@ function RsyncPatternsFromAdd {
fi
if [ -e "$patternFrom" ]; then
RSYNC_PATTERNS="$RSYNC_PATTERNS --"$patternType"-from=\"$patternFrom\""
RSYNC_PATTERNS="$RSYNC_PATTERNS --$patternType-from=\"$patternFrom\""
fi
}
@ -1983,14 +1993,10 @@ function PostInit {
# Define remote commands
if [ -f "$SSH_RSA_PRIVATE_KEY" ]; then
SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
#WIP
#SSH_REVERSE_CMD="$(type -p ssh) $SSH_COMP -q -i $INITIATOR_SSH_RSA_PRIVATE_KEY $SSH_OPTS $INITIATOR_REMOTE_USER@$INITIATOR_REMOTE_HOST -p $INITIATOR_REMOTE_PORT"
SCP_CMD="$(type -p scp) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT"
elif [ -f "$SSH_PASSWORD_FILE" ]; then
SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
#WIP
#SSH_REVERSE_CMD="$(type -p sshpass) -f $INITIATOR_SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS $INITIATOR_REMOTE_USER@$INITIATOR_REMOTE_HOST -p $INITIATOR_REMOTE_PORT"
SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -q -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS -p $REMOTE_PORT"
else
@ -2221,7 +2227,7 @@ function VerComp () {
return 1
fi
if [[ $1 == $2 ]]
if [[ "$1" == "$2" ]]
then
echo 0
return
@ -3580,6 +3586,15 @@ function conflictList {
join -j 1 -t ';' -o 1.1,1.2,1.3,2.2,2.3 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${INITIATOR[$__type]}.$SCRIPT_PID.$TSTAMP" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${TARGET[$__type]}.$SCRIPT_PID.$TSTAMP" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.compare.$SCRIPT_PID.$TSTAMP"
retval=$?
#WIP
if [ $TRAVIS_RUN == true ]; then
echo "conflictList debug retval=$retval"
cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${INITIATOR[$__type]}.$SCRIPT_PID.$TSTAMP"
cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${TARGET[$__type]}.$SCRIPT_PID.$TSTAMP"
cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.compare.$SCRIPT_PID.$TSTAMP"
fi
if [ $retval -ne 0 ]; then
Logger "Cannot create conflict list file." "ERROR"
return $retval
@ -5169,6 +5184,12 @@ function LogConflicts {
local subject
local body
#WIP
if [ $TRAVIS_RUN == true ]; then
cat "$RUN_DIR/$PROGRAM.conflictList.compare.$SCRIPT_PID.$TSTAMP"
fi
# We keep this in a separate if check because of the subshell used for Logger with _LOGGER_PREFIX
if [ -f "$RUN_DIR/$PROGRAM.conflictList.compare.$SCRIPT_PID.$TSTAMP" ]; then
Logger "File conflicts: INITIATOR << >> TARGET" "ALWAYS"
@ -5176,6 +5197,7 @@ function LogConflicts {
(
_LOGGER_PREFIX=""
if [ -f "$RUN_DIR/$PROGRAM.conflictList.compare.$SCRIPT_PID.$TSTAMP" ]; then
echo "" > "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}"
while read -r line; do

@ -10,83 +10,113 @@ PROGRAM_BINARY=$PROGRAM".sh"
PROGRAM_BATCH=$PROGRAM"-batch.sh"
SSH_FILTER="ssh_filter.sh"
SCRIPT_BUILD=2018090301
SCRIPT_BUILD=2018100201
INSTANCE_ID="installer-$SCRIPT_BUILD"
## osync / obackup / pmocr / zsnap install script
## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11
## Please adapt this to fit your distro needs
_OFUNCTIONS_VERSION=2.3.0-RC2
_OFUNCTIONS_BUILD=2018100204
_OFUNCTIONS_BOOTSTRAP=true
# Get current install.sh path from http://stackoverflow.com/a/246128/2635443
SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
if ! type "$BASH" > /dev/null; then
echo "Please run this script only with bash shell. Tested on bash >= 3.2"
exit 127
fi
## Correct output of sort command (language agnostic sorting)
export LC_ALL=C
## Default umask for file creation
umask 0077
# Standard alert mail body
MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
# Environment variables that can be overriden by programs
_DRYRUN=false
_LOGGER_SILENT=false
_STATS=1
ACTION="install"
FAKEROOT=""
_LOGGER_VERBOSE=false
_LOGGER_ERR_ONLY=false
_LOGGER_PREFIX="date"
if [ "$KEEP_LOGGING" == "" ]; then
KEEP_LOGGING=1801
fi
function GetCommandlineArguments {
for i in "$@"; do
case $i in
--prefix=*)
FAKEROOT="${i##*=}"
;;
--silent)
_LOGGER_SILENT=true
;;
--no-stats)
_STATS=0
;;
--remove)
ACTION="uninstall"
;;
--help|-h|-?)
Usage
;;
*)
Logger "Unknown option '$i'" "SIMPLE"
Usage
exit
;;
esac
done
}
# Initial error status, logging 'WARN', 'ERROR' or 'CRITICAL' will enable alerts flags
ERROR_ALERT=false
WARN_ALERT=false
GetCommandlineArguments "$@"
CONF_DIR=$FAKEROOT/etc/$PROGRAM
BIN_DIR="$FAKEROOT/usr/local/bin"
SERVICE_DIR_INIT=$FAKEROOT/etc/init.d
# Should be /usr/lib/systemd/system, but /lib/systemd/system exists on debian & rhel / fedora
SERVICE_DIR_SYSTEMD_SYSTEM=$FAKEROOT/lib/systemd/system
SERVICE_DIR_SYSTEMD_USER=$FAKEROOT/etc/systemd/user
SERVICE_DIR_OPENRC=$FAKEROOT/etc/init.d
## allow debugging from command line with _DEBUG=yes
if [ ! "$_DEBUG" == "yes" ]; then
_DEBUG=no
_LOGGER_VERBOSE=false
else
trap 'TrapError ${LINENO} $?' ERR
_LOGGER_VERBOSE=true
fi
if [ "$PROGRAM" == "osync" ]; then
SERVICE_NAME="osync-srv"
elif [ "$PROGRAM" == "pmocr" ]; then
SERVICE_NAME="pmocr-srv"
if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console
SLEEP_TIME=.05
fi
SERVICE_FILE_INIT="$SERVICE_NAME"
SERVICE_FILE_SYSTEMD_SYSTEM="$SERVICE_NAME@.service"
SERVICE_FILE_SYSTEMD_USER="$SERVICE_NAME@.service.user"
SERVICE_FILE_OPENRC="$SERVICE_NAME-openrc"
SCRIPT_PID=$$
## Generic code
LOCAL_USER=$(whoami)
LOCAL_HOST=$(hostname)
## Default log file
if [ -w "$FAKEROOT/var/log" ]; then
LOG_FILE="$FAKEROOT/var/log/$PROGRAM-install.log"
if [ "$PROGRAM" == "" ]; then
PROGRAM="ofunctions"
fi
## Default log file until config file is loaded
if [ -w /var/log ]; then
LOG_FILE="/var/log/$PROGRAM.log"
elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
LOG_FILE="$HOME/$PROGRAM-install.log"
LOG_FILE="$HOME/$PROGRAM.log"
elif [ -w . ]; then
LOG_FILE="./$PROGRAM.log"
else
LOG_FILE="./$PROGRAM-install.log"
LOG_FILE="/tmp/$PROGRAM.log"
fi
## Default directory where to store temporary run files
if [ -w /tmp ]; then
RUN_DIR=/tmp
elif [ -w /var/tmp ]; then
RUN_DIR=/var/tmp
else
RUN_DIR=.
fi
#### RemoteLogger SUBSET ####
# Get a random number on Windows BusyBox alike, also works on most Unixes
function PoorMansRandomGenerator {
local digits="${1}" # The number of digits to generate
local number
# Some read bytes can't be used, se we read twice the number of required bytes
dd if=/dev/urandom bs=$digits count=2 2> /dev/null | while read -r -n1 char; do
number=$number$(printf "%d" "'$char")
if [ ${#number} -ge $digits ]; then
echo ${number:0:$digits}
break;
fi
done
}
# Initial TSTMAP value before function declaration
TSTAMP=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 4)
# Default alert attachment filename
ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.$SCRIPT_PID.$TSTAMP.last.log"
# Set error exit code if a piped command fails
set -o pipefail
set -o errtrace
# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array
# usage: joinString separaratorChar Array
@ -170,17 +200,11 @@ function RemoteLogger {
_Logger "" "$prefix$value"
return
fi
elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG
if [ "$_PARANOIA_DEBUG" == "yes" ]; then #__WITH_PARANOIA_DEBUG
_Logger "" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG
return #__WITH_PARANOIA_DEBUG
fi #__WITH_PARANOIA_DEBUG
else
_Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true
_Logger "" "Value was: $prefix$value" true
fi
}
#### RemoteLogger SUBSET END ####
# General log function with log levels:
@ -249,11 +273,6 @@ function Logger {
_Logger "$prefix$value" "$prefix$value"
return
fi
elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG
if [ "$_PARANOIA_DEBUG" == "yes" ]; then #__WITH_PARANOIA_DEBUG
_Logger "$prefix$value" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG
return #__WITH_PARANOIA_DEBUG
fi #__WITH_PARANOIA_DEBUG
elif [ "$level" == "SIMPLE" ]; then
if [ "$_LOGGER_SILENT" == true ]; then
_Logger "$preix$value"
@ -266,6 +285,167 @@ function Logger {
_Logger "Value was: $prefix$value" "Value was: $prefix$value" true
fi
}
# Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X
function KillChilds {
local pid="${1}" # Parent pid to kill childs
local self="${2:-false}" # Should parent be killed too ?
# Paranoid checks, we can safely assume that $pid should not be 0 nor 1
if [ $(IsInteger "$pid") -eq 0 ] || [ "$pid" == "" ] || [ "$pid" == "0" ] || [ "$pid" == "1" ]; then
Logger "Bogus pid given [$pid]." "CRITICAL"
return 1
fi
if kill -0 "$pid" > /dev/null 2>&1; then
if children="$(pgrep -P "$pid")"; then
if [[ "$pid" == *"$children"* ]]; then
Logger "Bogus pgrep implementation." "CRITICAL"
children="${children/$pid/}"
fi
for child in $children; do
KillChilds "$child" true
done
fi
fi
# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
if [ "$self" == true ]; then
# We need to check for pid again because it may have disappeared after recursive function call
if kill -0 "$pid" > /dev/null 2>&1; then
kill -s TERM "$pid"
Logger "Sent SIGTERM to process [$pid]." "DEBUG"
if [ $? != 0 ]; then
sleep 15
Logger "Sending SIGTERM to process [$pid] failed." "DEBUG"
kill -9 "$pid"
if [ $? != 0 ]; then
Logger "Sending SIGKILL to process [$pid] failed." "DEBUG"
return 1
fi # Simplify the return 0 logic here
else
return 0
fi
else
return 0
fi
else
return 0
fi
}
function KillAllChilds {
local pids="${1}" # List of parent pids to kill separated by semi-colon
local self="${2:-false}" # Should parent be killed too ?
local errorcount=0
IFS=';' read -a pidsArray <<< "$pids"
for pid in "${pidsArray[@]}"; do
KillChilds $pid $self
if [ $? != 0 ]; then
errorcount=$((errorcount+1))
fi
done
return $errorcount
}
function CleanUp {
if [ "$_DEBUG" != "yes" ]; then
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP"
# Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements)
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp"
fi
}
function GenericTrapQuit {
local exitcode=0
# Get ERROR / WARN alert flags from subprocesses that call Logger
if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then
WARN_ALERT=true
exitcode=2
fi
if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then
ERROR_ALERT=true
exitcode=1
fi
CleanUp
exit $exitcode
}
# Get current install.sh path from http://stackoverflow.com/a/246128/2635443
SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
_LOGGER_SILENT=false
_STATS=1
ACTION="install"
FAKEROOT=""
function GetCommandlineArguments {
for i in "$@"; do
case $i in
--prefix=*)
FAKEROOT="${i##*=}"
;;
--silent)
_LOGGER_SILENT=true
;;
--no-stats)
_STATS=0
;;
--remove)
ACTION="uninstall"
;;
--help|-h|-?)
Usage
;;
*)
Logger "Unknown option '$i'" "SIMPLE"
Usage
exit
;;
esac
done
}
GetCommandlineArguments "$@"
CONF_DIR=$FAKEROOT/etc/$PROGRAM
BIN_DIR="$FAKEROOT/usr/local/bin"
SERVICE_DIR_INIT=$FAKEROOT/etc/init.d
# Should be /usr/lib/systemd/system, but /lib/systemd/system exists on debian & rhel / fedora
SERVICE_DIR_SYSTEMD_SYSTEM=$FAKEROOT/lib/systemd/system
SERVICE_DIR_SYSTEMD_USER=$FAKEROOT/etc/systemd/user
SERVICE_DIR_OPENRC=$FAKEROOT/etc/init.d
if [ "$PROGRAM" == "osync" ]; then
SERVICE_NAME="osync-srv"
elif [ "$PROGRAM" == "pmocr" ]; then
SERVICE_NAME="pmocr-srv"
fi
SERVICE_FILE_INIT="$SERVICE_NAME"
SERVICE_FILE_SYSTEMD_SYSTEM="$SERVICE_NAME@.service"
SERVICE_FILE_SYSTEMD_USER="$SERVICE_NAME@.service.user"
SERVICE_FILE_OPENRC="$SERVICE_NAME-openrc"
## Generic code
## Default log file
if [ -w "$FAKEROOT/var/log" ]; then
LOG_FILE="$FAKEROOT/var/log/$PROGRAM-install.log"
elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
LOG_FILE="$HOME/$PROGRAM-install.log"
else
LOG_FILE="./$PROGRAM-install.log"
fi
## Modified version of https://gist.github.com/cdown/1163649
function UrlEncode {
local length="${#1}"
@ -658,19 +838,27 @@ function Usage {
exit 127
}
function TrapQuit {
local exitcode=0
# Get ERROR / WARN alert flags from subprocesses that call Logger
if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then
WARN_ALERT=true
exitcode=2
fi
if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then
ERROR_ALERT=true
exitcode=1
fi
CleanUp
exit $exitcode
}
############################## Script entry point
if [ "$LOGFILE" == "" ]; then
if [ -w /var/log ]; then
LOG_FILE="/var/log/$PROGRAM.$INSTANCE_ID.log"
elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
LOG_FILE="$HOME/$PROGRAM.$INSTANCE_ID.log"
else
LOG_FILE="./$PROGRAM.$INSTANCE_ID.log"
fi
else
LOG_FILE="$LOGFILE"
fi
trap TrapQuit TERM EXIT HUP QUIT
if [ ! -w "$(dirname $LOG_FILE)" ]; then
echo "Cannot write to log [$(dirname $LOG_FILE)]."
else

@ -1,9 +1,9 @@
#!/usr/bin/env bash
SUBPROGRAM=osync
PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones
AUTHOR="(L) 2013-2017 by Orsiris de Jong"
AUTHOR="(L) 2013-2018 by Orsiris de Jong"
CONTACT="http://www.netpower.fr - ozy@netpower.fr"
PROGRAM_BUILD=2016120401
PROGRAM_BUILD=2018100101
## Runs an osync /obackup instance for every conf file found
## If an instance fails, run it again if time permits
@ -26,34 +26,185 @@ else
LOG_FILE=./$SUBPROGRAM-batch.log
fi
## Default directory where to store temporary run files
if [ -w /tmp ]; then
RUN_DIR=/tmp
elif [ -w /var/tmp ]; then
RUN_DIR=/var/tmp
else
RUN_DIR=.
fi
# No need to edit under this line ##############################################################
function _logger {
local value="${1}" # What to log
echo -e "$value" >> "$LOG_FILE"
#### RemoteLogger SUBSET ####
# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array
# usage: joinString separaratorChar Array
function joinString {
local IFS="$1"; shift; echo "$*";
}
# Sub function of Logger
function _Logger {
local logValue="${1}" # Log to file
local stdValue="${2}" # Log to screeen
local toStdErr="${3:-false}" # Log to stderr instead of stdout
if [ "$logValue" != "" ]; then
echo -e "$logValue" >> "$LOG_FILE"
# Build current log file for alerts if we have a sufficient environment
if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then
echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP"
fi
fi
if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then
if [ $toStdErr == true ]; then
# Force stderr color in subshell
(>&2 echo -e "$stdValue")
else
echo -e "$stdValue"
fi
fi
}
# Remote logger similar to below Logger, without log to file and alert flags
function RemoteLogger {
local value="${1}" # Sentence to log (in double quotes)
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="R $(date) - "
else
prefix=""
fi
if [ "$level" == "CRITICAL" ]; then
_Logger "" "$prefix\e[1;33;41m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi
return
elif [ "$level" == "ERROR" ]; then
_Logger "" "$prefix\e[31m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi
return
elif [ "$level" == "WARN" ]; then
_Logger "" "$prefix\e[33m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi
return
elif [ "$level" == "NOTICE" ]; then
if [ $_LOGGER_ERR_ONLY != true ]; then
_Logger "" "$prefix$value"
fi
return
elif [ "$level" == "VERBOSE" ]; then
if [ $_LOGGER_VERBOSE == true ]; then
_Logger "" "$prefix$value"
fi
return
elif [ "$level" == "ALWAYS" ]; then
_Logger "" "$prefix$value"
return
elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then
_Logger "" "$prefix$value"
return
fi
else
_Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true
_Logger "" "Value was: $prefix$value" true
fi
}
#### RemoteLogger SUBSET END ####
# General log function with log levels:
# Environment variables
# _LOGGER_SILENT: Disables any output to stdout & stderr
# _LOGGER_ERR_ONLY: Disables any output to stdout except for ALWAYS loglevel
# _LOGGER_VERBOSE: Allows VERBOSE loglevel messages to be sent to stdout
# Loglevels
# Except for VERBOSE, all loglevels are ALWAYS sent to log file
# CRITICAL, ERROR, WARN sent to stderr, color depending on level, level also logged
# NOTICE sent to stdout
# VERBOSE sent to stdout if _LOGGER_VERBOSE = true
# ALWAYS is sent to stdout unless _LOGGER_SILENT = true
# DEBUG & PARANOIA_DEBUG are only sent to stdout if _DEBUG=yes
# SIMPLE is a wrapper for QuickLogger that does not use advanced functionality
function Logger {
local value="${1}" # What to log
local level="${2}" # Log level: DEBUG, NOTICE, WARN, ERROR, CRITIAL
local value="${1}" # Sentence to log (in double quotes)
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
prefix="$(date) - "
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="$(date '+%Y-%m-%d %H:%M:%S') - "
else
prefix=""
fi
## Obfuscate _REMOTE_TOKEN in logs (for ssh_filter usage only in osync and obackup)
value="${value/env _REMOTE_TOKEN=$_REMOTE_TOKEN/__(o_O)__}"
value="${value/env _REMOTE_TOKEN=\$_REMOTE_TOKEN/__(o_O)__}"
if [ "$level" == "CRITICAL" ]; then
_logger "$prefix\e[41m$value\e[0m"
_Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true
ERROR_ALERT=true
# ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. Need to keep this flag.
echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
return
elif [ "$level" == "ERROR" ]; then
_logger "$prefix\e[91m$value\e[0m"
_Logger "$prefix($level):$value" "$prefix\e[91m$value\e[0m" true
ERROR_ALERT=true
echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
return
elif [ "$level" == "WARN" ]; then
_logger "$prefix\e[93m$value\e[0m"
_Logger "$prefix($level):$value" "$prefix\e[33m$value\e[0m" true
WARN_ALERT=true
echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID.$TSTAMP"
return
elif [ "$level" == "NOTICE" ]; then
_logger "$prefix$value"
if [ "$_LOGGER_ERR_ONLY" != true ]; then
_Logger "$prefix$value" "$prefix$value"
fi
return
elif [ "$level" == "VERBOSE" ]; then
if [ $_LOGGER_VERBOSE == true ]; then
_Logger "$prefix($level):$value" "$prefix$value"
fi
return
elif [ "$level" == "ALWAYS" ]; then
_Logger "$prefix$value" "$prefix$value"
return
elif [ "$level" == "DEBUG" ]; then
if [ "$DEBUG" == "yes" ]; then
_logger "$prefix$value"
if [ "$_DEBUG" == "yes" ]; then
_Logger "$prefix$value" "$prefix$value"
return
fi
elif [ "$level" == "SIMPLE" ]; then
if [ "$_LOGGER_SILENT" == true ]; then
_Logger "$preix$value"
else
_Logger "$preix$value" "$prefix$value"
fi
return
else
_logger "\e[41mLogger function called without proper loglevel.\e[0m"
_logger "$prefix$value"
_Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true
_Logger "Value was: $prefix$value" "Value was: $prefix$value" true
fi
}

@ -2,17 +2,19 @@
#TODO treeList, deleteList, _getFileCtimeMtime, conflictList should be called without having statedir informed. Just give the full path ?
#Check dryruns with nosuffix mode for timestampList
#WIP: currently TRAVIS_RUN debug lines in n_osync and run_tests for conflictLog
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2018 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.3.0-beta1
PROGRAM_BUILD=2018100105
PROGRAM_BUILD=2018100106
IS_STABLE=no
_OFUNCTIONS_VERSION=2.3.0-RC1
_OFUNCTIONS_BUILD=2018100103
_OFUNCTIONS_BUILD=2018100105
_OFUNCTIONS_BOOTSTRAP=true
## To use in a program, define the following variables:
@ -98,7 +100,6 @@ else
RUN_DIR=.
fi
#### PoorMansRandomGenerator SUBSET ####
# Get a random number on Windows BusyBox alike, also works on most Unixes
function PoorMansRandomGenerator {
local digits="${1}" # The number of digits to generate
@ -113,7 +114,6 @@ function PoorMansRandomGenerator {
fi
done
}
#### PoorMansRandomGenerator SUBSET END ####
# Initial TSTMAP value before function declaration
TSTAMP=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 4)
@ -1729,9 +1729,9 @@ function RsyncPatternsAdd {
rest="${rest#*$PATH_SEPARATOR_CHAR}"
fi
if [ "$RSYNC_PATTERNS" == "" ]; then
RSYNC_PATTERNS="--"$patternType"=\"$str\""
RSYNC_PATTERNS="--$patternType=\"$str\""
else
RSYNC_PATTERNS="$RSYNC_PATTERNS --"$patternType"=\"$str\""
RSYNC_PATTERNS="$RSYNC_PATTERNS --$patternType=\"$str\""
fi
done
set +f
@ -1747,7 +1747,7 @@ function RsyncPatternsFromAdd {
fi
if [ -e "$patternFrom" ]; then
RSYNC_PATTERNS="$RSYNC_PATTERNS --"$patternType"-from=\"$patternFrom\""
RSYNC_PATTERNS="$RSYNC_PATTERNS --$patternType-from=\"$patternFrom\""
fi
}
@ -1834,14 +1834,10 @@ function PostInit {
# Define remote commands
if [ -f "$SSH_RSA_PRIVATE_KEY" ]; then
SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
#WIP
#SSH_REVERSE_CMD="$(type -p ssh) $SSH_COMP -q -i $INITIATOR_SSH_RSA_PRIVATE_KEY $SSH_OPTS $INITIATOR_REMOTE_USER@$INITIATOR_REMOTE_HOST -p $INITIATOR_REMOTE_PORT"
SCP_CMD="$(type -p scp) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT"
elif [ -f "$SSH_PASSWORD_FILE" ]; then
SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
#WIP
#SSH_REVERSE_CMD="$(type -p sshpass) -f $INITIATOR_SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS $INITIATOR_REMOTE_USER@$INITIATOR_REMOTE_HOST -p $INITIATOR_REMOTE_PORT"
SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -q -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS -p $REMOTE_PORT"
else
@ -2070,7 +2066,7 @@ function VerComp () {
return 1
fi
if [[ $1 == $2 ]]
if [[ "$1" == "$2" ]]
then
echo 0
return
@ -3393,6 +3389,15 @@ function conflictList {
join -j 1 -t ';' -o 1.1,1.2,1.3,2.2,2.3 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${INITIATOR[$__type]}.$SCRIPT_PID.$TSTAMP" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${TARGET[$__type]}.$SCRIPT_PID.$TSTAMP" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.compare.$SCRIPT_PID.$TSTAMP"
retval=$?
#WIP
if [ $TRAVIS_RUN == true ]; then
echo "conflictList debug retval=$retval"
cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${INITIATOR[$__type]}.$SCRIPT_PID.$TSTAMP"
cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${TARGET[$__type]}.$SCRIPT_PID.$TSTAMP"
cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.compare.$SCRIPT_PID.$TSTAMP"
fi
if [ $retval -ne 0 ]; then
Logger "Cannot create conflict list file." "ERROR"
return $retval
@ -4947,6 +4952,12 @@ function LogConflicts {
local subject
local body
#WIP
if [ $TRAVIS_RUN == true ]; then
cat "$RUN_DIR/$PROGRAM.conflictList.compare.$SCRIPT_PID.$TSTAMP"
fi
# We keep this in a separate if check because of the subshell used for Logger with _LOGGER_PREFIX
if [ -f "$RUN_DIR/$PROGRAM.conflictList.compare.$SCRIPT_PID.$TSTAMP" ]; then
Logger "File conflicts: INITIATOR << >> TARGET" "ALWAYS"
@ -4954,6 +4965,7 @@ function LogConflicts {
(
_LOGGER_PREFIX=""
if [ -f "$RUN_DIR/$PROGRAM.conflictList.compare.$SCRIPT_PID.$TSTAMP" ]; then
echo "" > "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}"
while read -r line; do

Loading…
Cancel
Save