Initial upload of target helper service

pull/115/head
deajan 7 years ago
parent 29dc7a3e9e
commit 1375ab425b

@ -0,0 +1,455 @@
#!/usr/bin/env bash
PROGRAM="osync-target-helper" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2017 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.2.2-dev
PROGRAM_BUILD=2017061901
IS_STABLE=no
include #### OFUNCTIONS FULL SUBSET ####
# If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap
include #### _OFUNCTIONS_BOOTSTRAP SUBSET ####
[ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1
_LOGGER_PREFIX="time"
## Working directory. This directory exists in any replica and contains state files, backups, soft deleted files etc
OSYNC_DIR=".osync_workdir"
function TrapQuit {
local exitcode
# Get ERROR / WARN alert flags from subprocesses that call Logger
if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then
WARN_ALERT=true
fi
if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then
ERROR_ALERT=true
fi
if [ $ERROR_ALERT == true ]; then
Logger "$PROGRAM finished with errors." "ERROR"
if [ "$_DEBUG" != "yes" ]
then
SendAlert
else
Logger "Debug mode, no alert mail will be sent." "NOTICE"
fi
exitcode=1
elif [ $WARN_ALERT == true ]; then
Logger "$PROGRAM finished with warnings." "WARN"
if [ "$_DEBUG" != "yes" ]
then
SendAlert
else
Logger "Debug mode, no alert mail will be sent." "NOTICE"
fi
exitcode=2 # Warning exit code must not force daemon mode to quit
else
Logger "$PROGRAM finished." "ALWAYS"
exitcode=0
fi
CleanUp
KillChilds $SCRIPT_PID > /dev/null 2>&1
exit $exitcode
}
function CheckEnvironment {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
if ! type ssh > /dev/null 2>&1 ; then
Logger "ssh not present. Cannot start sync." "CRITICAL"
exit 1
fi
if [ "$SSH_PASSWORD_FILE" != "" ] && ! type sshpass > /dev/null 2>&1 ; then
Logger "sshpass not present. Cannot use password authentication." "CRITICAL"
exit 1
fi
}
# Only gets checked in config file mode where all values should be present
function CheckCurrentConfig {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
# Check all variables that should contain "yes" or "no"
declare -a yes_no_vars=(SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING)
for i in "${yes_no_vars[@]}"; do
test="if [ \"\$$i\" != \"yes\" ] && [ \"\$$i\" != \"no\" ]; then Logger \"Bogus $i value [\$$i] defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
eval "$test"
done
# Check all variables that should contain a numerical value >= 0
declare -a num_vars=(MIN_WAIT MAX_WAIT)
for i in "${num_vars[@]}"; do
test="if [ $(IsNumericExpand \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value [\$$i] defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
eval "$test"
done
}
# Gets checked in quicksync and config file mode
function CheckCurrentConfigAll {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
local tmp
if [ "$INSTANCE_ID" == "" ]; then
Logger "No INSTANCE_ID defined in config file." "CRITICAL"
exit 1
fi
if [ "$INITIATOR_SYNC_DIR" == "" ]; then
Logger "No INITIATOR_SYNC_DIR set in config file." "CRITICAL"
exit 1
fi
if [ "$TARGET_SYNC_DIR" == "" ]; then
Logger "Not TARGET_SYNC_DIR set in config file." "CRITICAL"
exit 1
fi
if ([ ! -f "$SSH_RSA_PRIVATE_KEY" ] && [ ! -f "$SSH_PASSWORD_FILE" ]); then
Logger "Cannot find rsa private key [$SSH_RSA_PRIVATE_KEY] nor password file [$SSH_PASSWORD_FILE]. No authentication method provided." "CRITICAL"
exit 1
fi
}
function TriggerInitiatorUpdate {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
$SSH_CMD env _REMOTE_TOKEN="$_REMOTE_TOKEN" \
env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \
env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" TSTAMP="'$TSTAMP'" env INSTANCE_ID="'$INSTANCE_ID'" \
env PUSH_FILE="'$(EscapeSpaces "${INITIATOR[$__updateTriggerFIle]}")'" \
env LC_ALL=C $COMMAND_SUDO' bash -s' << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1
include #### DEBUG SUBSET ####
include #### TrapError SUBSET ####
include #### RemoteLogger SUBSET ####
echo "$INSTANCE_ID $(date '+%Y%m%dT%H%M%S.%N')" >> "$PUSH_FILE"
ENDSSH
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ] || [ $? != 0 ]; then
(
_LOGGER_PREFIX="RR"
Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "ERROR"
)
return 1
fi
return 0
}
function Init {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
# Set error exit code if a piped command fails
set -o pipefail
set -o errtrace
trap TrapQuit TERM EXIT HUP QUIT
local uri
local hosturiandpath
local hosturi
## Test if target dir is a ssh uri, and if yes, break it down it its values
if [ "${INITIATOR_SYNC_DIR:0:6}" == "ssh://" ]; then
REMOTE_OPERATION="yes"
# remove leadng 'ssh://'
uri=${INITIATOR_SYNC_DIR#ssh://*}
if [[ "$uri" == *"@"* ]]; then
# remove everything after '@'
REMOTE_USER=${uri%@*}
else
REMOTE_USER=$LOCAL_USER
fi
if [ "$SSH_RSA_PRIVATE_KEY" == "" ]; then
if [ ! -f "$SSH_PASSWORD_FILE" ]; then
# Assume that there might exist a standard rsa key
SSH_RSA_PRIVATE_KEY=~/.ssh/id_rsa
fi
fi
# remove everything before '@'
hosturiandpath=${uri#*@}
# remove everything after first '/'
hosturi=${hosturiandpath%%/*}
if [[ "$hosturi" == *":"* ]]; then
REMOTE_PORT=${hosturi##*:}
else
REMOTE_PORT=22
fi
REMOTE_HOST=${hosturi%%:*}
# remove everything before first '/'
TARGET_SYNC_DIR=${hosturiandpath#*/}
else
Logger "No valid remote initiator URI found in [$INITIATOR_SYNC_DIR]." "CRITICAL"
exit 1
fi
if [ "$INITIATOR_SYNC_DIR" == "" ] || [ "$TARGET_SYNC_DIR" == "" ]; then
Logger "Initiator or target path empty." "CRITICAL"
exit 1
fi
## Make sure there is only one trailing slash on path
INITIATOR_SYNC_DIR="${INITIATOR_SYNC_DIR%/}/"
TARGET_SYNC_DIR="${TARGET_SYNC_DIR%/}/"
# Expand ~ if exists
INITIATOR_SYNC_DIR="${INITIATOR_SYNC_DIR/#\~/$HOME}"
TARGET_SYNC_DIR="${TARGET_SYNC_DIR/#\~/$HOME}"
SSH_RSA_PRIVATE_KEY="${SSH_RSA_PRIVATE_KEY/#\~/$HOME}"
SSH_PASSWORD_FILE="${SSH_PASSWORD_FILE/#\~/$HOME}"
## Replica format
## Why the f*** does bash not have simple objects ?
# Local variables used for state filenames
local lockFilename="lock"
local stateDir="state"
local backupDir="backup"
local deleteDir="deleted"
local partialDir="_partial"
local lastAction="last-action"
local resumeCount="resume-count"
if [ "$_DRYRUN" == true ]; then
local drySuffix="-dry"
else
local drySuffix=
fi
# The following associative like array definitions are used for bash ver < 4 compat
readonly __type=0
readonly __replicaDir=1
readonly __lockFile=2
readonly __stateDir=3
readonly __backupDir=4
readonly __deleteDir=5
readonly __partialDir=6
readonly __initiatorLastActionFile=7
readonly __targetLastActionFile=8
readonly __resumeCount=9
readonly __treeCurrentFile=10
readonly __treeAfterFile=11
readonly __treeAfterFileNoSuffix=12
readonly __deletedListFile=13
readonly __failedDeletedListFile=14
readonly __successDeletedListFile=15
readonly __timestampCurrentFile=16
readonly __timestampAfterFile=17
readonly __timestampAfterFileNoSuffix=18
readonly __conflictListFile=19
readonly __updateTriggerFile=20
INITIATOR=()
INITIATOR[$__type]='initiator'
INITIATOR[$__replicaDir]="$INITIATOR_SYNC_DIR"
INITIATOR[$__lockFile]="$INITIATOR_SYNC_DIR$OSYNC_DIR/$lockFilename"
INITIATOR[$__stateDir]="$OSYNC_DIR/$stateDir"
INITIATOR[$__backupDir]="$OSYNC_DIR/$backupDir"
INITIATOR[$__deleteDir]="$OSYNC_DIR/$deleteDir"
INITIATOR[$__partialDir]="$OSYNC_DIR/$partialDir"
INITIATOR[$__initiatorLastActionFile]="$INITIATOR_SYNC_DIR$OSYNC_DIR/$stateDir/initiator-$lastAction-$INSTANCE_ID$drySuffix"
INITIATOR[$__targetLastActionFile]="$INITIATOR_SYNC_DIR$OSYNC_DIR/$stateDir/target-$lastAction-$INSTANCE_ID$drySuffix"
INITIATOR[$__resumeCount]="$INITIATOR_SYNC_DIR$OSYNC_DIR/$stateDir/$resumeCount-$INSTANCE_ID$drySuffix"
INITIATOR[$__treeCurrentFile]="-tree-current-$INSTANCE_ID$drySuffix"
INITIATOR[$__treeAfterFile]="-tree-after-$INSTANCE_ID$drySuffix"
INITIATOR[$__treeAfterFileNoSuffix]="-tree-after-$INSTANCE_ID"
INITIATOR[$__deletedListFile]="-deleted-list-$INSTANCE_ID$drySuffix"
INITIATOR[$__failedDeletedListFile]="-failed-delete-$INSTANCE_ID$drySuffix"
INITIATOR[$__successDeletedListFile]="-success-delete-$INSTANCE_ID$drySuffix"
INITIATOR[$__timestampCurrentFile]="-timestamps-current-$INSTANCE_ID$drySuffix"
INITIATOR[$__timestampAfterFile]="-timestamps-after-$INSTANCE_ID$drySuffix"
INITIATOR[$__timestampAfterFileNoSuffix]="-timestamps-after-$INSTANCE_ID"
INITIATOR[$__conflictListFile]="conflicts-$INSTANCE_ID$drySuffix"
INITIATOR[$__updateTriggerFile]="$INITIATOR_SYNC_DIR$OSYNC_DIR/.osnyc-update.push"
TARGET=()
TARGET[$__type]='target'
TARGET[$__replicaDir]="$TARGET_SYNC_DIR"
TARGET[$__lockFile]="$TARGET_SYNC_DIR$OSYNC_DIR/$lockFilename"
TARGET[$__stateDir]="$OSYNC_DIR/$stateDir"
TARGET[$__backupDir]="$OSYNC_DIR/$backupDir"
TARGET[$__deleteDir]="$OSYNC_DIR/$deleteDir"
TARGET[$__partialDir]="$OSYNC_DIR/$partialDir" # unused
TARGET[$__initiatorLastActionFile]="$TARGET_SYNC_DIR$OSYNC_DIR/$stateDir/initiator-$lastAction-$INSTANCE_ID$drySuffix" # unused
TARGET[$__targetLastActionFile]="$TARGET_SYNC_DIR$OSYNC_DIR/$stateDir/target-$lastAction-$INSTANCE_ID$drySuffix" # unused
TARGET[$__resumeCount]="$TARGET_SYNC_DIR$OSYNC_DIR/$stateDir/$resumeCount-$INSTANCE_ID$drySuffix" # unused
TARGET[$__treeCurrentFile]="-tree-current-$INSTANCE_ID$drySuffix" # unused
TARGET[$__treeAfterFile]="-tree-after-$INSTANCE_ID$drySuffix" # unused
TARGET[$__treeAfterFileNoSuffix]="-tree-after-$INSTANCE_ID" # unused
TARGET[$__deletedListFile]="-deleted-list-$INSTANCE_ID$drySuffix" # unused
TARGET[$__failedDeletedListFile]="-failed-delete-$INSTANCE_ID$drySuffix"
TARGET[$__successDeletedListFile]="-success-delete-$INSTANCE_ID$drySuffix"
TARGET[$__timestampCurrentFile]="-timestamps-current-$INSTANCE_ID$drySuffix"
TARGET[$__timestampAfterFile]="-timestamps-after-$INSTANCE_ID$drySuffix"
TARGET[$__timestampAfterFileNoSuffix]="-timestamps-after-$INSTANCE_ID"
TARGET[$__conflictListFile]="conflicts-$INSTANCE_ID$drySuffix"
TARGET[$__updateTriggerFile]="$TARGET_SYNC_DIR$OSYNC_DIR/.osync-update.push"
}
function Usage {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
if [ "$IS_STABLE" != "yes" ]; then
echo -e "\e[93mThis is an unstable dev build. Please use with caution.\e[0m"
fi
echo "$PROGRAM $PROGRAM_VERSION $PROGRAM_BUILD"
echo "$AUTHOR"
echo "$CONTACT"
echo ""
echo "You must use $PROGRAM with a full blown configuration file."
echo "Usage: $0 /path/to/config/file [OPTIONS]"
echo ""
echo "[OPTIONS]"
echo "--no-prefix Will suppress time / date suffix from output"
echo "--silent Will run osync without any output to stdout, used for cron jobs"
echo "--errors-only Output only errors (can be combined with silent or verbose)"
echo "--verbose Increases output"
echo "--on-changes Will launch a sync task after a short wait period if there is some file activity on initiator replica. You should try daemon mode instead"
echo ""
exit 128
}
function OnChangesHelper {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
local cmd
local retval
if [ "$LOCAL_OS" == "MacOSX" ]; then
if ! type fswatch > /dev/null 2>&1 ; then
Logger "No inotifywait command found. Cannot monitor changes." "CRITICAL"
exit 1
fi
else
if ! type inotifywait > /dev/null 2>&1 ; then
Logger "No inotifywait command found. Cannot monitor changes." "CRITICAL"
exit 1
fi
fi
if [ ! -d "$TARGET_SYNC_DIR" ]; then
Logger "Target directory [$TARGET_SYNC_DIR] does not exist. Cannot monitor." "CRITICAL"
exit 1
fi
Logger "#### Running $PROGRAM in file monitor mode." "NOTICE"
while true; do
if [ "$LOCAL_OS" == "MacOSX" ]; then
fswatch $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude "$OSYNC_DIR" -1 "$TARGET_SYNC_DIR" > /dev/null &
# Mac fswatch doesn't have timeout switch, replacing wait $! with WaitForTaskCompletion without warning nor spinner and increased SLEEP_TIME to avoid cpu hogging. This sims wait $! with timeout
WaitForTaskCompletion $! 0 $MAX_WAIT 1 0 true false true
else
inotifywait $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude "$OSYNC_DIR" -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$TARGET_SYNC_DIR" &
wait $!
fi
retval=$?
if [ $retval -eq 0 ]; then
Logger "#### Changes detected, waiting $MIN_WAIT seconds before triggering update on initiator." "NOTICE"
sleep $MIN_WAIT
# inotifywait --timeout result is 2, WaitForTaskCompletion HardTimeout is 1
elif [ "$LOCAL_OS" == "MacOSX" ]; then
Logger "#### Changes or error detected, waiting $MIN_WAIT seconds before triggering update on initiator." "NOTICE"
elif [ $retval -eq 2 ]; then
Logger "#### $MAX_WAIT timeout reached, running sync." "NOTICE"
elif [ $retval -eq 1 ]; then
Logger "#### inotify error detected, waiting $MIN_WAIT seconds before triggering update on initiator." "ERROR" $retval
sleep $MIN_WAIT
fi
TriggerInitiatorUpdate
done
}
#### SCRIPT ENTRY POINT
DESTINATION_MAILS=""
ERROR_ALERT=false
WARN_ALERT=false
if [ $# -eq 0 ]
then
Usage
fi
first=1
for i in "$@"; do
case $i in
--silent)
_LOGGER_SILENT=true
;;
--verbose)
_LOGGER_VERBOSE=true
;;
--help|-h|--version|-v)
Usage
;;
--errors-only)
_LOGGER_ERR_ONLY=true
;;
--no-prefix)
_LOGGER_PREFIX=""
;;
*)
if [ $first == "0" ]; then
Logger "Unknown option '$i'" "CRITICAL"
Usage
fi
;;
esac
first=0
done
# Remove leading space if there is one
opts="${opts# *}"
ConfigFile="${1}"
LoadConfigFile "$ConfigFile"
if [ "$LOGFILE" == "" ]; then
if [ -w /var/log ]; then
LOG_FILE="/var/log/$PROGRAM.$INSTANCE_ID.log"
elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
LOG_FILE="$HOME/$PROGRAM.$INSTANCE_ID.log"
else
LOG_FILE="./$PROGRAM.$INSTANCE_ID.log"
fi
else
LOG_FILE="$LOGFILE"
fi
if [ ! -w "$(dirname $LOG_FILE)" ]; then
echo "Cannot write to log [$(dirname $LOG_FILE)]."
else
Logger "Script begin, logging to [$LOG_FILE]." "DEBUG"
fi
if [ "$IS_STABLE" != "yes" ]; then
Logger "This is an unstable dev build [$PROGRAM_BUILD]. Please use with caution." "WARN"
fi
GetLocalOS
InitLocalOSDependingSettings
PreInit
Init
CheckEnvironment
PostInit
CheckCurrentConfig
CheckCurrentConfigAll
DATE=$(date)
Logger "-------------------------------------------------------------" "NOTICE"
Logger "$DRY_WARNING$DATE - $PROGRAM $PROGRAM_VERSION script begin." "ALWAYS"
Logger "-------------------------------------------------------------" "NOTICE"
Logger "Sync task [$INSTANCE_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" "NOTICE"
OnChangesHelper

@ -0,0 +1,79 @@
#!/usr/bin/env bash
###### osync - Rsync based two way sync engine with fault tolerance
###### (C) 2013-2017 by Orsiris de Jong (www.netpower.fr)
###### osync-target-helper v1.2.2+ config file rev 2017061901
## ---------- GENERAL OPTIONS
## Sync job identification
INSTANCE_ID="sync_test"
## Directories to synchronize.
## Initiator is the system main osync runs on. The initiator directory must be a remote path for osync target helper to contact.
INITIATOR_SYNC_DIR="ssh://backupuser@yourhost.old:22//home/git/osync/dir1"
## Target is the system osync synchronizes to. The target directory must be a local.
TARGET_SYNC_DIR="/home/git/osync/dir2"
## If the target system is remote, you can specify a RSA key (please use full path). If not defined, the default ~/.ssh/id_rsa will be used. See documentation for further information.
SSH_RSA_PRIVATE_KEY="/home/backupuser/.ssh/id_rsa"
## Alternatively, you may specify an SSH password file (less secure). Needs sshpass utility installed.
SSH_PASSWORD_FILE=""
## When using ssh filter, you must specify a remote token matching the one setup in authorized_keys
_REMOTE_TOKEN=SomeAlphaNumericToken9
## Log file location. Leaving this empty will create a logfile at /var/log/osync_version_SYNC_ID.log (or current directory if /var/log doesn't exist)
LOGFILE=""
## If enabled, synchronization on remote system will be processed as superuser. See documentation for /etc/sudoers file configuration.
SUDO_EXEC=no
## ---------- REMOTE SYNC OPTIONS
## ssh compression should be used unless your remote connection is good enough (LAN)
SSH_COMPRESSION=yes
## Ignore ssh known hosts. DANGER WILL ROBINSON DANGER ! This can lead to security issues. Only enable this if you know what you're doing.
SSH_IGNORE_KNOWN_HOSTS=no
## Check for connectivity to remote host before launching remote sync task. Be sure the hosts responds to ping. Failing to ping will stop sync.
REMOTE_HOST_PING=no
## Check for internet access by pinging one or more 3rd party hosts before remote sync task. Leave empty if you don't want this check to be be performed. Failing to ping will stop sync.
## If you use this function, you should set more than one 3rd party host, and be sure you can ping them.
## Be aware some DNS like opendns redirect false hostnames. Also, this adds an extra execution time of a bit less than a minute.
REMOTE_3RD_PARTY_HOSTS="www.kernel.org www.google.com"
## Minimum time (in seconds) in file monitor /daemon mode between modification detection and sync task in order to let copy operations finish.
MIN_WAIT=60
## Maximum time (in seconds) waiting in file monitor / daemon mode. After this time, sync is run.
## Use 0 to wait indefinitely.
MAX_WAIT=7200
## ---------- ALERT OPTIONS
## List of alert mails separated by spaces
## Most Unix systems (including Win10 bash) have mail support out of the box
## Just make sure that the current user has enough privileges to use mail / mutt / sendmail and that the mail system is configured to allow outgoing mails
## on pfSense platform, smtp support needs to be configured in System > Advanced > Notifications
DESTINATION_MAILS="your@alert.tld"
## Optional change of mail body encoding (using iconv)
## By default, all mails are sent in UTF-8 format without header (because of maximum compatibility of all platforms)
## You may specify an optional encoding here (like "ISO-8859-1" or whatever iconv can handle)
MAIL_BODY_CHARSET=""
## Additional mail parameters needed for Android / Busybox / Cygwin / MSYS
## Android & Busybox use sendmail (and openssl if encryption is needed)
## MSYS & Cygwin Windows mail support relies on mailsend.exe from muquit, http://github.com/muquit/mailsend which needs to be in %PATH% environment variable
SENDER_MAIL="alert@your.system.tld"
SMTP_SERVER=smtp.your.isp.tld
SMTP_PORT=25
# encryption can be tls, ssl or none
SMTP_ENCRYPTION=none
SMTP_USER=
SMTP_PASSWORD=
Loading…
Cancel
Save