From 8d9e95c6cd5c9c1078dd234aa9c78fde0d323e02 Mon Sep 17 00:00:00 2001 From: deajan Date: Fri, 5 Jul 2019 22:37:37 +0200 Subject: [PATCH] Rebuilt targets --- dev/debug_osync.sh | 47 +++++++++++++++++++++++++++++++--- install.sh | 6 +++++ osync.sh | 64 +++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 107 insertions(+), 10 deletions(-) diff --git a/dev/debug_osync.sh b/dev/debug_osync.sh index 89dda21..23727d0 100755 --- a/dev/debug_osync.sh +++ b/dev/debug_osync.sh @@ -7,7 +7,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2019 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.3.0-beta3+dev -PROGRAM_BUILD=2019052203 +PROGRAM_BUILD=2019070501 IS_STABLE=false CONFIG_FILE_REVISION_REQUIRED=1.3.0 @@ -123,6 +123,8 @@ else fi ## Special note when remote target is on the same host as initiator (happens for unit tests): we'll have to differentiate RUN_DIR so remote CleanUp won't affect initiator. +## If the same program gets remotely executed, add _REMOTE_EXECUTION=true to it's environment so it knows it has to write into a separate directory +## This will thus not affect local $RUN_DIR variables if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote" RUN_DIR="$RUN_DIR/$PROGRAM.remote" @@ -2011,7 +2013,7 @@ function PreInit { ## SSH ControlMaster Multiplexing if [ "$SSH_CONTROLMASTER" == true ]; then - SSH_OPTS="$SSH_OPTS -o ControlMaster=auto -o ControlPersist=yes -o ControlPath=$RUN_DIR/%r@%h" + SSH_OPTS="$SSH_OPTS -o ControlMaster=auto -o ControlPersist=yes -o ControlPath=\"$RUN_DIR/$PROGRAM.ctrlm.%r@%h.$SCRIPT_PID.$TSTAMP\"" fi ## Support for older config files without RSYNC_EXECUTABLE option @@ -2691,6 +2693,8 @@ else fi ## Special note when remote target is on the same host as initiator (happens for unit tests): we'll have to differentiate RUN_DIR so remote CleanUp won't affect initiator. +## If the same program gets remotely executed, add _REMOTE_EXECUTION=true to it's environment so it knows it has to write into a separate directory +## This will thus not affect local $RUN_DIR variables if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote" RUN_DIR="$RUN_DIR/$PROGRAM.remote" @@ -3073,6 +3077,8 @@ else fi ## Special note when remote target is on the same host as initiator (happens for unit tests): we'll have to differentiate RUN_DIR so remote CleanUp won't affect initiator. +## If the same program gets remotely executed, add _REMOTE_EXECUTION=true to it's environment so it knows it has to write into a separate directory +## This will thus not affect local $RUN_DIR variables if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote" RUN_DIR="$RUN_DIR/$PROGRAM.remote" @@ -3241,6 +3247,10 @@ function CleanUp { # Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements) rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp" fi + + if [ "$SSH_CONTROLMASTER" == true ]; then + $SSH_CMD -O exit + fi } function _HandleLocksRemoteSub { @@ -3672,6 +3682,8 @@ else fi ## Special note when remote target is on the same host as initiator (happens for unit tests): we'll have to differentiate RUN_DIR so remote CleanUp won't affect initiator. +## If the same program gets remotely executed, add _REMOTE_EXECUTION=true to it's environment so it knows it has to write into a separate directory +## This will thus not affect local $RUN_DIR variables if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote" RUN_DIR="$RUN_DIR/$PROGRAM.remote" @@ -3852,6 +3864,10 @@ function CleanUp { # Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements) rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp" fi + + if [ "$SSH_CONTROLMASTER" == true ]; then + $SSH_CMD -O exit + fi } function _getFileCtimeMtimeRemoteSub { @@ -4371,6 +4387,8 @@ else fi ## Special note when remote target is on the same host as initiator (happens for unit tests): we'll have to differentiate RUN_DIR so remote CleanUp won't affect initiator. +## If the same program gets remotely executed, add _REMOTE_EXECUTION=true to it's environment so it knows it has to write into a separate directory +## This will thus not affect local $RUN_DIR variables if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote" RUN_DIR="$RUN_DIR/$PROGRAM.remote" @@ -4502,6 +4520,10 @@ function CleanUp { # Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements) rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp" fi + + if [ "$SSH_CONTROLMASTER" == true ]; then + $SSH_CMD -O exit + fi } function _deleteRemoteSub { @@ -5297,6 +5319,8 @@ else fi ## Special note when remote target is on the same host as initiator (happens for unit tests): we'll have to differentiate RUN_DIR so remote CleanUp won't affect initiator. +## If the same program gets remotely executed, add _REMOTE_EXECUTION=true to it's environment so it knows it has to write into a separate directory +## This will thus not affect local $RUN_DIR variables if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote" RUN_DIR="$RUN_DIR/$PROGRAM.remote" @@ -5477,6 +5501,10 @@ function CleanUp { # Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements) rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp" fi + + if [ "$SSH_CONTROLMASTER" == true ]; then + $SSH_CMD -O exit + fi } function _SoftDeleteRemoteSub { @@ -5620,6 +5648,8 @@ else fi ## Special note when remote target is on the same host as initiator (happens for unit tests): we'll have to differentiate RUN_DIR so remote CleanUp won't affect initiator. +## If the same program gets remotely executed, add _REMOTE_EXECUTION=true to it's environment so it knows it has to write into a separate directory +## This will thus not affect local $RUN_DIR variables if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote" RUN_DIR="$RUN_DIR/$PROGRAM.remote" @@ -5751,6 +5781,10 @@ function CleanUp { # Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements) rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp" fi + + if [ "$SSH_CONTROLMASTER" == true ]; then + $SSH_CMD -O exit + fi } if [ -d $(dirname "$PUSH_FILE") ]; then @@ -6189,6 +6223,7 @@ function Usage { echo "--initiator=\"\" Master replica path. Will contain state and backup directory (is mandatory)" echo "--target=\"\" Local or remote target replica path. Can be a ssh uri like ssh://user@host.com:22//path/to/target/replica (is mandatory)" echo "--rsakey=\"\" Alternative path to rsa private key for ssh connection to target replica" + echo "--ssh-controlmaster Allow using a single TCP connection for all ssh calls. Will make remote sync faster, but may fail easier on lossy links" echo "--password-file=\"\" If no rsa private key is used for ssh authentication, a password file can be used" echo "--remote-token=\"\" When using ssh filter protection, you must specify the remote token set in ssh_filter.sh" echo "--instance-id=\"\" Optional sync task name to identify this synchronization task when using multiple targets" @@ -6405,6 +6440,10 @@ function GetCommandlineArguments { SSH_PASSWORD_FILE="${i##*=}" opts=$opts" --password-file=\"$SSH_PASSWORD_FILE\"" ;; + --ssh-controlmaster) + SSH_CONTROLMASTER=true + opts=$opts" --ssh-controlmaster" + ;; --instance-id=*) INSTANCE_ID="${i##*=}" opts=$opts" --instance-id=\"$INSTANCE_ID\"" @@ -6555,8 +6594,8 @@ if [ "$LOGFILE" == "" ]; then else LOG_FILE="$LOGFILE" fi -if [ ! -w "$(dirname $LOG_FILE)" ]; then - echo "Cannot write to log [$(dirname $LOG_FILE)]." +if [ ! -w "$(dirname "$LOG_FILE")" ]; then + echo "Cannot write to log [$(dirname "$LOG_FILE")]." else Logger "Script begin, logging to [$LOG_FILE]." "DEBUG" fi diff --git a/install.sh b/install.sh index d4b22fb..bac6133 100755 --- a/install.sh +++ b/install.sh @@ -94,6 +94,8 @@ else fi ## Special note when remote target is on the same host as initiator (happens for unit tests): we'll have to differentiate RUN_DIR so remote CleanUp won't affect initiator. +## If the same program gets remotely executed, add _REMOTE_EXECUTION=true to it's environment so it knows it has to write into a separate directory +## This will thus not affect local $RUN_DIR variables if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote" RUN_DIR="$RUN_DIR/$PROGRAM.remote" @@ -404,6 +406,10 @@ function CleanUp { # Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements) rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp" fi + + if [ "$SSH_CONTROLMASTER" == true ]; then + $SSH_CMD -O exit + fi } function GenericTrapQuit { diff --git a/osync.sh b/osync.sh index a1eb606..7f54ff0 100755 --- a/osync.sh +++ b/osync.sh @@ -7,7 +7,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2019 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.3.0-beta3+dev -PROGRAM_BUILD=2019052203 +PROGRAM_BUILD=2019070501 IS_STABLE=false CONFIG_FILE_REVISION_REQUIRED=1.3.0 @@ -90,6 +90,8 @@ else fi ## Special note when remote target is on the same host as initiator (happens for unit tests): we'll have to differentiate RUN_DIR so remote CleanUp won't affect initiator. +## If the same program gets remotely executed, add _REMOTE_EXECUTION=true to it's environment so it knows it has to write into a separate directory +## This will thus not affect local $RUN_DIR variables if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote" RUN_DIR="$RUN_DIR/$PROGRAM.remote" @@ -400,6 +402,10 @@ function CleanUp { # Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements) rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp" fi + + if [ "$SSH_CONTROLMASTER" == true ]; then + $SSH_CMD -O exit + fi } function GenericTrapQuit { @@ -1859,6 +1865,11 @@ function PreInit { SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" fi + ## SSH ControlMaster Multiplexing + if [ "$SSH_CONTROLMASTER" == true ]; then + SSH_OPTS="$SSH_OPTS -o ControlMaster=auto -o ControlPersist=yes -o ControlPath=\"$RUN_DIR/$PROGRAM.ctrlm.%r@%h.$SCRIPT_PID.$TSTAMP\"" + fi + ## Support for older config files without RSYNC_EXECUTABLE option if [ "$RSYNC_EXECUTABLE" == "" ]; then RSYNC_EXECUTABLE=rsync @@ -2374,11 +2385,11 @@ function CheckCurrentConfig { # Full check is for initiator driven runs if [ $fullCheck == true ]; then - declare -a booleans=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_PERMISSIONS PRESERVE_OWNER PRESERVE_GROUP PRESERVE_EXECUTABILITY PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR) + declare -a booleans=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_PERMISSIONS PRESERVE_OWNER PRESERVE_GROUP PRESERVE_EXECUTABILITY PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR SSH_CONTROLMASTER) declare -a num_vars=(MINIMUM_SPACE BANDWIDTH SOFT_MAX_EXEC_TIME HARD_MAX_EXEC_TIME KEEP_LOGGING MIN_WAIT MAX_WAIT CONFLICT_BACKUP_DAYS SOFT_DELETE_DAYS RESUME_TRY MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER) # target-helper runs need less configuration else - declare -a booleans=(SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING) + declare -a booleans=(SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING SSH_CONTROLMASTER) declare -a num_vars=(KEEP_LOGGING MIN_WAIT MAX_WAIT) fi @@ -2402,7 +2413,7 @@ function UpdateBooleans { local update local booleans - declare -a booleans=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_PERMISSIONS PRESERVE_OWNER PRESERVE_GROUP PRESERVE_EXECUTABILITY PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR) + declare -a booleans=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_PERMISSIONS PRESERVE_OWNER PRESERVE_GROUP PRESERVE_EXECUTABILITY PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR SSH_CONTROLMASTER) for i in "${booleans[@]}"; do update="if [ \"\$$i\" == \"yes\" ]; then $i=true; fi; if [ \"\$$i\" == \"no\" ]; then $i=false; fi" @@ -2528,6 +2539,8 @@ else fi ## Special note when remote target is on the same host as initiator (happens for unit tests): we'll have to differentiate RUN_DIR so remote CleanUp won't affect initiator. +## If the same program gets remotely executed, add _REMOTE_EXECUTION=true to it's environment so it knows it has to write into a separate directory +## This will thus not affect local $RUN_DIR variables if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote" RUN_DIR="$RUN_DIR/$PROGRAM.remote" @@ -2699,6 +2712,10 @@ function CleanUp { # Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements) rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp" fi + + if [ "$SSH_CONTROLMASTER" == true ]; then + $SSH_CMD -O exit + fi } function _CheckReplicasRemoteSub { @@ -2894,6 +2911,8 @@ else fi ## Special note when remote target is on the same host as initiator (happens for unit tests): we'll have to differentiate RUN_DIR so remote CleanUp won't affect initiator. +## If the same program gets remotely executed, add _REMOTE_EXECUTION=true to it's environment so it knows it has to write into a separate directory +## This will thus not affect local $RUN_DIR variables if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote" RUN_DIR="$RUN_DIR/$PROGRAM.remote" @@ -3053,6 +3072,10 @@ function CleanUp { # Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements) rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp" fi + + if [ "$SSH_CONTROLMASTER" == true ]; then + $SSH_CMD -O exit + fi } function _HandleLocksRemoteSub { @@ -3476,6 +3499,8 @@ else fi ## Special note when remote target is on the same host as initiator (happens for unit tests): we'll have to differentiate RUN_DIR so remote CleanUp won't affect initiator. +## If the same program gets remotely executed, add _REMOTE_EXECUTION=true to it's environment so it knows it has to write into a separate directory +## This will thus not affect local $RUN_DIR variables if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote" RUN_DIR="$RUN_DIR/$PROGRAM.remote" @@ -3647,6 +3672,10 @@ function CleanUp { # Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements) rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp" fi + + if [ "$SSH_CONTROLMASTER" == true ]; then + $SSH_CMD -O exit + fi } function _getFileCtimeMtimeRemoteSub { @@ -4160,6 +4189,8 @@ else fi ## Special note when remote target is on the same host as initiator (happens for unit tests): we'll have to differentiate RUN_DIR so remote CleanUp won't affect initiator. +## If the same program gets remotely executed, add _REMOTE_EXECUTION=true to it's environment so it knows it has to write into a separate directory +## This will thus not affect local $RUN_DIR variables if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote" RUN_DIR="$RUN_DIR/$PROGRAM.remote" @@ -4282,6 +4313,10 @@ function CleanUp { # Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements) rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp" fi + + if [ "$SSH_CONTROLMASTER" == true ]; then + $SSH_CMD -O exit + fi } function _deleteRemoteSub { @@ -5072,6 +5107,8 @@ else fi ## Special note when remote target is on the same host as initiator (happens for unit tests): we'll have to differentiate RUN_DIR so remote CleanUp won't affect initiator. +## If the same program gets remotely executed, add _REMOTE_EXECUTION=true to it's environment so it knows it has to write into a separate directory +## This will thus not affect local $RUN_DIR variables if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote" RUN_DIR="$RUN_DIR/$PROGRAM.remote" @@ -5243,6 +5280,10 @@ function CleanUp { # Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements) rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp" fi + + if [ "$SSH_CONTROLMASTER" == true ]; then + $SSH_CMD -O exit + fi } function _SoftDeleteRemoteSub { @@ -5383,6 +5424,8 @@ else fi ## Special note when remote target is on the same host as initiator (happens for unit tests): we'll have to differentiate RUN_DIR so remote CleanUp won't affect initiator. +## If the same program gets remotely executed, add _REMOTE_EXECUTION=true to it's environment so it knows it has to write into a separate directory +## This will thus not affect local $RUN_DIR variables if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote" RUN_DIR="$RUN_DIR/$PROGRAM.remote" @@ -5505,6 +5548,10 @@ function CleanUp { # Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements) rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp" fi + + if [ "$SSH_CONTROLMASTER" == true ]; then + $SSH_CMD -O exit + fi } if [ -d $(dirname "$PUSH_FILE") ]; then @@ -5935,6 +5982,7 @@ function Usage { echo "--initiator=\"\" Master replica path. Will contain state and backup directory (is mandatory)" echo "--target=\"\" Local or remote target replica path. Can be a ssh uri like ssh://user@host.com:22//path/to/target/replica (is mandatory)" echo "--rsakey=\"\" Alternative path to rsa private key for ssh connection to target replica" + echo "--ssh-controlmaster Allow using a single TCP connection for all ssh calls. Will make remote sync faster, but may fail easier on lossy links" echo "--password-file=\"\" If no rsa private key is used for ssh authentication, a password file can be used" echo "--remote-token=\"\" When using ssh filter protection, you must specify the remote token set in ssh_filter.sh" echo "--instance-id=\"\" Optional sync task name to identify this synchronization task when using multiple targets" @@ -6150,6 +6198,10 @@ function GetCommandlineArguments { SSH_PASSWORD_FILE="${i##*=}" opts=$opts" --password-file=\"$SSH_PASSWORD_FILE\"" ;; + --ssh-controlmaster) + SSH_CONTROLMASTER=true + opts=$opts" --ssh-controlmaster" + ;; --instance-id=*) INSTANCE_ID="${i##*=}" opts=$opts" --instance-id=\"$INSTANCE_ID\"" @@ -6300,8 +6352,8 @@ if [ "$LOGFILE" == "" ]; then else LOG_FILE="$LOGFILE" fi -if [ ! -w "$(dirname $LOG_FILE)" ]; then - echo "Cannot write to log [$(dirname $LOG_FILE)]." +if [ ! -w "$(dirname "$LOG_FILE")" ]; then + echo "Cannot write to log [$(dirname "$LOG_FILE")]." else Logger "Script begin, logging to [$LOG_FILE]." "DEBUG" fi