|
|
|
@ -8,7 +8,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
|
|
|
|
|
AUTHOR="(C) 2013-2018 by Orsiris de Jong"
|
|
|
|
|
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
|
|
|
|
|
PROGRAM_VERSION=1.3.0-beta1
|
|
|
|
|
PROGRAM_BUILD=2018080801
|
|
|
|
|
PROGRAM_BUILD=2018093001
|
|
|
|
|
IS_STABLE=no
|
|
|
|
|
|
|
|
|
|
##### Execution order #__WITH_PARANOIA_DEBUG
|
|
|
|
@ -41,8 +41,8 @@ IS_STABLE=no
|
|
|
|
|
# UnlockReplicas yes #__WITH_PARANOIA_DEBUG
|
|
|
|
|
# CleanUp no #__WITH_PARANOIA_DEBUG
|
|
|
|
|
|
|
|
|
|
_OFUNCTIONS_VERSION=2.3.0-beta1
|
|
|
|
|
_OFUNCTIONS_BUILD=2018080801
|
|
|
|
|
_OFUNCTIONS_VERSION=2.3.0-RC1
|
|
|
|
|
_OFUNCTIONS_BUILD=2018093002
|
|
|
|
|
_OFUNCTIONS_BOOTSTRAP=true
|
|
|
|
|
|
|
|
|
|
## To use in a program, define the following variables:
|
|
|
|
@ -305,7 +305,7 @@ function Logger {
|
|
|
|
|
if [ "$_LOGGER_PREFIX" == "time" ]; then
|
|
|
|
|
prefix="TIME: $SECONDS - "
|
|
|
|
|
elif [ "$_LOGGER_PREFIX" == "date" ]; then
|
|
|
|
|
prefix="$(date) - "
|
|
|
|
|
prefix="$(date '+%Y-%m-%d %H:%M:%S') - "
|
|
|
|
|
else
|
|
|
|
|
prefix=""
|
|
|
|
|
fi
|
|
|
|
@ -2131,8 +2131,8 @@ function InitRemoteOSDependingSettings {
|
|
|
|
|
REMOTE_STAT_CTIME_MTIME_CMD="stat -c \\\"%n;%Z;%Y\\\""
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
## Set rsync default arguments
|
|
|
|
|
RSYNC_ARGS="-rltD -8"
|
|
|
|
|
## Set rsync default arguments (complete with -r or -d depending on recursivity later)
|
|
|
|
|
RSYNC_DEFAULT_ARGS="-ltD -8"
|
|
|
|
|
if [ "$_DRYRUN" == true ]; then
|
|
|
|
|
RSYNC_DRY_ARG="-n"
|
|
|
|
|
DRY_WARNING="/!\ DRY RUN "
|
|
|
|
@ -2170,43 +2170,46 @@ function InitRemoteOSDependingSettings {
|
|
|
|
|
fi
|
|
|
|
|
if [ "$RSYNC_COMPRESS" == "yes" ]; then
|
|
|
|
|
if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then
|
|
|
|
|
RSYNC_ARGS=$RSYNC_ARGS" -zz --skip-compress=gz/xz/lz/lzma/lzo/rz/jpg/mp3/mp4/7z/bz2/rar/zip/sfark/s7z/ace/apk/arc/cab/dmg/jar/kgb/lzh/lha/lzx/pak/sfx"
|
|
|
|
|
RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -zz --skip-compress=gz/xz/lz/lzma/lzo/rz/jpg/mp3/mp4/7z/bz2/rar/zip/sfark/s7z/ace/apk/arc/cab/dmg/jar/kgb/lzh/lha/lzx/pak/sfx"
|
|
|
|
|
else
|
|
|
|
|
Logger "Disabling compression skips on synchronization on [$LOCAL_OS] due to lack of support." "NOTICE"
|
|
|
|
|
fi
|
|
|
|
|
fi
|
|
|
|
|
if [ "$COPY_SYMLINKS" == "yes" ]; then
|
|
|
|
|
RSYNC_ARGS=$RSYNC_ARGS" -L"
|
|
|
|
|
RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -L"
|
|
|
|
|
fi
|
|
|
|
|
if [ "$KEEP_DIRLINKS" == "yes" ]; then
|
|
|
|
|
RSYNC_ARGS=$RSYNC_ARGS" -K"
|
|
|
|
|
RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -K"
|
|
|
|
|
fi
|
|
|
|
|
if [ "$RSYNC_OPTIONAL_ARGS" != "" ]; then
|
|
|
|
|
RSYNC_ARGS=$RSYNC_ARGS" "$RSYNC_OPTIONAL_ARGS
|
|
|
|
|
RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" "$RSYNC_OPTIONAL_ARGS
|
|
|
|
|
fi
|
|
|
|
|
if [ "$PRESERVE_HARDLINKS" == "yes" ]; then
|
|
|
|
|
RSYNC_ARGS=$RSYNC_ARGS" -H"
|
|
|
|
|
RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" -H"
|
|
|
|
|
fi
|
|
|
|
|
if [ "$CHECKSUM" == "yes" ]; then
|
|
|
|
|
RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum"
|
|
|
|
|
fi
|
|
|
|
|
if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then
|
|
|
|
|
RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH"
|
|
|
|
|
RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" --bwlimit=$BANDWIDTH"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ "$PARTIAL" == "yes" ]; then
|
|
|
|
|
RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\""
|
|
|
|
|
RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\""
|
|
|
|
|
RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\""
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ "$DELTA_COPIES" != "no" ]; then
|
|
|
|
|
RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file"
|
|
|
|
|
RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" --no-whole-file"
|
|
|
|
|
else
|
|
|
|
|
RSYNC_ARGS=$RSYNC_ARGS" --whole-file"
|
|
|
|
|
RSYNC_DEFAULT_ARGS=$RSYNC_DEFAULT_ARGS" --whole-file"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# Set compression options again after we know what remote OS we are dealing with
|
|
|
|
|
SetCompression
|
|
|
|
|
|
|
|
|
|
RSYNC_DEFAULT_ARGS="-r $RSYNC_DEFAULT_ARGS"
|
|
|
|
|
RSYNC_DEFAULT_NONRECURSIVE_ARGS="-d $RSYNC_DEFAULT_ARGS"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
## IFS debug function
|
|
|
|
@ -3326,9 +3329,9 @@ function treeList {
|
|
|
|
|
if [ "$REMOTE_OPERATION" == "yes" ] && [ "$replicaType" == "${TARGET[$__type]}" ]; then
|
|
|
|
|
CheckConnectivity3rdPartyHosts
|
|
|
|
|
CheckConnectivityRemoteHost
|
|
|
|
|
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escapedReplicaPath\" 2> \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP\" | (grep -E \"^-|^d|^l\" || :) | (sed $SED_REGEX_ARG 's/^.{10} +[0-9,]+ [0-9/]{10} [0-9:]{8} //' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\""
|
|
|
|
|
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_DEFAULT_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escapedReplicaPath\" 2> \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP\" | (grep -E \"^-|^d|^l\" || :) | (sed $SED_REGEX_ARG 's/^.{10} +[0-9,]+ [0-9/]{10} [0-9:]{8} //' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\""
|
|
|
|
|
else
|
|
|
|
|
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replicaPath\" 2> \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP\" | (grep -E \"^-|^d|^l\" || :) | (sed $SED_REGEX_ARG 's/^.{10} +[0-9,]+ [0-9/]{10} [0-9:]{8} //' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\""
|
|
|
|
|
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_DEFAULT_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replicaPath\" 2> \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP\" | (grep -E \"^-|^d|^l\" || :) | (sed $SED_REGEX_ARG 's/^.{10} +[0-9,]+ [0-9/]{10} [0-9:]{8} //' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\""
|
|
|
|
|
fi
|
|
|
|
|
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
|
|
|
|
|
eval "$rsyncCmd"
|
|
|
|
@ -3617,9 +3620,9 @@ function syncAttrs {
|
|
|
|
|
if [ "$REMOTE_OPERATION" == "yes" ]; then
|
|
|
|
|
CheckConnectivity3rdPartyHosts
|
|
|
|
|
CheckConnectivityRemoteHost
|
|
|
|
|
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" -i -n $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE \"$initiatorReplica\" $REMOTE_USER@$REMOTE_HOST:\"$targetReplica\" >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1 &"
|
|
|
|
|
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" -i -n $RSYNC_DEFAULT_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE \"$initiatorReplica\" $REMOTE_USER@$REMOTE_HOST:\"$targetReplica\" >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1 &"
|
|
|
|
|
else
|
|
|
|
|
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" -i -n $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE \"$initiatorReplica\" \"$targetReplica\" >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1 &"
|
|
|
|
|
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" -i -n $RSYNC_DEFAULT_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE \"$initiatorReplica\" \"$targetReplica\" >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1 &"
|
|
|
|
|
fi
|
|
|
|
|
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
|
|
|
|
|
eval "$rsyncCmd"
|
|
|
|
@ -3765,12 +3768,12 @@ function syncUpdate {
|
|
|
|
|
CheckConnectivity3rdPartyHosts
|
|
|
|
|
CheckConnectivityRemoteHost
|
|
|
|
|
if [ "$sourceReplica" == "${INITIATOR[$__type]}" ]; then
|
|
|
|
|
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $backupArgs --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$sourceReplica${INITIATOR[$__deletedListFile]}\" --exclude-from=\"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$destinationReplica${INITIATOR[$__deletedListFile]}\" \"$sourceDir\" $REMOTE_USER@$REMOTE_HOST:\"$escDestDir\" >> $RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP 2>&1"
|
|
|
|
|
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_DEFAULT_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $backupArgs --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$sourceReplica${INITIATOR[$__deletedListFile]}\" --exclude-from=\"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$destinationReplica${INITIATOR[$__deletedListFile]}\" \"$sourceDir\" $REMOTE_USER@$REMOTE_HOST:\"$escDestDir\" >> $RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP 2>&1"
|
|
|
|
|
else
|
|
|
|
|
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $backupArgs --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$destinationReplica${INITIATOR[$__deletedListFile]}\" --exclude-from=\"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$sourceReplica${INITIATOR[$__deletedListFile]}\" $REMOTE_USER@$REMOTE_HOST:\"$escSourceDir\" \"$destDir\" >> $RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP 2>&1"
|
|
|
|
|
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_DEFAULT_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $backupArgs --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$destinationReplica${INITIATOR[$__deletedListFile]}\" --exclude-from=\"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$sourceReplica${INITIATOR[$__deletedListFile]}\" $REMOTE_USER@$REMOTE_HOST:\"$escSourceDir\" \"$destDir\" >> $RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP 2>&1"
|
|
|
|
|
fi
|
|
|
|
|
else
|
|
|
|
|
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $SYNC_OPTS $backupArgs --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$sourceReplica${INITIATOR[$__deletedListFile]}\" --exclude-from=\"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$destinationReplica${INITIATOR[$__deletedListFile]}\" \"$sourceDir\" \"$destDir\" >> $RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP 2>&1"
|
|
|
|
|
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_DEFAULT_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $SYNC_OPTS $backupArgs --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$sourceReplica${INITIATOR[$__deletedListFile]}\" --exclude-from=\"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$destinationReplica${INITIATOR[$__deletedListFile]}\" \"$sourceDir\" \"$destDir\" >> $RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP 2>&1"
|
|
|
|
|
fi
|
|
|
|
|
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
|
|
|
|
|
eval "$rsyncCmd"
|
|
|
|
|