diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 633b7a7..960ae87 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -15,9 +15,11 @@ jobs: run: | sudo apt-get install inotify-tools acl - name: Execute tests and generate coverage report - env: - RUNNING_ON_GITHUB_ACTIONS: true run: | - sudo bash ./dev/tests/run_tests.sh + export RUNNING_ON_GITHUB_ACTIONS=true + export SSH_PORT=22 + echo "Running on github actions: ${RUNNING_ON_GITHUB_ACTIONS}" + echo "Running on ssh port ${SSH_PORT}" + sudo -E bash ./dev/tests/run_tests.sh - name: Upload Coverage to Codecov uses: codecov/codecov-action@v1 diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 2bae489..a22c7f5 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -20,9 +20,9 @@ jobs: brew install fswatch echo "/usr/local/bin" >> $GITHUB_PATH - name: Execute tests and generate coverage report - env: - RUNNING_ON_GITHUB_ACTIONS: true run: | - sudo bash ./dev/tests/run_tests.sh + export RUNNING_ON_GITHUB_ACTIONS=true + export SSH_PORT=22 + sudo -E bash ./dev/tests/run_tests.sh - name: Upload Coverage to Codecov uses: codecov/codecov-action@v1 diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 889f46b..61521ef 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -16,12 +16,14 @@ jobs: additional-packages: dos2unix rsync + openssh-server - name: Execute tests and generate coverage report shell: wsl-bash {0} - env: - RUNNING_ON_GITHUB_ACTIONS: true run: | - find . -type f -print0 | xargs -0 -n 1 -P 4 dos2unix + export RUNNING_ON_GITHUB_ACTIONS=true + export SSH_PORT=22 + find ./ -type f ! -path "./.git/*" -print0 | xargs -0 -n 1 -P 4 dos2unix + service ssh start ./dev/tests/run_tests.sh - name: Upload Coverage to Codecov uses: codecov/codecov-action@v1 diff --git a/CHANGELOG.md b/CHANGELOG.md old mode 100644 new mode 100755 index 0c9058a..cffa9a5 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,14 @@ ## RECENT CHANGES -### dd Mmm YYYY: To be done +### Current master - Make --log-conflicts non experimental (randomly fails) - ! new option FORCE_CONFLICT_PREVALANCE which will always use Initiator or Target, regardless of best time - ! target-helper: destination mails etc on target, also, no cmd after on configs -### dd Mmm YYYY: osync v1.3 release (for full changelog since v1.2 branch see all v1.3-beta/RC entries) +### 16 June 2023: osync v1.3 release (for full changelog since v1.2 branch see all v1.3-beta/RC entries) +- Fix for new RSYNC protocol - New options ALWAYS_SEND_MAILS to allow sending logs regardless of execution states ### 29 June 2020: osync v1.3-RC1 release diff --git a/README.md b/README.md index 624ac84..06af294 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,12 @@ -# osync [![Build Status](https://travis-ci.org/deajan/osync.svg?branch=master)](https://travis-ci.org/deajan/osync) [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) [![GitHub Release](https://img.shields.io/github/release/deajan/osync.svg?label=Latest)](https://github.com/deajan/osync/releases/latest) [![Percentage of issues still open](http://isitmaintained.com/badge/open/deajan/osync.svg)](http://isitmaintained.com/project/deajan/osync "Percentage of issues still open") [![Codacy Badge](https://api.codacy.com/project/badge/Grade/651acb2fd64642eb91078ba523b7f887)](https://www.codacy.com/app/ozy/osync?utm_source=github.com&utm_medium=referral&utm_content=deajan/osync&utm_campaign=Badge_Grade) +# osync +[![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) +[![GitHub Release](https://img.shields.io/github/release/deajan/osync.svg?label=Latest)](https://github.com/deajan/osync/releases/latest) +[![Percentage of issues still open](http://isitmaintained.com/badge/open/deajan/osync.svg)](http://isitmaintained.com/project/deajan/osync "Percentage of issues still open") +[![Codacy Badge](https://api.codacy.com/project/badge/Grade/651acb2fd64642eb91078ba523b7f887)](https://www.codacy.com/app/ozy/osync?utm_source=github.com&utm_medium=referral&utm_content=deajan/osync&utm_campaign=Badge_Grade) +[![linux tests](https://github.com/deajan/osync/actions/workflows/linux.yml/badge.svg)](https://github.com/deajan/osync/actions/workflows/linux.yml/badge.svg) +[![windows tests](https://github.com/deajan/osync/actions/workflows/windows.yml/badge.svg)](https://github.com/deajan/osync/actions/workflows/windows.yml/badge.svg) +[![macos tests](https://github.com/deajan/osync/actions/workflows/macos.yml/badge.svg)](https://github.com/deajan/osync/actions/workflows/macos.yml/badge.svg) + A two way filesync script running on bash Linux, BSD, Android, MacOSX, Cygwin, MSYS2, Win10 bash and virtually any system supporting bash). File synchronization is bidirectional, and can be run manually, as scheduled task, or triggered on file changes in daemon mode. @@ -30,8 +38,8 @@ osync uses pidlocks to prevent multiple concurrent sync processes on/to the same You may launch concurrent sync processes on the same system but as long as the replicas to synchronize are different. Multiple osync tasks may be launched sequentially by osync osync-batch tool. -Currently, it has been tested on CentOS 5.x, 6.x, 7.x, Fedora 22-25, Debian 6-8, Linux Mint 14-18, Ubuntu 12.04-12.10, FreeBSD 8.3-11, Mac OS X, QTS 4.5.1(x86) and pfSense 2.3.x. -Microsoft Windows is supported via MSYS or Cygwin and now via Windows 10 bash. +Currently, it has been tested on CentOS 5.x, 6.x, 7.x, AlmaLinux 9, Fedora 22-25, Debian 6-11, Linux Mint 14-18, Ubuntu 12.04-22.04, FreeBSD 8.3-11, Mac OS X, QTS 4.5.1(x86) and pfSense 2.3.x. +Microsoft Windows is supported via MSYS, Cygwin and or via WSL. Android support works via Termux. Some users also have successfully used osync on Gentoo and created an openRC init scriptt for it. @@ -40,12 +48,6 @@ Some users also have successfully used osync on Gentoo and created an openRC ini osync has been designed to not delete any data, but rather make backups of conflictual files or soft deletes. Nevertheless, you should always have a neat backup of your data before trying a new sync tool. -You may get osync on github (stable or latest dev snapshot) or on the author's site (stable version) -Getting osync via author's site on - - $ wget http://www.netpower.fr/projects/osync/osync.v1.2.tar.gz - $ tar xvf osync.v1.2.tar.gz - Getting osync via github (remove the -b "stable" if you want latest dev snapshot) $ git clone -b "stable" https://github.com/deajan/osync @@ -74,11 +76,11 @@ Archlinux packages are available at Since osync v1.1 the config file format has changed in semantics and adds new config options. Also, master is now called initiator and slave is now called target. -osync v1.2 also added multiple new configuration options. +osync v1.3 also added multiple new configuration options. -You can upgrade all v1.0x-v1.2-dev config files by running the upgrade script +You can upgrade all v1.0x-v1.3-dev config files by running the upgrade script - $ ./upgrade-v1.0x-v1.2x.sh /etc/osync/your-config-file.conf + $ ./upgrade-v1.0x-v1.3x.sh /etc/osync/your-config-file.conf The script will backup your config file, update it's content and try to connect to initiator and target replicas to update the state dir. diff --git a/dev/common_install.sh b/dev/common_install.sh index 6f57262..217b937 100644 --- a/dev/common_install.sh +++ b/dev/common_install.sh @@ -10,7 +10,7 @@ PROGRAM_BINARY=$PROGRAM".sh" PROGRAM_BATCH=$PROGRAM"-batch.sh" SSH_FILTER="ssh_filter.sh" -SCRIPT_BUILD=2020112901 +SCRIPT_BUILD=2023061101 INSTANCE_ID="installer-$SCRIPT_BUILD" ## osync / obackup / pmocr / zsnap install script @@ -80,15 +80,26 @@ function SetLocalOSSettings { } function GetInit { + init="none" if [ -f /sbin/openrc-run ]; then init="openrc" Logger "Detected openrc." "NOTICE" + elif [ -f /usr/lib/systemd/systemd ]; then + init="systemd" + Logger "Detected systemd." "NOTICE" elif [ -f /sbin/init ]; then - if file /sbin/init | grep systemd > /dev/null; then - init="systemd" - Logger "Detected systemd." "NOTICE" + if type -p file > /dev/null 2>&1; then + if file /sbin/init | grep systemd > /dev/null; then + init="systemd" + Logger "Detected systemd." "NOTICE" + else + init="initV" + fi else init="initV" + fi + + if [ $init == "initV" ]; then Logger "Detected initV." "NOTICE" fi else diff --git a/dev/debug_osync.sh b/dev/debug_osync.sh index dbee41d..7702c4e 100755 --- a/dev/debug_osync.sh +++ b/dev/debug_osync.sh @@ -4,10 +4,10 @@ #Check dryruns with nosuffix mode for timestampList PROGRAM="osync" # Rsync based two way sync engine with fault tolerance -AUTHOR="(C) 2013-2022 by Orsiris de Jong" +AUTHOR="(C) 2013-2023 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" -PROGRAM_VERSION=1.3.0-rc3 -PROGRAM_BUILD=2021062901 +PROGRAM_VERSION=1.3.0 +PROGRAM_BUILD=2023061401 IS_STABLE=true CONFIG_FILE_REVISION_REQUIRED=1.3.0 @@ -42,8 +42,8 @@ CONFIG_FILE_REVISION_REQUIRED=1.3.0 # UnlockReplicas yes #__WITH_PARANOIA_DEBUG # CleanUp no #__WITH_PARANOIA_DEBUG -_OFUNCTIONS_VERSION=2.4.3 -_OFUNCTIONS_BUILD=2022050801 +_OFUNCTIONS_VERSION=2.5.1 +_OFUNCTIONS_BUILD=2023061401 _OFUNCTIONS_BOOTSTRAP=true if ! type "$BASH" > /dev/null; then @@ -66,6 +66,8 @@ _LOGGER_SILENT=false _LOGGER_VERBOSE=false _LOGGER_ERR_ONLY=false _LOGGER_PREFIX="date" +_LOGGER_WRITE_PARTIAL_LOGS=false # Writes partial log files to /tmp so sending logs via alerts can feed on them +_OFUNCTIONS_SHOW_SPINNER=true # Show spinner in ExecTasks function if [ "$KEEP_LOGGING" == "" ]; then KEEP_LOGGING=1801 fi @@ -174,7 +176,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -290,18 +292,18 @@ function Logger { if [ "$level" == "CRITICAL" ]; then _Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true ERROR_ALERT=true - # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. Need to keep this flag. - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. We need to create these flag files for ERROR_ALERT / WARN_ALERT to be picked up by Alert + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "ERROR" ]; then _Logger "$prefix($level):$value" "$prefix\e[91m$value\e[0m" true ERROR_ALERT=true - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "WARN" ]; then _Logger "$prefix($level):$value" "$prefix\e[33m$value\e[0m" true WARN_ALERT=true - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID.$TSTAMP" + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.WARN_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "NOTICE" ]; then if [ "$_LOGGER_ERR_ONLY" != true ]; then @@ -423,11 +425,11 @@ function GenericTrapQuit { local exitcode=0 # Get ERROR / WARN alert flags from subprocesses that call Logger - if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.WARN_ALERT.$SCRIPT_PID.$TSTAMP" ]; then WARN_ALERT=true exitcode=2 fi - if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" ]; then ERROR_ALERT=true exitcode=1 fi @@ -487,7 +489,11 @@ function SendAlert { fi fi - body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of current log"$'\n\n'"$(tail -n 1000 "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP")" + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ]; then + body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of current log"$'\n\n'"$(tail -n 1000 "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP")" + else + body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of current log"$'\n\n'"$(tail -n 1000 "$LOG_FILE")" + fi if [ $ERROR_ALERT == true ]; then subject="Error alert for $INSTANCE_ID" @@ -498,7 +504,7 @@ function SendAlert { fi if [ $runAlert == true ]; then - subject="Currently runing - $subject" + subject="Currently running - $subject" else subject="Finished run - $subject" fi @@ -980,7 +986,7 @@ function ExecTasks { # soft / hard execution time checks that needs to be a subfunction since it is called both from main loop and from parallelExec sub loop function _ExecTasksTimeCheck { - if [ $spinner == true ]; then + if [ $spinner == true ] && [ "$_OFUNCTIONS_SHOW_SPINNER" != false ]; then Spinner fi if [ $counting == true ]; then @@ -2248,6 +2254,16 @@ function InitRemoteOSDependingSettings { ## Set rsync default arguments (complete with -r or -d depending on recursivity later) RSYNC_DEFAULT_ARGS="-ltD -8" + + ## NPF-MOD: Regarding #242, we need to add --old-args if rsync > 3.2.3 + #rsync_version=$("${RSYNC_EXECUTABLE}" --version 2>/dev/null| head -1 | awk '{print $3}') + #if [ $(VerComp $rsync_version 3.2.3) -eq 1 ]; then + # RSYNC_DEFAULT_ARGS="$RSYNC_DEFAULT_ARGS --old-args" + #fi + # NPF-MOD: Strangely enough, also happens on RHEL7 rsync 3.1.1 + # Let's resolve this easier + export RSYNC_OLD_ARGS=1 + if [ "$_DRYRUN" == true ]; then RSYNC_DRY_ARG="-n" DRY_WARNING="/!\ DRY RUN " @@ -2255,6 +2271,7 @@ function InitRemoteOSDependingSettings { RSYNC_DRY_ARG="" fi + RSYNC_ATTR_ARGS="" if [ "$PRESERVE_PERMISSIONS" != false ]; then RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -p" @@ -2477,16 +2494,8 @@ function FileMove () { mv -f "$source" "$dest" return $? elif [ -w "$source" ]; then - if [ -f "$dest" ]; then # for files we don't need recursive delete - rm -f "$dest" - elif [ -d "$dest" ]; then # for directories we need recursive delete - rm -rf "$dest" - fi - if [ -f "$source" ]; then - cp -p "$source" "$dest" && rm -f "$source" # for files we don't need recursive copy & delete - elif [ -d "$source" ]; then - cp -rp "$source" "$dest" && rm -rf "$source" # for directories we need recursive copy & delete - fi + [ -f "$dest" ] && rm -f "$dest" + cp -p "$source" "$dest" && rm -f "$source" return $? else return -1 @@ -2641,6 +2650,7 @@ _OFUNCTIONS_BOOTSTRAP=true [ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0) or build it with merge.sh" && exit 1 _LOGGER_PREFIX="time" +_LOGGER_WRITE_PARTIAL_LOGS=true ## Working directory. This directory exists in any replica and contains state files, backups, soft deleted files etc OSYNC_DIR=".osync_workdir" @@ -3056,7 +3066,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -3485,7 +3495,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -4118,7 +4128,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -4768,7 +4778,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -4873,16 +4883,8 @@ function FileMove () { mv -f "$source" "$dest" return $? elif [ -w "$source" ]; then - if [ -f "$dest" ]; then # for files we don't need recursive delete - rm -f "$dest" - elif [ -d "$dest" ]; then # for directories we need recursive delete - rm -rf "$dest" - fi - if [ -f "$source" ]; then - cp -p "$source" "$dest" && rm -f "$source" # for files we don't need recursive copy & delete - elif [ -d "$source" ]; then - cp -rp "$source" "$dest" && rm -rf "$source" # for directories we need recursive copy & delete - fi + [ -f "$dest" ] && rm -f "$dest" + cp -p "$source" "$dest" && rm -f "$source" return $? else return -1 @@ -5783,7 +5785,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -6071,7 +6073,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -6596,6 +6598,7 @@ function Usage { echo "--no-prefix Will suppress time / date suffix from output" echo "--silent Will run osync without any output to stdout, used for cron jobs" echo "--errors-only Output only errors (can be combined with silent or verbose)" + echo "--non-interactive Don't show running animation in cron / service mode" echo "--summary Outputs a list of transferred / deleted files at the end of the run" echo "--log-conflicts [EXPERIMENTAL] Outputs a list of conflicted files" echo "--alert-conflicts Send an email if conflictual files found (implies --log-conflicts)" @@ -6872,6 +6875,10 @@ function GetCommandlineArguments { opts=$opts" --errors-only" _LOGGER_ERR_ONLY=true ;; + --non-interactive) + opts=$opts" --non-interactive" + _OFUNCTIONS_SHOW_SPINNER=false + ;; --summary) opts=$opts" --summary" _SUMMARY=true diff --git a/dev/n_osync.sh b/dev/n_osync.sh old mode 100644 new mode 100755 index 1ec9022..ef68e14 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -4,10 +4,10 @@ #Check dryruns with nosuffix mode for timestampList PROGRAM="osync" # Rsync based two way sync engine with fault tolerance -AUTHOR="(C) 2013-2022 by Orsiris de Jong" +AUTHOR="(C) 2013-2023 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" -PROGRAM_VERSION=1.3.0-rc3 -PROGRAM_BUILD=2021062901 +PROGRAM_VERSION=1.3.0 +PROGRAM_BUILD=2023061401 IS_STABLE=true CONFIG_FILE_REVISION_REQUIRED=1.3.0 @@ -48,6 +48,7 @@ include #### _OFUNCTIONS_BOOTSTRAP SUBSET #### [ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0) or build it with merge.sh" && exit 1 _LOGGER_PREFIX="time" +_LOGGER_WRITE_PARTIAL_LOGS=true ## Working directory. This directory exists in any replica and contains state files, backups, soft deleted files etc OSYNC_DIR=".osync_workdir" @@ -2898,6 +2899,7 @@ function Usage { echo "--no-prefix Will suppress time / date suffix from output" echo "--silent Will run osync without any output to stdout, used for cron jobs" echo "--errors-only Output only errors (can be combined with silent or verbose)" + echo "--non-interactive Don't show running animation in cron / service mode" echo "--summary Outputs a list of transferred / deleted files at the end of the run" echo "--log-conflicts [EXPERIMENTAL] Outputs a list of conflicted files" echo "--alert-conflicts Send an email if conflictual files found (implies --log-conflicts)" @@ -3174,6 +3176,10 @@ function GetCommandlineArguments { opts=$opts" --errors-only" _LOGGER_ERR_ONLY=true ;; + --non-interactive) + opts=$opts" --non-interactive" + _OFUNCTIONS_SHOW_SPINNER=false + ;; --summary) opts=$opts" --summary" _SUMMARY=true diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh old mode 100644 new mode 100755 index 71eb703..205803d --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -14,6 +14,7 @@ ## _LOGGER_VERBOSE=true/false ## _LOGGER_ERR_ONLY=true/false ## _LOGGER_PREFIX="date"/"time"/"" +## _LOGGER_WRITE_PARTIAL_LOGS=true/false ## Also, set the following trap in order to clean temporary files ## trap GenericTrapQuit TERM EXIT HUP QUIT @@ -30,8 +31,8 @@ #### OFUNCTIONS FULL SUBSET #### #### OFUNCTIONS MINI SUBSET #### #### OFUNCTIONS MICRO SUBSET #### -_OFUNCTIONS_VERSION=2.4.3 -_OFUNCTIONS_BUILD=2022050801 +_OFUNCTIONS_VERSION=2.5.1 +_OFUNCTIONS_BUILD=2023061401 #### _OFUNCTIONS_BOOTSTRAP SUBSET #### _OFUNCTIONS_BOOTSTRAP=true #### _OFUNCTIONS_BOOTSTRAP SUBSET END #### @@ -56,6 +57,8 @@ _LOGGER_SILENT=false _LOGGER_VERBOSE=false _LOGGER_ERR_ONLY=false _LOGGER_PREFIX="date" +_LOGGER_WRITE_PARTIAL_LOGS=false # Writes partial log files to /tmp so sending logs via alerts can feed on them +_OFUNCTIONS_SHOW_SPINNER=true # Show spinner in ExecTasks function if [ "$KEEP_LOGGING" == "" ]; then KEEP_LOGGING=1801 fi @@ -172,7 +175,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -289,18 +292,18 @@ function Logger { if [ "$level" == "CRITICAL" ]; then _Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true ERROR_ALERT=true - # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. Need to keep this flag. - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. We need to create these flag files for ERROR_ALERT / WARN_ALERT to be picked up by Alert + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "ERROR" ]; then _Logger "$prefix($level):$value" "$prefix\e[91m$value\e[0m" true ERROR_ALERT=true - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "WARN" ]; then _Logger "$prefix($level):$value" "$prefix\e[33m$value\e[0m" true WARN_ALERT=true - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID.$TSTAMP" + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.WARN_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "NOTICE" ]; then if [ "$_LOGGER_ERR_ONLY" != true ]; then @@ -426,11 +429,11 @@ function GenericTrapQuit { local exitcode=0 # Get ERROR / WARN alert flags from subprocesses that call Logger - if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.WARN_ALERT.$SCRIPT_PID.$TSTAMP" ]; then WARN_ALERT=true exitcode=2 fi - if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" ]; then ERROR_ALERT=true exitcode=1 fi @@ -494,7 +497,11 @@ function SendAlert { fi fi - body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of current log"$'\n\n'"$(tail -n 1000 "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP")" + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ]; then + body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of current log"$'\n\n'"$(tail -n 1000 "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP")" + else + body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of current log"$'\n\n'"$(tail -n 1000 "$LOG_FILE")" + fi if [ $ERROR_ALERT == true ]; then subject="Error alert for $INSTANCE_ID" @@ -505,7 +512,7 @@ function SendAlert { fi if [ $runAlert == true ]; then - subject="Currently runing - $subject" + subject="Currently running - $subject" else subject="Finished run - $subject" fi @@ -989,7 +996,7 @@ function ExecTasks { # soft / hard execution time checks that needs to be a subfunction since it is called both from main loop and from parallelExec sub loop function _ExecTasksTimeCheck { - if [ $spinner == true ]; then + if [ $spinner == true ] && [ "$_OFUNCTIONS_SHOW_SPINNER" != false ]; then Spinner fi if [ $counting == true ]; then @@ -2268,6 +2275,16 @@ function InitRemoteOSDependingSettings { ## Set rsync default arguments (complete with -r or -d depending on recursivity later) RSYNC_DEFAULT_ARGS="-ltD -8" + + ## NPF-MOD: Regarding #242, we need to add --old-args if rsync > 3.2.3 + #rsync_version=$("${RSYNC_EXECUTABLE}" --version 2>/dev/null| head -1 | awk '{print $3}') + #if [ $(VerComp $rsync_version 3.2.3) -eq 1 ]; then + # RSYNC_DEFAULT_ARGS="$RSYNC_DEFAULT_ARGS --old-args" + #fi + # NPF-MOD: Strangely enough, also happens on RHEL7 rsync 3.1.1 + # Let's resolve this easier + export RSYNC_OLD_ARGS=1 + if [ "$_DRYRUN" == true ]; then RSYNC_DRY_ARG="-n" DRY_WARNING="/!\ DRY RUN " @@ -2275,6 +2292,7 @@ function InitRemoteOSDependingSettings { RSYNC_DRY_ARG="" fi + RSYNC_ATTR_ARGS="" if [ "$PRESERVE_PERMISSIONS" != false ]; then RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -p" @@ -2504,16 +2522,8 @@ function FileMove () { mv -f "$source" "$dest" return $? elif [ -w "$source" ]; then - if [ -f "$dest" ]; then # for files we don't need recursive delete - rm -f "$dest" - elif [ -d "$dest" ]; then # for directories we need recursive delete - rm -rf "$dest" - fi - if [ -f "$source" ]; then - cp -p "$source" "$dest" && rm -f "$source" # for files we don't need recursive copy & delete - elif [ -d "$source" ]; then - cp -rp "$source" "$dest" && rm -rf "$source" # for directories we need recursive copy & delete - fi + [ -f "$dest" ] && rm -f "$dest" + cp -p "$source" "$dest" && rm -f "$source" return $? else return -1 diff --git a/dev/tests/run_tests.sh b/dev/tests/run_tests.sh index 4f9ce17..bb1da19 100755 --- a/dev/tests/run_tests.sh +++ b/dev/tests/run_tests.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# osync test suite 2022070702 +# osync test suite 2023061401 # Allows the following environment variables @@ -50,20 +50,23 @@ if [ "$SKIP_REMOTE" = "" ]; then SKIP_REMOTE=false + REMOTE_USER=root fi +homedir=$(eval echo ~${REMOTE_USER}) + # drupal servers are often unreachable for whetever reason or give 0 bytes files #LARGE_FILESET_URL="http://ftp.drupal.org/files/projects/drupal-8.2.2.tar.gz" LARGE_FILESET_URL="http://www.netpower.fr/sites/default/files/osync-test-files.tar.gz" -# Fakeroot for install / uninstall and test of executables -FAKEROOT="${HOME}/osync_test_install" - OSYNC_DIR="$(pwd)" OSYNC_DIR=${OSYNC_DIR%%/dev*} DEV_DIR="$OSYNC_DIR/dev" TESTS_DIR="$DEV_DIR/tests" +# Fakeroot for install / uninstall and test of executables +FAKEROOT="${homedir}/osync_test_install" + CONF_DIR="$TESTS_DIR/conf" LOCAL_CONF="local.conf" REMOTE_CONF="remote.conf" @@ -76,7 +79,7 @@ OSYNC_UPGRADE="upgrade-v1.0x-v1.3x.sh" TMP_FILE="$DEV_DIR/tmp" -OSYNC_TESTS_DIR="${HOME}/osync-tests" +OSYNC_TESTS_DIR="${homedir}/osync-tests" INITIATOR_DIR="$OSYNC_TESTS_DIR/initiator" TARGET_DIR="$OSYNC_TESTS_DIR/target" OSYNC_WORKDIR=".osync_workdir" @@ -93,46 +96,52 @@ PRIVKEY_NAME="id_rsa_local_osync_tests" PUBKEY_NAME="${PRIVKEY_NAME}.pub" function SetupSSH { - echo "Setting up an ssh key to ${HOME}/.ssh/${PRIVKEY_NAME}" - echo -e 'y\n'| ssh-keygen -t rsa -b 2048 -N "" -f "${HOME}/.ssh/${PRIVKEY_NAME}" - - SSH_AUTH_LINE="from=\"*\",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,command=\"$FAKEROOT/usr/local/bin/ssh_filter.sh SomeAlphaNumericToken9\" $(cat ${HOME}/.ssh/${PUBKEY_NAME})" - echo "ls -alh ${HOME}" - ls -alh "${HOME}" - echo "ls -alh ${HOME}/.ssh" - ls -alh "${HOME}/.ssh" - - if [ -f "${HOME}/.ssh/authorized_keys" ]; then - if ! grep "$(cat ${HOME}/.ssh/${PUBKEY_NAME})" "${HOME}/.ssh/authorized_keys"; then - echo "$SSH_AUTH_LINE" >> "${HOME}/.ssh/authorized_keys" + echo "Setting up an ssh key to ${homedir}/.ssh/${PRIVKEY_NAME}" + echo -e 'y\n'| ssh-keygen -t rsa -b 2048 -N "" -f "${homedir}/.ssh/${PRIVKEY_NAME}" + + + + SSH_AUTH_LINE="from=\"*\",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,command=\"$FAKEROOT/usr/local/bin/ssh_filter.sh SomeAlphaNumericToken9\" $(cat ${homedir}/.ssh/${PUBKEY_NAME})" + echo "ls -alh ${homedir}" + ls -alh "${homedir}" + echo "ls -alh ${homedir}/.ssh" + ls -alh "${homedir}/.ssh" + + if [ -f "${homedir}/.ssh/authorized_keys" ]; then + if ! grep "$(cat ${homedir}/.ssh/${PUBKEY_NAME})" "${homedir}/.ssh/authorized_keys"; then + echo "Adding auth line in authorized_keys file ${homedir}/.ssh/authorized_keys" + echo "$SSH_AUTH_LINE" >> "${homedir}/.ssh/authorized_keys" fi else - echo "$SSH_AUTH_LINE" >> "${HOME}/.ssh/authorized_keys" + echo "Creating authorized_keys file ${homedir}/.ssh/authorized_keys" + echo "$SSH_AUTH_LINE" >> "${homedir}/.ssh/authorized_keys" fi - chmod 600 "${HOME}/.ssh/authorized_keys" + chmod 600 "${homedir}/.ssh/authorized_keys" # Add localhost to known hosts so self connect works if [ -z "$(ssh-keygen -F localhost)" ]; then - ssh-keyscan -H localhost >> "${HOME}/.ssh/known_hosts" + ssh-keyscan -H localhost >> "${homedir}/.ssh/known_hosts" fi - # Update remote conf files with SSH port - sed -i.tmp 's#ssh://.*@localhost:[0-9]*/${HOME}/osync-tests/target#ssh://'$REMOTE_USER'@localhost:'$SSH_PORT'/${HOME}/osync-tests/target#' "$CONF_DIR/$REMOTE_CONF" + # Update remote conf files with SSH port and file id location + sed -i.tmp 's#ssh://.*@localhost:[0-9]*/${HOME}/osync-tests/target#ssh://'$REMOTE_USER'@localhost:'$SSH_PORT'/'${homedir}'/osync-tests/target#' "$CONF_DIR/$REMOTE_CONF" + sed -i.tmp2 's#SSH_RSA_PRIVATE_KEY="${HOME}/.ssh/id_rsa_local_osync_tests"#SSH_RSA_PRIVATE_KEY="'${homedir}'/.ssh/id_rsa_local_osync_tests"#' "$CONF_DIR/$REMOTE_CONF" - echp "ls -alh ${HOME}/.ssh" - ls -alh "${HOME}/.ssh" - echo "cat ${HOME}/.ssh.authorized_keys" - cat "${HOME}/.ssh/authorized_keys" + echo "ls -alh ${homedir}/.ssh" + ls -alh "${homedir}/.ssh" + echo "cat ${homedir}/.ssh/authorized_keys" + cat "${homedir}/.ssh/authorized_keys" echo "###" echo "END SETUP SSH" } function RemoveSSH { - if [ -f "${HOME}/.ssh/id_rsa_local_osync_tests" ]; then + echo "Now removing SSH keys" + if [ -f "${homedir}/.ssh/id_rsa_local_osync_tests" ]; then echo "Restoring SSH authorized_keys file" - sed -i.bak "s|.*$(cat "${HOME}/.ssh/id_rsa_local_osync_tests.pub")||g" "${HOME}/.ssh/authorized_keys" - rm -f "${HOME}/.ssh/{id_rsa_local_osync_tests.pub,id_rsa_local_osync_tests}" + sed -i.bak "s|.*$(cat "${homedir}/.ssh/id_rsa_local_osync_tests.pub")||g" "${homedir}/.ssh/authorized_keys" + rm -f "${homedir}/.ssh/{id_rsa_local_osync_tests.pub,id_rsa_local_osync_tests}" fi } @@ -192,8 +201,16 @@ function PrepareLocalDirs () { function oneTimeSetUp () { START_TIME=$SECONDS - mkdir -p "$FAKEROOT" + #echo "Running forced merge" + #cd "${DEV_DIR}" + #$SUDO_CMD ./merge.sh osync + echo "Setting security for files" + $SUDO_CMD find ${OSYNC_DIR} -exec chmod 755 {} \+ + echo "Show content of osync dir" + ls -alh ${OSYNC_DIR} + echo "Running install.sh from ${OSYNC_DIR}" + $SUDO_CMD ${OSYNC_DIR}/install.sh --no-stats --prefix="${FAKEROOT}" source "$DEV_DIR/ofunctions.sh" # Fix default umask because of ACL test that expects 0022 when creating test files @@ -206,16 +223,20 @@ function oneTimeSetUp () { # Set some travis related changes if [ "$RUNNING_ON_GITHUB_ACTIONS" == true ]; then echo "Running with GITHUB ACTIONS settings" - REMOTE_USER="runner" + #REMOTE_USER="runner" + REMOTE_USER="root" # WIP + homedir=$(eval echo ~${REMOTE_USER}) RHOST_PING=false SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_3RD_PARTY_HOSTS" "" SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_HOST_PING" false SetConfFileValue "$CONF_DIR/$OLD_CONF" "REMOTE_3RD_PARTY_HOSTS" "" SetConfFileValue "$CONF_DIR/$OLD_CONF" "REMOTE_HOST_PING" false + else echo "Running with local settings" REMOTE_USER="root" + homedir=$(eval echo ~${REMOTE_USER}) RHOST_PING=true SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_3RD_PARTY_HOSTS" "\"www.kernel.org www.google.com\"" SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_HOST_PING" true @@ -224,9 +245,21 @@ function oneTimeSetUp () { SetConfFileValue "$CONF_DIR/$OLD_CONF" "REMOTE_HOST_PING" true fi + + # Fix test directories for Github actions + SetConfFileValue "$CONF_DIR/$LOCAL_CONF" INITIATOR_SYNC_DIR "\"${homedir}/osync-tests/initiator\"" + SetConfFileValue "$CONF_DIR/$LOCAL_CONF" TARGET_SYNC_DIR "\"${homedir}/osync-tests/target\"" + + SetConfFileValue "$CONF_DIR/$REMOTE_CONF" INITIATOR_SYNC_DIR "\"${homedir}/osync-tests/initiator\"" + + SetConfFileValue "$CONF_DIR/$OLD_CONF" MASTER_SYNC_DIR "\"${homedir}/osync-tests/initiator\"" + SetConfFileValue "$CONF_DIR/$OLD_CONF" SLAVE_SYNC_DIR "\"${homedir}/osync-tests/target\"" + + # Get default ssh port from env if [ "$SSH_PORT" == "" ]; then SSH_PORT=22 + echo "Running with SSH_PORT=${SSH_PORT}" fi # Setup modes per test @@ -236,8 +269,8 @@ function oneTimeSetUp () { readonly __confRemote=3 osyncParameters=() - osyncParameters[$__quickLocal]="--initiator=$INITIATOR_DIR --target=$TARGET_DIR --instance-id=quicklocal" - osyncParameters[$__confLocal]="$CONF_DIR/$LOCAL_CONF" + osyncParameters[$__quickLocal]="--initiator=$INITIATOR_DIR --target=$TARGET_DIR --instance-id=quicklocal --non-interactive" + osyncParameters[$__confLocal]="$CONF_DIR/$LOCAL_CONF --non-interactive" osyncDaemonParameters=() @@ -248,8 +281,8 @@ function oneTimeSetUp () { # Do not check remote config on msys or cygwin since we don't have a local SSH server if [ "$LOCAL_OS" != "msys" ] && [ "$LOCAL_OS" != "Cygwin" ] && [ $SKIP_REMOTE != true ]; then - osyncParameters[$__quickRemote]="--initiator=$INITIATOR_DIR --target=ssh://localhost:$SSH_PORT/$TARGET_DIR --rsakey=${HOME}/.ssh/id_rsa_local_osync_tests --instance-id=quickremote --remote-token=SomeAlphaNumericToken9" - osyncParameters[$__confRemote]="$CONF_DIR/$REMOTE_CONF" + osyncParameters[$__quickRemote]="--initiator=$INITIATOR_DIR --target=ssh://localhost:$SSH_PORT/$TARGET_DIR --rsakey=${homedir}/.ssh/id_rsa_local_osync_tests --instance-id=quickremote --remote-token=SomeAlphaNumericToken9 --non-interactive" + osyncParameters[$__confRemote]="$CONF_DIR/$REMOTE_CONF --non-interactive" osyncDaemonParameters[$__remote]="$CONF_DIR/$REMOTE_CONF --on-changes" @@ -331,20 +364,28 @@ function test_SSH { failure=false + # Testing as "remote user" + echo "ls -alh ${homedir}/.ssh" + ls -alh "${homedir}/.ssh" + echo "Running SSH test as ${REMOTE_USER}" # SSH_PORT and SSH_USER are set by oneTimeSetup - ssh -i "${REMOTE_USER}/.ssh/${PUBKEY_NAME}" -p $SSH_PORT ${REMOTE_USER}@localhost "echo \"Remotely:\"; whoami; echo \"TEST OK\"" + $SUDO_CMD ssh -i "${homedir}/.ssh/${PRIVKEY_NAME}" -p $SSH_PORT ${REMOTE_USER}@localhost "env _REMOTE_TOKEN=SomeAlphaNumericToken9 echo \"Remotely:\"; whoami; echo \"TEST OK\"" if [ $? -ne 0 ]; then echo "SSH test failed" failure=true fi + + # Testing as current user + #echo "ls -alh ${homedir}/.ssh" + #ls -alh "${homedir}/.ssh" - echo "Running SSH test as $(whoami)" - ssh -i "$(whoami)/.ssh/${PUBKEY_NAME}" -p $SSH_PORT $(whoami)@localhost "echo \"Remotely:\"; whoami; echo \"TEST OK\"" - if [ $? -ne 0 ]; then - echo "SSH test failed" - failure=true - fi + #echo "Running SSH test as $(whoami)" + #$SUDO_CMD ssh -i "${homedir}/.ssh/${PRIVKEY_NAME}" -p $SSH_PORT $(whoami)@localhost "env _REMOTE_TOKEN=SomeAlphaNumericToken9 echo \"Remotely:\"; whoami; echo \"TEST OK\"" + #if [ $? -ne 0 ]; then + # echo "SSH test failed" + # failure=true + #fi if [ $failure == true ]; then exit 1 # Try to see if we can abort all tests @@ -382,7 +423,7 @@ function test_Merge () { assertEquals "Install failed" "0" $? } -function xtest_LargeFileSet () { +function test_LargeFileSet () { for i in "${osyncParameters[@]}"; do cd "$OSYNC_DIR" @@ -400,7 +441,7 @@ function xtest_LargeFileSet () { done } -function xtest_controlMaster () { +function test_controlMaster () { cd "$OSYNC_DIR" PrepareLocalDirs @@ -409,7 +450,7 @@ function xtest_controlMaster () { assertEquals "Running quick remote test with controlmaster enabled." "0" $? } -function xtest_Exclusions () { +function test_Exclusions () { # Will sync except php files # RSYNC_EXCLUDE_PATTERN="*.php" is set at runtime for quicksync and in config files for other runs @@ -437,7 +478,7 @@ function xtest_Exclusions () { done } -function xtest_Deletetion () { +function test_Deletetion () { local iFile1="$INITIATOR_DIR/i fic" local iFile2="$INITIATOR_DIR/i foc (something)" local tFile1="$TARGET_DIR/t fic" @@ -481,7 +522,7 @@ function xtest_Deletetion () { done } -function xtest_deletion_failure () { +function test_deletion_failure () { if [ "$LOCAL_OS" == "WinNT10" ] || [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then echo "Skipping deletion failure test as Win10 does not have chattr support." return 0 @@ -548,7 +589,7 @@ function xtest_deletion_failure () { done } -function xtest_skip_deletion () { +function test_skip_deletion () { local modes if [ "$OSYNC_MIN_VERSION" == "1" ]; then @@ -624,7 +665,7 @@ function xtest_skip_deletion () { SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "SKIP_DELETION" "" } -function xtest_handle_symlinks () { +function test_handle_symlinks () { if [ "$OSYNC_MIN_VERSION" == "1" ]; then echo "Skipping symlink tests as osync v1.1x didn't handle this." return 0 @@ -805,7 +846,7 @@ function xtest_handle_symlinks () { done } -function xtest_softdeletion_cleanup () { +function test_softdeletion_cleanup () { #declare -A files files=() @@ -882,7 +923,7 @@ function xtest_softdeletion_cleanup () { } -function xtest_FileAttributePropagation () { +function test_FileAttributePropagation () { if [ "$RUNNING_ON_GITHUB_ACTIONS" == true ]; then echo "Skipping FileAttributePropagation tests as travis does not support getfacl / setfacl." @@ -970,7 +1011,7 @@ function xtest_FileAttributePropagation () { SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "PRESERVE_XATTR" false } -function xtest_ConflictBackups () { +function test_ConflictBackups () { for i in "${osyncParameters[@]}"; do cd "$OSYNC_DIR" PrepareLocalDirs @@ -1006,7 +1047,7 @@ function xtest_ConflictBackups () { done } -function xtest_MultipleConflictBackups () { +function test_MultipleConflictBackups () { local additionalParameters @@ -1068,7 +1109,7 @@ function xtest_MultipleConflictBackups () { SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "CONFLICT_BACKUP_MULTIPLE" false } -function xtest_Locking () { +function test_Locking () { # local not running = resume # remote same instance_id = resume # remote different instance_id = stop @@ -1175,7 +1216,7 @@ function xtest_Locking () { SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "FORCE_STRANGER_LOCK_RESUME" false } -function xtest_ConflictDetetion () { +function test_ConflictDetetion () { # Tests compatible with v1.4+ if [ $OSYNC_MIN_VERSION -lt 4 ]; then @@ -1226,7 +1267,7 @@ function xtest_ConflictDetetion () { return 0 } -function xtest_WaitForTaskCompletion () { +function test_WaitForTaskCompletion () { local pids # Tests compatible with v1.1 syntax @@ -1320,7 +1361,7 @@ function xtest_WaitForTaskCompletion () { assertEquals "WaitForTaskCompletion test 5" "2" $? } -function xtest_ParallelExec () { +function test_ParallelExec () { if [ "$OSYNC_MIN_VERSION" == "1" ]; then echo "Skipping ParallelExec test because osync v1.1 ofunctions don't have this function." return 0 @@ -1381,7 +1422,7 @@ function xtest_ParallelExec () { assertNotEquals "ParallelExec full test 3" "0" $? } -function xtest_timedExecution () { +function test_timedExecution () { local arguments # Clever usage of indexes and exit codes @@ -1427,7 +1468,7 @@ function xtest_timedExecution () { done } -function xtest_UpgradeConfRun () { +function test_UpgradeConfRun () { if [ "$OSYNC_MIN_VERSION" == "1" ]; then echo "Skipping Upgrade script test because no further dev will happen on this for v1.1" return 0 @@ -1444,7 +1485,7 @@ function xtest_UpgradeConfRun () { assertEquals "Conf file upgrade" "0" $? # Update remote conf files with SSH port - sed -i.tmp 's#ssh://.*@localhost:[0-9]*/${HOME}/osync-tests/target#ssh://'$REMOTE_USER'@localhost:'$SSH_PORT'/${HOME}/osync-tests/target#' "$CONF_DIR/$TMP_OLD_CONF" + sed -i.tmp 's#ssh://.*@localhost:[0-9]*/${homedir}/osync-tests/target#ssh://'$REMOTE_USER'@localhost:'$SSH_PORT'/${homedir}/osync-tests/target#' "$CONF_DIR/$TMP_OLD_CONF" $OSYNC_EXECUTABLE "$CONF_DIR/$TMP_OLD_CONF" assertEquals "Upgraded conf file execution test" "0" $? @@ -1453,7 +1494,7 @@ function xtest_UpgradeConfRun () { rm -f "$CONF_DIR/$TMP_OLD_CONF.save" } -function xtest_DaemonMode () { +function test_DaemonMode () { if [ "$LOCAL_OS" == "WinNT10" ] || [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then echo "Skipping daemon mode test as [$LOCAL_OS] does not have inotifywait support." return 0 @@ -1509,7 +1550,7 @@ function xtest_DaemonMode () { } -function xtest_NoRemoteAccessTest () { +function test_NoRemoteAccessTest () { RemoveSSH cd "$OSYNC_DIR" diff --git a/install.sh b/install.sh index 0b540be..f6a3556 100755 --- a/install.sh +++ b/install.sh @@ -10,15 +10,15 @@ PROGRAM_BINARY=$PROGRAM".sh" PROGRAM_BATCH=$PROGRAM"-batch.sh" SSH_FILTER="ssh_filter.sh" -SCRIPT_BUILD=2020112901 +SCRIPT_BUILD=2023061101 INSTANCE_ID="installer-$SCRIPT_BUILD" ## osync / obackup / pmocr / zsnap install script ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11 ## Please adapt this to fit your distro needs -_OFUNCTIONS_VERSION=2.4.3 -_OFUNCTIONS_BUILD=2022050801 +_OFUNCTIONS_VERSION=2.5.1 +_OFUNCTIONS_BUILD=2023061401 _OFUNCTIONS_BOOTSTRAP=true if ! type "$BASH" > /dev/null; then @@ -41,6 +41,8 @@ _LOGGER_SILENT=false _LOGGER_VERBOSE=false _LOGGER_ERR_ONLY=false _LOGGER_PREFIX="date" +_LOGGER_WRITE_PARTIAL_LOGS=false # Writes partial log files to /tmp so sending logs via alerts can feed on them +_OFUNCTIONS_SHOW_SPINNER=true # Show spinner in ExecTasks function if [ "$KEEP_LOGGING" == "" ]; then KEEP_LOGGING=1801 fi @@ -145,7 +147,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -256,18 +258,18 @@ function Logger { if [ "$level" == "CRITICAL" ]; then _Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true ERROR_ALERT=true - # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. Need to keep this flag. - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. We need to create these flag files for ERROR_ALERT / WARN_ALERT to be picked up by Alert + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "ERROR" ]; then _Logger "$prefix($level):$value" "$prefix\e[91m$value\e[0m" true ERROR_ALERT=true - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "WARN" ]; then _Logger "$prefix($level):$value" "$prefix\e[33m$value\e[0m" true WARN_ALERT=true - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID.$TSTAMP" + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.WARN_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "NOTICE" ]; then if [ "$_LOGGER_ERR_ONLY" != true ]; then @@ -382,11 +384,11 @@ function GenericTrapQuit { local exitcode=0 # Get ERROR / WARN alert flags from subprocesses that call Logger - if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.WARN_ALERT.$SCRIPT_PID.$TSTAMP" ]; then WARN_ALERT=true exitcode=2 fi - if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" ]; then ERROR_ALERT=true exitcode=1 fi @@ -601,11 +603,11 @@ function GenericTrapQuit { local exitcode=0 # Get ERROR / WARN alert flags from subprocesses that call Logger - if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.WARN_ALERT.$SCRIPT_PID.$TSTAMP" ]; then WARN_ALERT=true exitcode=2 fi - if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" ]; then ERROR_ALERT=true exitcode=1 fi @@ -653,15 +655,26 @@ function SetLocalOSSettings { } function GetInit { + init="none" if [ -f /sbin/openrc-run ]; then init="openrc" Logger "Detected openrc." "NOTICE" + elif [ -f /usr/lib/systemd/systemd ]; then + init="systemd" + Logger "Detected systemd." "NOTICE" elif [ -f /sbin/init ]; then - if file /sbin/init | grep systemd > /dev/null; then - init="systemd" - Logger "Detected systemd." "NOTICE" + if type -p file > /dev/null 2>&1; then + if file /sbin/init | grep systemd > /dev/null; then + init="systemd" + Logger "Detected systemd." "NOTICE" + else + init="initV" + fi else init="initV" + fi + + if [ $init == "initV" ]; then Logger "Detected initV." "NOTICE" fi else diff --git a/osync-batch.sh b/osync-batch.sh index 3d0083b..1d57374 100755 --- a/osync-batch.sh +++ b/osync-batch.sh @@ -54,7 +54,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -166,18 +166,18 @@ function Logger { if [ "$level" == "CRITICAL" ]; then _Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true ERROR_ALERT=true - # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. Need to keep this flag. - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. We need to create these flag files for ERROR_ALERT / WARN_ALERT to be picked up by Alert + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "ERROR" ]; then _Logger "$prefix($level):$value" "$prefix\e[91m$value\e[0m" true ERROR_ALERT=true - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "WARN" ]; then _Logger "$prefix($level):$value" "$prefix\e[33m$value\e[0m" true WARN_ALERT=true - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID.$TSTAMP" + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.WARN_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "NOTICE" ]; then if [ "$_LOGGER_ERR_ONLY" != true ]; then @@ -224,11 +224,11 @@ function GenericTrapQuit { local exitcode=0 # Get ERROR / WARN alert flags from subprocesses that call Logger - if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.WARN_ALERT.$SCRIPT_PID.$TSTAMP" ]; then WARN_ALERT=true exitcode=2 fi - if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" ]; then ERROR_ALERT=true exitcode=1 fi diff --git a/osync.sh b/osync.sh index 2631f41..0d486f5 100755 --- a/osync.sh +++ b/osync.sh @@ -4,17 +4,17 @@ #Check dryruns with nosuffix mode for timestampList PROGRAM="osync" # Rsync based two way sync engine with fault tolerance -AUTHOR="(C) 2013-2022 by Orsiris de Jong" +AUTHOR="(C) 2013-2023 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" -PROGRAM_VERSION=1.3.0-rc3 -PROGRAM_BUILD=2021062901 +PROGRAM_VERSION=1.3.0 +PROGRAM_BUILD=2023061401 IS_STABLE=true CONFIG_FILE_REVISION_REQUIRED=1.3.0 -_OFUNCTIONS_VERSION=2.4.3 -_OFUNCTIONS_BUILD=2022050801 +_OFUNCTIONS_VERSION=2.5.1 +_OFUNCTIONS_BUILD=2023061401 _OFUNCTIONS_BOOTSTRAP=true if ! type "$BASH" > /dev/null; then @@ -37,6 +37,8 @@ _LOGGER_SILENT=false _LOGGER_VERBOSE=false _LOGGER_ERR_ONLY=false _LOGGER_PREFIX="date" +_LOGGER_WRITE_PARTIAL_LOGS=false # Writes partial log files to /tmp so sending logs via alerts can feed on them +_OFUNCTIONS_SHOW_SPINNER=true # Show spinner in ExecTasks function if [ "$KEEP_LOGGING" == "" ]; then KEEP_LOGGING=1801 fi @@ -141,7 +143,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -252,18 +254,18 @@ function Logger { if [ "$level" == "CRITICAL" ]; then _Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true ERROR_ALERT=true - # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. Need to keep this flag. - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. We need to create these flag files for ERROR_ALERT / WARN_ALERT to be picked up by Alert + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "ERROR" ]; then _Logger "$prefix($level):$value" "$prefix\e[91m$value\e[0m" true ERROR_ALERT=true - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "WARN" ]; then _Logger "$prefix($level):$value" "$prefix\e[33m$value\e[0m" true WARN_ALERT=true - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID.$TSTAMP" + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.WARN_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "NOTICE" ]; then if [ "$_LOGGER_ERR_ONLY" != true ]; then @@ -378,11 +380,11 @@ function GenericTrapQuit { local exitcode=0 # Get ERROR / WARN alert flags from subprocesses that call Logger - if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.WARN_ALERT.$SCRIPT_PID.$TSTAMP" ]; then WARN_ALERT=true exitcode=2 fi - if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" ]; then ERROR_ALERT=true exitcode=1 fi @@ -441,7 +443,11 @@ function SendAlert { fi fi - body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of current log"$'\n\n'"$(tail -n 1000 "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP")" + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ]; then + body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of current log"$'\n\n'"$(tail -n 1000 "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP")" + else + body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of current log"$'\n\n'"$(tail -n 1000 "$LOG_FILE")" + fi if [ $ERROR_ALERT == true ]; then subject="Error alert for $INSTANCE_ID" @@ -912,7 +918,7 @@ function ExecTasks { # soft / hard execution time checks that needs to be a subfunction since it is called both from main loop and from parallelExec sub loop function _ExecTasksTimeCheck { - if [ $spinner == true ]; then + if [ $spinner == true ] && [ "$_OFUNCTIONS_SHOW_SPINNER" != false ]; then Spinner fi if [ $counting == true ]; then @@ -2098,6 +2104,16 @@ function InitRemoteOSDependingSettings { ## Set rsync default arguments (complete with -r or -d depending on recursivity later) RSYNC_DEFAULT_ARGS="-ltD -8" + + ## NPF-MOD: Regarding #242, we need to add --old-args if rsync > 3.2.3 + #rsync_version=$("${RSYNC_EXECUTABLE}" --version 2>/dev/null| head -1 | awk '{print $3}') + #if [ $(VerComp $rsync_version 3.2.3) -eq 1 ]; then + # RSYNC_DEFAULT_ARGS="$RSYNC_DEFAULT_ARGS --old-args" + #fi + # NPF-MOD: Strangely enough, also happens on RHEL7 rsync 3.1.1 + # Let's resolve this easier + export RSYNC_OLD_ARGS=1 + if [ "$_DRYRUN" == true ]; then RSYNC_DRY_ARG="-n" DRY_WARNING="/!\ DRY RUN " @@ -2105,6 +2121,7 @@ function InitRemoteOSDependingSettings { RSYNC_DRY_ARG="" fi + RSYNC_ATTR_ARGS="" if [ "$PRESERVE_PERMISSIONS" != false ]; then RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -p" @@ -2327,16 +2344,8 @@ function FileMove () { mv -f "$source" "$dest" return $? elif [ -w "$source" ]; then - if [ -f "$dest" ]; then # for files we don't need recursive delete - rm -f "$dest" - elif [ -d "$dest" ]; then # for directories we need recursive delete - rm -rf "$dest" - fi - if [ -f "$source" ]; then - cp -p "$source" "$dest" && rm -f "$source" # for files we don't need recursive copy & delete - elif [ -d "$source" ]; then - cp -rp "$source" "$dest" && rm -rf "$source" # for directories we need recursive copy & delete - fi + [ -f "$dest" ] && rm -f "$dest" + cp -p "$source" "$dest" && rm -f "$source" return $? else return -1 @@ -2491,6 +2500,7 @@ _OFUNCTIONS_BOOTSTRAP=true [ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0) or build it with merge.sh" && exit 1 _LOGGER_PREFIX="time" +_LOGGER_WRITE_PARTIAL_LOGS=true ## Working directory. This directory exists in any replica and contains state files, backups, soft deleted files etc OSYNC_DIR=".osync_workdir" @@ -2897,7 +2907,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -3314,7 +3324,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -3930,7 +3940,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -4565,7 +4575,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -4665,16 +4675,8 @@ function FileMove () { mv -f "$source" "$dest" return $? elif [ -w "$source" ]; then - if [ -f "$dest" ]; then # for files we don't need recursive delete - rm -f "$dest" - elif [ -d "$dest" ]; then # for directories we need recursive delete - rm -rf "$dest" - fi - if [ -f "$source" ]; then - cp -p "$source" "$dest" && rm -f "$source" # for files we don't need recursive copy & delete - elif [ -d "$source" ]; then - cp -rp "$source" "$dest" && rm -rf "$source" # for directories we need recursive copy & delete - fi + [ -f "$dest" ] && rm -f "$dest" + cp -p "$source" "$dest" && rm -f "$source" return $? else return -1 @@ -5566,7 +5568,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -5842,7 +5844,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -6354,6 +6356,7 @@ function Usage { echo "--no-prefix Will suppress time / date suffix from output" echo "--silent Will run osync without any output to stdout, used for cron jobs" echo "--errors-only Output only errors (can be combined with silent or verbose)" + echo "--non-interactive Don't show running animation in cron / service mode" echo "--summary Outputs a list of transferred / deleted files at the end of the run" echo "--log-conflicts [EXPERIMENTAL] Outputs a list of conflicted files" echo "--alert-conflicts Send an email if conflictual files found (implies --log-conflicts)" @@ -6629,6 +6632,10 @@ function GetCommandlineArguments { opts=$opts" --errors-only" _LOGGER_ERR_ONLY=true ;; + --non-interactive) + opts=$opts" --non-interactive" + _OFUNCTIONS_SHOW_SPINNER=false + ;; --summary) opts=$opts" --summary" _SUMMARY=true diff --git a/sync.conf.example b/sync.conf.example index 439abc0..aa42c59 100644 --- a/sync.conf.example +++ b/sync.conf.example @@ -100,7 +100,7 @@ REMOTE_3RD_PARTY_HOSTS="www.kernel.org www.google.com" [MISC_OPTIONS] ## Optional arguments passed to rsync executable. The following are already managed by the program and shoul never be passed here -## -r -l -p -t -g -o -D -E - u- i- n --executability -A -X -L -K -H -8 -zz –skip-compress –checksum –bwlimit –partial –partial-dir –no-whole-file –whole-file –backup –backup-dir –suffix +## -r -l -p -t -g -o -D -E - u- i- n --executability -A -X -L -K -H -8 --zz -–skip-compress -–checksum –-bwlimit –-partial –-partial-dir –-no-whole-file –-whole-file –-backup –-backup-dir –-suffix ## --exclude --exclude-from --include --include-from --list-only --stats ## When dealing with different filesystems for sync, or using SMB mountpoints, try adding --modify-window=2 --omit-dir-times as optional arguments. RSYNC_OPTIONAL_ARGS=""