From 4bf065f94e5fe1b2939b6b1366b461f5405471d6 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sat, 10 Jun 2023 11:07:51 +0200 Subject: [PATCH 01/38] Make tests run on github runner's WSL and macos versions too --- dev/tests/run_tests.sh | 97 +++++++++++++++++++++++++----------------- 1 file changed, 58 insertions(+), 39 deletions(-) diff --git a/dev/tests/run_tests.sh b/dev/tests/run_tests.sh index 4f9ce17..ae8940c 100755 --- a/dev/tests/run_tests.sh +++ b/dev/tests/run_tests.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# osync test suite 2022070702 +# osync test suite 2023061001 # Allows the following environment variables @@ -50,20 +50,23 @@ if [ "$SKIP_REMOTE" = "" ]; then SKIP_REMOTE=false + REMOTE_USER=root fi +homedir=$(eval echo ~${REMOTE_USER}) + # drupal servers are often unreachable for whetever reason or give 0 bytes files #LARGE_FILESET_URL="http://ftp.drupal.org/files/projects/drupal-8.2.2.tar.gz" LARGE_FILESET_URL="http://www.netpower.fr/sites/default/files/osync-test-files.tar.gz" -# Fakeroot for install / uninstall and test of executables -FAKEROOT="${HOME}/osync_test_install" - OSYNC_DIR="$(pwd)" OSYNC_DIR=${OSYNC_DIR%%/dev*} DEV_DIR="$OSYNC_DIR/dev" TESTS_DIR="$DEV_DIR/tests" +# Fakeroot for install / uninstall and test of executables +FAKEROOT="${homedir}/osync_test_install" + CONF_DIR="$TESTS_DIR/conf" LOCAL_CONF="local.conf" REMOTE_CONF="remote.conf" @@ -76,7 +79,7 @@ OSYNC_UPGRADE="upgrade-v1.0x-v1.3x.sh" TMP_FILE="$DEV_DIR/tmp" -OSYNC_TESTS_DIR="${HOME}/osync-tests" +OSYNC_TESTS_DIR="${homedir}/osync-tests" INITIATOR_DIR="$OSYNC_TESTS_DIR/initiator" TARGET_DIR="$OSYNC_TESTS_DIR/target" OSYNC_WORKDIR=".osync_workdir" @@ -93,46 +96,51 @@ PRIVKEY_NAME="id_rsa_local_osync_tests" PUBKEY_NAME="${PRIVKEY_NAME}.pub" function SetupSSH { - echo "Setting up an ssh key to ${HOME}/.ssh/${PRIVKEY_NAME}" - echo -e 'y\n'| ssh-keygen -t rsa -b 2048 -N "" -f "${HOME}/.ssh/${PRIVKEY_NAME}" - - SSH_AUTH_LINE="from=\"*\",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,command=\"$FAKEROOT/usr/local/bin/ssh_filter.sh SomeAlphaNumericToken9\" $(cat ${HOME}/.ssh/${PUBKEY_NAME})" - echo "ls -alh ${HOME}" - ls -alh "${HOME}" - echo "ls -alh ${HOME}/.ssh" - ls -alh "${HOME}/.ssh" - - if [ -f "${HOME}/.ssh/authorized_keys" ]; then - if ! grep "$(cat ${HOME}/.ssh/${PUBKEY_NAME})" "${HOME}/.ssh/authorized_keys"; then - echo "$SSH_AUTH_LINE" >> "${HOME}/.ssh/authorized_keys" + echo "Setting up an ssh key to ${homedir}/.ssh/${PRIVKEY_NAME}" + echo -e 'y\n'| ssh-keygen -t rsa -b 2048 -N "" -f "${homedir}/.ssh/${PRIVKEY_NAME}" + + + + SSH_AUTH_LINE="from=\"*\",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,command=\"$FAKEROOT/usr/local/bin/ssh_filter.sh SomeAlphaNumericToken9\" $(cat ${homedir}/.ssh/${PUBKEY_NAME})" + echo "ls -alh ${homedir}" + ls -alh "${homedir}" + echo "ls -alh ${homedir}/.ssh" + ls -alh "${homedir}/.ssh" + + if [ -f "${homedir}/.ssh/authorized_keys" ]; then + if ! grep "$(cat ${homedir}/.ssh/${PUBKEY_NAME})" "${homedir}/.ssh/authorized_keys"; then + echo "Adding auth line in authorized_keys file ${homedir}/.ssh/authorized_keys" + echo "$SSH_AUTH_LINE" >> "${homedir}/.ssh/authorized_keys" fi else - echo "$SSH_AUTH_LINE" >> "${HOME}/.ssh/authorized_keys" + echo "Creating authorized_keys file ${homedir}/.ssh/authorized_keys" + echo "$SSH_AUTH_LINE" >> "${homedir}/.ssh/authorized_keys" fi - chmod 600 "${HOME}/.ssh/authorized_keys" + chmod 600 "${homedir}/.ssh/authorized_keys" # Add localhost to known hosts so self connect works if [ -z "$(ssh-keygen -F localhost)" ]; then - ssh-keyscan -H localhost >> "${HOME}/.ssh/known_hosts" + ssh-keyscan -H localhost >> "${homedir}/.ssh/known_hosts" fi # Update remote conf files with SSH port - sed -i.tmp 's#ssh://.*@localhost:[0-9]*/${HOME}/osync-tests/target#ssh://'$REMOTE_USER'@localhost:'$SSH_PORT'/${HOME}/osync-tests/target#' "$CONF_DIR/$REMOTE_CONF" + sed -i.tmp 's#ssh://.*@localhost:[0-9]*/${homedir}/osync-tests/target#ssh://'$REMOTE_USER'@localhost:'$SSH_PORT'/${homedir}/osync-tests/target#' "$CONF_DIR/$REMOTE_CONF" - echp "ls -alh ${HOME}/.ssh" - ls -alh "${HOME}/.ssh" - echo "cat ${HOME}/.ssh.authorized_keys" - cat "${HOME}/.ssh/authorized_keys" + echo "ls -alh ${homedir}/.ssh" + ls -alh "${homedir}/.ssh" + echo "cat ${homedir}/.ssh/authorized_keys" + cat "${homedir}/.ssh/authorized_keys" echo "###" echo "END SETUP SSH" } function RemoveSSH { - if [ -f "${HOME}/.ssh/id_rsa_local_osync_tests" ]; then + echo "Now removing SSH keys" + if [ -f "${homedir}/.ssh/id_rsa_local_osync_tests" ]; then echo "Restoring SSH authorized_keys file" - sed -i.bak "s|.*$(cat "${HOME}/.ssh/id_rsa_local_osync_tests.pub")||g" "${HOME}/.ssh/authorized_keys" - rm -f "${HOME}/.ssh/{id_rsa_local_osync_tests.pub,id_rsa_local_osync_tests}" + sed -i.bak "s|.*$(cat "${homedir}/.ssh/id_rsa_local_osync_tests.pub")||g" "${homedir}/.ssh/authorized_keys" + rm -f "${homedir}/.ssh/{id_rsa_local_osync_tests.pub,id_rsa_local_osync_tests}" fi } @@ -192,8 +200,9 @@ function PrepareLocalDirs () { function oneTimeSetUp () { START_TIME=$SECONDS - mkdir -p "$FAKEROOT" - + echo "Running install.sh from ${OSYNC_DIR}" + ls -alh ${OSYNC_DIR} + $SUDO_CMD ${OSYNC_DIR}/install.sh --prefix="${FAKEROOT}" source "$DEV_DIR/ofunctions.sh" # Fix default umask because of ACL test that expects 0022 when creating test files @@ -207,6 +216,7 @@ function oneTimeSetUp () { if [ "$RUNNING_ON_GITHUB_ACTIONS" == true ]; then echo "Running with GITHUB ACTIONS settings" REMOTE_USER="runner" + homedir=$(eval echo ~${REMOTE_USER}) RHOST_PING=false SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_3RD_PARTY_HOSTS" "" SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_HOST_PING" false @@ -216,6 +226,7 @@ function oneTimeSetUp () { else echo "Running with local settings" REMOTE_USER="root" + homedir=$(eval echo ~${REMOTE_USER}) RHOST_PING=true SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_3RD_PARTY_HOSTS" "\"www.kernel.org www.google.com\"" SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_HOST_PING" true @@ -248,7 +259,7 @@ function oneTimeSetUp () { # Do not check remote config on msys or cygwin since we don't have a local SSH server if [ "$LOCAL_OS" != "msys" ] && [ "$LOCAL_OS" != "Cygwin" ] && [ $SKIP_REMOTE != true ]; then - osyncParameters[$__quickRemote]="--initiator=$INITIATOR_DIR --target=ssh://localhost:$SSH_PORT/$TARGET_DIR --rsakey=${HOME}/.ssh/id_rsa_local_osync_tests --instance-id=quickremote --remote-token=SomeAlphaNumericToken9" + osyncParameters[$__quickRemote]="--initiator=$INITIATOR_DIR --target=ssh://localhost:$SSH_PORT/$TARGET_DIR --rsakey=${homedir}/.ssh/id_rsa_local_osync_tests --instance-id=quickremote --remote-token=SomeAlphaNumericToken9" osyncParameters[$__confRemote]="$CONF_DIR/$REMOTE_CONF" osyncDaemonParameters[$__remote]="$CONF_DIR/$REMOTE_CONF --on-changes" @@ -331,20 +342,28 @@ function test_SSH { failure=false + # Testing as "remote user" + echo "ls -alh ${homedir}/.ssh" + ls -alh "${homedir}/.ssh" + echo "Running SSH test as ${REMOTE_USER}" # SSH_PORT and SSH_USER are set by oneTimeSetup - ssh -i "${REMOTE_USER}/.ssh/${PUBKEY_NAME}" -p $SSH_PORT ${REMOTE_USER}@localhost "echo \"Remotely:\"; whoami; echo \"TEST OK\"" + $SUDO_CMD ssh -i "${homedir}/.ssh/${PRIVKEY_NAME}" -p $SSH_PORT ${REMOTE_USER}@localhost "env _REMOTE_TOKEN=SomeAlphaNumericToken9 echo \"Remotely:\"; whoami; echo \"TEST OK\"" if [ $? -ne 0 ]; then echo "SSH test failed" failure=true fi + + # Testing as current user + #echo "ls -alh ${homedir}/.ssh" + #ls -alh "${homedir}/.ssh" - echo "Running SSH test as $(whoami)" - ssh -i "$(whoami)/.ssh/${PUBKEY_NAME}" -p $SSH_PORT $(whoami)@localhost "echo \"Remotely:\"; whoami; echo \"TEST OK\"" - if [ $? -ne 0 ]; then - echo "SSH test failed" - failure=true - fi + #echo "Running SSH test as $(whoami)" + #$SUDO_CMD ssh -i "${homedir}/.ssh/${PRIVKEY_NAME}" -p $SSH_PORT $(whoami)@localhost "env _REMOTE_TOKEN=SomeAlphaNumericToken9 echo \"Remotely:\"; whoami; echo \"TEST OK\"" + #if [ $? -ne 0 ]; then + # echo "SSH test failed" + # failure=true + #fi if [ $failure == true ]; then exit 1 # Try to see if we can abort all tests @@ -1444,7 +1463,7 @@ function xtest_UpgradeConfRun () { assertEquals "Conf file upgrade" "0" $? # Update remote conf files with SSH port - sed -i.tmp 's#ssh://.*@localhost:[0-9]*/${HOME}/osync-tests/target#ssh://'$REMOTE_USER'@localhost:'$SSH_PORT'/${HOME}/osync-tests/target#' "$CONF_DIR/$TMP_OLD_CONF" + sed -i.tmp 's#ssh://.*@localhost:[0-9]*/${homedir}/osync-tests/target#ssh://'$REMOTE_USER'@localhost:'$SSH_PORT'/${homedir}/osync-tests/target#' "$CONF_DIR/$TMP_OLD_CONF" $OSYNC_EXECUTABLE "$CONF_DIR/$TMP_OLD_CONF" assertEquals "Upgraded conf file execution test" "0" $? From a9da549e4ff71217bd57829e0f878b91a4ae6dd2 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sat, 10 Jun 2023 11:08:32 +0200 Subject: [PATCH 02/38] Add openssh-server for remote tests --- .github/workflows/windows.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 889f46b..5910f22 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -16,12 +16,14 @@ jobs: additional-packages: dos2unix rsync + openssh-server - name: Execute tests and generate coverage report shell: wsl-bash {0} env: RUNNING_ON_GITHUB_ACTIONS: true run: | - find . -type f -print0 | xargs -0 -n 1 -P 4 dos2unix + find . -type f ! -name ".git*" -print0 | xargs -0 -n 1 -P 4 dos2unix + service ssh start ./dev/tests/run_tests.sh - name: Upload Coverage to Codecov uses: codecov/codecov-action@v1 From 5f4e7af91006a13ddd22650467b52f3fcd1606b3 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sat, 10 Jun 2023 11:14:46 +0200 Subject: [PATCH 03/38] Re-enable all tests since we got remote tests to work --- dev/tests/run_tests.sh | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/dev/tests/run_tests.sh b/dev/tests/run_tests.sh index ae8940c..310343d 100755 --- a/dev/tests/run_tests.sh +++ b/dev/tests/run_tests.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# osync test suite 2023061001 +# osync test suite 2023060901 # Allows the following environment variables @@ -401,7 +401,7 @@ function test_Merge () { assertEquals "Install failed" "0" $? } -function xtest_LargeFileSet () { +function test_LargeFileSet () { for i in "${osyncParameters[@]}"; do cd "$OSYNC_DIR" @@ -419,7 +419,7 @@ function xtest_LargeFileSet () { done } -function xtest_controlMaster () { +function test_controlMaster () { cd "$OSYNC_DIR" PrepareLocalDirs @@ -428,7 +428,7 @@ function xtest_controlMaster () { assertEquals "Running quick remote test with controlmaster enabled." "0" $? } -function xtest_Exclusions () { +function test_Exclusions () { # Will sync except php files # RSYNC_EXCLUDE_PATTERN="*.php" is set at runtime for quicksync and in config files for other runs @@ -456,7 +456,7 @@ function xtest_Exclusions () { done } -function xtest_Deletetion () { +function test_Deletetion () { local iFile1="$INITIATOR_DIR/i fic" local iFile2="$INITIATOR_DIR/i foc (something)" local tFile1="$TARGET_DIR/t fic" @@ -500,7 +500,7 @@ function xtest_Deletetion () { done } -function xtest_deletion_failure () { +function test_deletion_failure () { if [ "$LOCAL_OS" == "WinNT10" ] || [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then echo "Skipping deletion failure test as Win10 does not have chattr support." return 0 @@ -567,7 +567,7 @@ function xtest_deletion_failure () { done } -function xtest_skip_deletion () { +function test_skip_deletion () { local modes if [ "$OSYNC_MIN_VERSION" == "1" ]; then @@ -643,7 +643,7 @@ function xtest_skip_deletion () { SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "SKIP_DELETION" "" } -function xtest_handle_symlinks () { +function test_handle_symlinks () { if [ "$OSYNC_MIN_VERSION" == "1" ]; then echo "Skipping symlink tests as osync v1.1x didn't handle this." return 0 @@ -824,7 +824,7 @@ function xtest_handle_symlinks () { done } -function xtest_softdeletion_cleanup () { +function test_softdeletion_cleanup () { #declare -A files files=() @@ -901,7 +901,7 @@ function xtest_softdeletion_cleanup () { } -function xtest_FileAttributePropagation () { +function test_FileAttributePropagation () { if [ "$RUNNING_ON_GITHUB_ACTIONS" == true ]; then echo "Skipping FileAttributePropagation tests as travis does not support getfacl / setfacl." @@ -989,7 +989,7 @@ function xtest_FileAttributePropagation () { SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "PRESERVE_XATTR" false } -function xtest_ConflictBackups () { +function test_ConflictBackups () { for i in "${osyncParameters[@]}"; do cd "$OSYNC_DIR" PrepareLocalDirs @@ -1025,7 +1025,7 @@ function xtest_ConflictBackups () { done } -function xtest_MultipleConflictBackups () { +function test_MultipleConflictBackups () { local additionalParameters @@ -1087,7 +1087,7 @@ function xtest_MultipleConflictBackups () { SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "CONFLICT_BACKUP_MULTIPLE" false } -function xtest_Locking () { +function test_Locking () { # local not running = resume # remote same instance_id = resume # remote different instance_id = stop @@ -1194,7 +1194,7 @@ function xtest_Locking () { SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "FORCE_STRANGER_LOCK_RESUME" false } -function xtest_ConflictDetetion () { +function test_ConflictDetetion () { # Tests compatible with v1.4+ if [ $OSYNC_MIN_VERSION -lt 4 ]; then @@ -1245,7 +1245,7 @@ function xtest_ConflictDetetion () { return 0 } -function xtest_WaitForTaskCompletion () { +function test_WaitForTaskCompletion () { local pids # Tests compatible with v1.1 syntax @@ -1339,7 +1339,7 @@ function xtest_WaitForTaskCompletion () { assertEquals "WaitForTaskCompletion test 5" "2" $? } -function xtest_ParallelExec () { +function test_ParallelExec () { if [ "$OSYNC_MIN_VERSION" == "1" ]; then echo "Skipping ParallelExec test because osync v1.1 ofunctions don't have this function." return 0 @@ -1400,7 +1400,7 @@ function xtest_ParallelExec () { assertNotEquals "ParallelExec full test 3" "0" $? } -function xtest_timedExecution () { +function test_timedExecution () { local arguments # Clever usage of indexes and exit codes @@ -1446,7 +1446,7 @@ function xtest_timedExecution () { done } -function xtest_UpgradeConfRun () { +function test_UpgradeConfRun () { if [ "$OSYNC_MIN_VERSION" == "1" ]; then echo "Skipping Upgrade script test because no further dev will happen on this for v1.1" return 0 @@ -1472,7 +1472,7 @@ function xtest_UpgradeConfRun () { rm -f "$CONF_DIR/$TMP_OLD_CONF.save" } -function xtest_DaemonMode () { +function test_DaemonMode () { if [ "$LOCAL_OS" == "WinNT10" ] || [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then echo "Skipping daemon mode test as [$LOCAL_OS] does not have inotifywait support." return 0 @@ -1528,7 +1528,7 @@ function xtest_DaemonMode () { } -function xtest_NoRemoteAccessTest () { +function test_NoRemoteAccessTest () { RemoveSSH cd "$OSYNC_DIR" From fa7f6d1088ce8f15496e5824d0c50ef10cbe7ca7 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 10 Jun 2023 11:58:12 +0200 Subject: [PATCH 04/38] Configure default SSH port for github actions --- .github/workflows/linux.yml | 1 + .github/workflows/macos.yml | 1 + .github/workflows/windows.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 633b7a7..53334ec 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -17,6 +17,7 @@ jobs: - name: Execute tests and generate coverage report env: RUNNING_ON_GITHUB_ACTIONS: true + SSH_PORT=22 run: | sudo bash ./dev/tests/run_tests.sh - name: Upload Coverage to Codecov diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 2bae489..7dc5993 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -22,6 +22,7 @@ jobs: - name: Execute tests and generate coverage report env: RUNNING_ON_GITHUB_ACTIONS: true + SSH_PORT=22 run: | sudo bash ./dev/tests/run_tests.sh - name: Upload Coverage to Codecov diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 5910f22..80b75fa 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -21,6 +21,7 @@ jobs: shell: wsl-bash {0} env: RUNNING_ON_GITHUB_ACTIONS: true + SSH_PORT=22 run: | find . -type f ! -name ".git*" -print0 | xargs -0 -n 1 -P 4 dos2unix service ssh start From 8ef2262421e82ce701f031f8fbf885b7381d08f2 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sat, 10 Jun 2023 12:04:34 +0200 Subject: [PATCH 05/38] Fix typo in env vars --- .github/workflows/linux.yml | 2 +- .github/workflows/macos.yml | 2 +- .github/workflows/windows.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 53334ec..89b0ba6 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -17,7 +17,7 @@ jobs: - name: Execute tests and generate coverage report env: RUNNING_ON_GITHUB_ACTIONS: true - SSH_PORT=22 + SSH_PORT: 22 run: | sudo bash ./dev/tests/run_tests.sh - name: Upload Coverage to Codecov diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 7dc5993..29fd747 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -22,7 +22,7 @@ jobs: - name: Execute tests and generate coverage report env: RUNNING_ON_GITHUB_ACTIONS: true - SSH_PORT=22 + SSH_PORT: 22 run: | sudo bash ./dev/tests/run_tests.sh - name: Upload Coverage to Codecov diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 80b75fa..b235ea7 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -21,7 +21,7 @@ jobs: shell: wsl-bash {0} env: RUNNING_ON_GITHUB_ACTIONS: true - SSH_PORT=22 + SSH_PORT: 22 run: | find . -type f ! -name ".git*" -print0 | xargs -0 -n 1 -P 4 dos2unix service ssh start From b95e75aa0bb7a6846457c3b51a19da00d77db4b7 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sat, 10 Jun 2023 12:31:14 +0200 Subject: [PATCH 06/38] Add SSH_PORT info --- dev/tests/run_tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/tests/run_tests.sh b/dev/tests/run_tests.sh index 310343d..5f552e7 100755 --- a/dev/tests/run_tests.sh +++ b/dev/tests/run_tests.sh @@ -238,6 +238,7 @@ function oneTimeSetUp () { # Get default ssh port from env if [ "$SSH_PORT" == "" ]; then SSH_PORT=22 + echo "Running with SSH_PORT=${SSH_PORT}" fi # Setup modes per test From 30d66003ef29db596746662932ecdb921f92bb0b Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sat, 10 Jun 2023 12:35:26 +0200 Subject: [PATCH 07/38] Use export for env variables --- .github/workflows/linux.yml | 5 ++--- .github/workflows/macos.yml | 5 ++--- .github/workflows/windows.yml | 7 +++---- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 89b0ba6..c225d1a 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -15,10 +15,9 @@ jobs: run: | sudo apt-get install inotify-tools acl - name: Execute tests and generate coverage report - env: - RUNNING_ON_GITHUB_ACTIONS: true - SSH_PORT: 22 run: | + export RUNNING_ON_GITHUB_ACTIONS=true + export SSH_PORT=22 sudo bash ./dev/tests/run_tests.sh - name: Upload Coverage to Codecov uses: codecov/codecov-action@v1 diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 29fd747..2c5e52e 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -20,10 +20,9 @@ jobs: brew install fswatch echo "/usr/local/bin" >> $GITHUB_PATH - name: Execute tests and generate coverage report - env: - RUNNING_ON_GITHUB_ACTIONS: true - SSH_PORT: 22 run: | + export RUNNING_ON_GITHUB_ACTIONS=true + export SSH_PORT=22 sudo bash ./dev/tests/run_tests.sh - name: Upload Coverage to Codecov uses: codecov/codecov-action@v1 diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index b235ea7..2b7c12d 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -19,11 +19,10 @@ jobs: openssh-server - name: Execute tests and generate coverage report shell: wsl-bash {0} - env: - RUNNING_ON_GITHUB_ACTIONS: true - SSH_PORT: 22 run: | - find . -type f ! -name ".git*" -print0 | xargs -0 -n 1 -P 4 dos2unix + RUNNING_ON_GITHUB_ACTIONS=true + SSH_PORT=22 + find . -type f -not -path ".git/*" -print0 | xargs -0 -n 1 -P 4 dos2unix service ssh start ./dev/tests/run_tests.sh - name: Upload Coverage to Codecov From 52a42bc1d70763000562368760861d010841c13d Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sat, 10 Jun 2023 12:51:41 +0200 Subject: [PATCH 08/38] Add export for env variables --- .github/workflows/windows.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 2b7c12d..d59e08d 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -20,8 +20,8 @@ jobs: - name: Execute tests and generate coverage report shell: wsl-bash {0} run: | - RUNNING_ON_GITHUB_ACTIONS=true - SSH_PORT=22 + export RUNNING_ON_GITHUB_ACTIONS=true + export SSH_PORT=22 find . -type f -not -path ".git/*" -print0 | xargs -0 -n 1 -P 4 dos2unix service ssh start ./dev/tests/run_tests.sh From d100841bd9ccc8dbf4dc87947fde0b79c2bc7bea Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sat, 10 Jun 2023 12:59:42 +0200 Subject: [PATCH 09/38] Try to fix github env variables --- .github/workflows/linux.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index c225d1a..4a2d1ce 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -15,9 +15,12 @@ jobs: run: | sudo apt-get install inotify-tools acl - name: Execute tests and generate coverage report + env: + RUNNING_ON_GITHUB_ACTIONS=true + SSH_PORT=22 run: | - export RUNNING_ON_GITHUB_ACTIONS=true - export SSH_PORT=22 + echo "Running on github actions: ${RUNNING_ON_GITHUB_ACTIONS}" + echo "Running on ssh port ${SSH_PORT}" sudo bash ./dev/tests/run_tests.sh - name: Upload Coverage to Codecov uses: codecov/codecov-action@v1 From 6b6f723094944bc0b492334c031f2811c8039c5f Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sat, 10 Jun 2023 13:03:05 +0200 Subject: [PATCH 10/38] Fix remote tests not updating conf file properly --- dev/tests/run_tests.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dev/tests/run_tests.sh b/dev/tests/run_tests.sh index 5f552e7..095427c 100755 --- a/dev/tests/run_tests.sh +++ b/dev/tests/run_tests.sh @@ -124,7 +124,7 @@ function SetupSSH { fi # Update remote conf files with SSH port - sed -i.tmp 's#ssh://.*@localhost:[0-9]*/${homedir}/osync-tests/target#ssh://'$REMOTE_USER'@localhost:'$SSH_PORT'/${homedir}/osync-tests/target#' "$CONF_DIR/$REMOTE_CONF" + sed -i.tmp 's#ssh://.*@localhost:[0-9]*/${HOME}/osync-tests/target#ssh://'$REMOTE_USER'@localhost:'$SSH_PORT'/'${homedir}'/osync-tests/target#' "$CONF_DIR/$REMOTE_CONF" echo "ls -alh ${homedir}/.ssh" ls -alh "${homedir}/.ssh" @@ -215,7 +215,8 @@ function oneTimeSetUp () { # Set some travis related changes if [ "$RUNNING_ON_GITHUB_ACTIONS" == true ]; then echo "Running with GITHUB ACTIONS settings" - REMOTE_USER="runner" + #REMOTE_USER="runner" + REMOTE_USER="root" # WIP homedir=$(eval echo ~${REMOTE_USER}) RHOST_PING=false SetConfFileValue "$CONF_DIR/$REMOTE_CONF" "REMOTE_3RD_PARTY_HOSTS" "" From 0ef4af40dc9d33c2b8daca8ca4a2eaa2a8ca570a Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sat, 10 Jun 2023 13:49:20 +0200 Subject: [PATCH 11/38] Coherent yaml workflow --- .github/workflows/linux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 4a2d1ce..312cbeb 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -16,8 +16,8 @@ jobs: sudo apt-get install inotify-tools acl - name: Execute tests and generate coverage report env: - RUNNING_ON_GITHUB_ACTIONS=true - SSH_PORT=22 + - RUNNING_ON_GITHUB_ACTIONS=true + - SSH_PORT=22 run: | echo "Running on github actions: ${RUNNING_ON_GITHUB_ACTIONS}" echo "Running on ssh port ${SSH_PORT}" From 91950446f0babb2e2f6b5ad539328c152db060e7 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sat, 10 Jun 2023 13:51:03 +0200 Subject: [PATCH 12/38] Coherent yaml workflow --- .github/workflows/linux.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 312cbeb..7d1916e 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -15,9 +15,9 @@ jobs: run: | sudo apt-get install inotify-tools acl - name: Execute tests and generate coverage report - env: - - RUNNING_ON_GITHUB_ACTIONS=true - - SSH_PORT=22 + env: | + RUNNING_ON_GITHUB_ACTIONS=true + SSH_PORT=22 run: | echo "Running on github actions: ${RUNNING_ON_GITHUB_ACTIONS}" echo "Running on ssh port ${SSH_PORT}" From b15e8e8f88eb08d3eb6cdbcfbaa18b8d73614345 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sat, 10 Jun 2023 19:20:43 +0200 Subject: [PATCH 13/38] Preserve env variables for sudo calls --- .github/workflows/linux.yml | 7 +++---- .github/workflows/macos.yml | 2 +- .github/workflows/windows.yml | 6 +++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 7d1916e..960ae87 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -15,12 +15,11 @@ jobs: run: | sudo apt-get install inotify-tools acl - name: Execute tests and generate coverage report - env: | - RUNNING_ON_GITHUB_ACTIONS=true - SSH_PORT=22 run: | + export RUNNING_ON_GITHUB_ACTIONS=true + export SSH_PORT=22 echo "Running on github actions: ${RUNNING_ON_GITHUB_ACTIONS}" echo "Running on ssh port ${SSH_PORT}" - sudo bash ./dev/tests/run_tests.sh + sudo -E bash ./dev/tests/run_tests.sh - name: Upload Coverage to Codecov uses: codecov/codecov-action@v1 diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 2c5e52e..a22c7f5 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -23,6 +23,6 @@ jobs: run: | export RUNNING_ON_GITHUB_ACTIONS=true export SSH_PORT=22 - sudo bash ./dev/tests/run_tests.sh + sudo -E bash ./dev/tests/run_tests.sh - name: Upload Coverage to Codecov uses: codecov/codecov-action@v1 diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index d59e08d..61521ef 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -20,9 +20,9 @@ jobs: - name: Execute tests and generate coverage report shell: wsl-bash {0} run: | - export RUNNING_ON_GITHUB_ACTIONS=true - export SSH_PORT=22 - find . -type f -not -path ".git/*" -print0 | xargs -0 -n 1 -P 4 dos2unix + export RUNNING_ON_GITHUB_ACTIONS=true + export SSH_PORT=22 + find ./ -type f ! -path "./.git/*" -print0 | xargs -0 -n 1 -P 4 dos2unix service ssh start ./dev/tests/run_tests.sh - name: Upload Coverage to Codecov From 43e1aaaaa14b9f4a29b681a189021cfe0a8690a6 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sat, 10 Jun 2023 19:40:24 +0200 Subject: [PATCH 14/38] Bump ofunctions version to 2.4.4 --- dev/ofunctions.sh | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index 71eb703..0f846c5 100644 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -14,6 +14,7 @@ ## _LOGGER_VERBOSE=true/false ## _LOGGER_ERR_ONLY=true/false ## _LOGGER_PREFIX="date"/"time"/"" +## _LOGGER_WRITE_PARTIAL_LOGS=true/false ## Also, set the following trap in order to clean temporary files ## trap GenericTrapQuit TERM EXIT HUP QUIT @@ -30,8 +31,8 @@ #### OFUNCTIONS FULL SUBSET #### #### OFUNCTIONS MINI SUBSET #### #### OFUNCTIONS MICRO SUBSET #### -_OFUNCTIONS_VERSION=2.4.3 -_OFUNCTIONS_BUILD=2022050801 +_OFUNCTIONS_VERSION=2.4.4 +_OFUNCTIONS_BUILD=2022081901 #### _OFUNCTIONS_BOOTSTRAP SUBSET #### _OFUNCTIONS_BOOTSTRAP=true #### _OFUNCTIONS_BOOTSTRAP SUBSET END #### @@ -56,6 +57,7 @@ _LOGGER_SILENT=false _LOGGER_VERBOSE=false _LOGGER_ERR_ONLY=false _LOGGER_PREFIX="date" +_LOGGER_WRITE_PARTIAL_LOGS=false # Writes partial log files to /tmp so sending logs via alerts can feed on them if [ "$KEEP_LOGGING" == "" ]; then KEEP_LOGGING=1801 fi @@ -172,7 +174,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -289,18 +291,18 @@ function Logger { if [ "$level" == "CRITICAL" ]; then _Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true ERROR_ALERT=true - # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. Need to keep this flag. - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. We need to create these flag files for ERROR_ALERT / WARN_ALERT to be picked up by Alert + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "ERROR" ]; then _Logger "$prefix($level):$value" "$prefix\e[91m$value\e[0m" true ERROR_ALERT=true - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "WARN" ]; then _Logger "$prefix($level):$value" "$prefix\e[33m$value\e[0m" true WARN_ALERT=true - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID.$TSTAMP" + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.WARN_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "NOTICE" ]; then if [ "$_LOGGER_ERR_ONLY" != true ]; then @@ -426,11 +428,11 @@ function GenericTrapQuit { local exitcode=0 # Get ERROR / WARN alert flags from subprocesses that call Logger - if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.WARN_ALERT.$SCRIPT_PID.$TSTAMP" ]; then WARN_ALERT=true exitcode=2 fi - if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" ]; then ERROR_ALERT=true exitcode=1 fi @@ -494,7 +496,11 @@ function SendAlert { fi fi - body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of current log"$'\n\n'"$(tail -n 1000 "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP")" + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ]; then + body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of current log"$'\n\n'"$(tail -n 1000 "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP")" + else + body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of current log"$'\n\n'"$(tail -n 1000 "$LOG_FILE")" + fi if [ $ERROR_ALERT == true ]; then subject="Error alert for $INSTANCE_ID" @@ -2504,16 +2510,8 @@ function FileMove () { mv -f "$source" "$dest" return $? elif [ -w "$source" ]; then - if [ -f "$dest" ]; then # for files we don't need recursive delete - rm -f "$dest" - elif [ -d "$dest" ]; then # for directories we need recursive delete - rm -rf "$dest" - fi - if [ -f "$source" ]; then - cp -p "$source" "$dest" && rm -f "$source" # for files we don't need recursive copy & delete - elif [ -d "$source" ]; then - cp -rp "$source" "$dest" && rm -rf "$source" # for directories we need recursive copy & delete - fi + [ -f "$dest" ] && rm -f "$dest" + cp -p "$source" "$dest" && rm -f "$source" return $? else return -1 From 50f975d2df89e5a2bd518914fb0eee0351f0db2e Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sat, 10 Jun 2023 19:43:16 +0200 Subject: [PATCH 15/38] Add --non-interactive switch --- dev/n_osync.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dev/n_osync.sh b/dev/n_osync.sh index 1ec9022..344f7b3 100644 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -6,8 +6,8 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2022 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" -PROGRAM_VERSION=1.3.0-rc3 -PROGRAM_BUILD=2021062901 +PROGRAM_VERSION=1.3.0-rc4-dev +PROGRAM_BUILD=2023061001 IS_STABLE=true CONFIG_FILE_REVISION_REQUIRED=1.3.0 @@ -2898,6 +2898,7 @@ function Usage { echo "--no-prefix Will suppress time / date suffix from output" echo "--silent Will run osync without any output to stdout, used for cron jobs" echo "--errors-only Output only errors (can be combined with silent or verbose)" + echo "--non-interactive Don't show running animation in cron / service mode" echo "--summary Outputs a list of transferred / deleted files at the end of the run" echo "--log-conflicts [EXPERIMENTAL] Outputs a list of conflicted files" echo "--alert-conflicts Send an email if conflictual files found (implies --log-conflicts)" @@ -3174,6 +3175,10 @@ function GetCommandlineArguments { opts=$opts" --errors-only" _LOGGER_ERR_ONLY=true ;; + --non-interactive) + opts$opts" --non-interactive" + _OFUNCTIONS_SHOW_SPINNER=false + ;; --summary) opts=$opts" --summary" _SUMMARY=true From edd35d42aad4d4dc42464b8f1ebdf1817035aa8a Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sat, 10 Jun 2023 19:44:37 +0200 Subject: [PATCH 16/38] Enable writing partial logs for alerting --- dev/n_osync.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/n_osync.sh b/dev/n_osync.sh index 344f7b3..4aef0dc 100644 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -48,6 +48,7 @@ include #### _OFUNCTIONS_BOOTSTRAP SUBSET #### [ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0) or build it with merge.sh" && exit 1 _LOGGER_PREFIX="time" +_LOGGER_WRITE_PARTIAL_LOGS=true ## Working directory. This directory exists in any replica and contains state files, backups, soft deleted files etc OSYNC_DIR=".osync_workdir" From 5a7455173c51031a8c35cedd251af1a6d3e91a10 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sat, 10 Jun 2023 19:45:33 +0200 Subject: [PATCH 17/38] Fix typo --- dev/ofunctions.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index 0f846c5..7d399bf 100644 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -31,8 +31,8 @@ #### OFUNCTIONS FULL SUBSET #### #### OFUNCTIONS MINI SUBSET #### #### OFUNCTIONS MICRO SUBSET #### -_OFUNCTIONS_VERSION=2.4.4 -_OFUNCTIONS_BUILD=2022081901 +_OFUNCTIONS_VERSION=2.5.0 +_OFUNCTIONS_BUILD=2023061001 #### _OFUNCTIONS_BOOTSTRAP SUBSET #### _OFUNCTIONS_BOOTSTRAP=true #### _OFUNCTIONS_BOOTSTRAP SUBSET END #### @@ -58,6 +58,7 @@ _LOGGER_VERBOSE=false _LOGGER_ERR_ONLY=false _LOGGER_PREFIX="date" _LOGGER_WRITE_PARTIAL_LOGS=false # Writes partial log files to /tmp so sending logs via alerts can feed on them +_OFUNCTIONS_SHOW_SPINNER=true # Show spinner in ExecTasks function if [ "$KEEP_LOGGING" == "" ]; then KEEP_LOGGING=1801 fi @@ -995,7 +996,7 @@ function ExecTasks { # soft / hard execution time checks that needs to be a subfunction since it is called both from main loop and from parallelExec sub loop function _ExecTasksTimeCheck { - if [ $spinner == true ]; then + if [ $spinner == true ] && [ "$_OFUNCTIONS_SHOW_SPINNER" != false ]; then Spinner fi if [ $counting == true ]; then From 83a0f5c328952c292fe653647f671683fe34ff35 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sat, 10 Jun 2023 19:45:58 +0200 Subject: [PATCH 18/38] Rebuilt targets --- dev/debug_osync.sh | 74 ++++++++++++++++++++++------------------------ install.sh | 24 ++++++++------- osync-batch.sh | 14 ++++----- osync.sh | 74 ++++++++++++++++++++++------------------------ 4 files changed, 90 insertions(+), 96 deletions(-) diff --git a/dev/debug_osync.sh b/dev/debug_osync.sh index dbee41d..fb72087 100755 --- a/dev/debug_osync.sh +++ b/dev/debug_osync.sh @@ -6,8 +6,8 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2022 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" -PROGRAM_VERSION=1.3.0-rc3 -PROGRAM_BUILD=2021062901 +PROGRAM_VERSION=1.3.0-rc4-dev +PROGRAM_BUILD=2023061001 IS_STABLE=true CONFIG_FILE_REVISION_REQUIRED=1.3.0 @@ -42,8 +42,8 @@ CONFIG_FILE_REVISION_REQUIRED=1.3.0 # UnlockReplicas yes #__WITH_PARANOIA_DEBUG # CleanUp no #__WITH_PARANOIA_DEBUG -_OFUNCTIONS_VERSION=2.4.3 -_OFUNCTIONS_BUILD=2022050801 +_OFUNCTIONS_VERSION=2.5.0 +_OFUNCTIONS_BUILD=2023061001 _OFUNCTIONS_BOOTSTRAP=true if ! type "$BASH" > /dev/null; then @@ -66,6 +66,8 @@ _LOGGER_SILENT=false _LOGGER_VERBOSE=false _LOGGER_ERR_ONLY=false _LOGGER_PREFIX="date" +_LOGGER_WRITE_PARTIAL_LOGS=false # Writes partial log files to /tmp so sending logs via alerts can feed on them +_OFUNCTIONS_SHOW_SPINNER=true # Show spinner in ExecTasks function if [ "$KEEP_LOGGING" == "" ]; then KEEP_LOGGING=1801 fi @@ -174,7 +176,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -290,18 +292,18 @@ function Logger { if [ "$level" == "CRITICAL" ]; then _Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true ERROR_ALERT=true - # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. Need to keep this flag. - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. We need to create these flag files for ERROR_ALERT / WARN_ALERT to be picked up by Alert + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "ERROR" ]; then _Logger "$prefix($level):$value" "$prefix\e[91m$value\e[0m" true ERROR_ALERT=true - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "WARN" ]; then _Logger "$prefix($level):$value" "$prefix\e[33m$value\e[0m" true WARN_ALERT=true - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID.$TSTAMP" + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.WARN_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "NOTICE" ]; then if [ "$_LOGGER_ERR_ONLY" != true ]; then @@ -423,11 +425,11 @@ function GenericTrapQuit { local exitcode=0 # Get ERROR / WARN alert flags from subprocesses that call Logger - if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.WARN_ALERT.$SCRIPT_PID.$TSTAMP" ]; then WARN_ALERT=true exitcode=2 fi - if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" ]; then ERROR_ALERT=true exitcode=1 fi @@ -487,7 +489,11 @@ function SendAlert { fi fi - body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of current log"$'\n\n'"$(tail -n 1000 "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP")" + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ]; then + body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of current log"$'\n\n'"$(tail -n 1000 "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP")" + else + body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of current log"$'\n\n'"$(tail -n 1000 "$LOG_FILE")" + fi if [ $ERROR_ALERT == true ]; then subject="Error alert for $INSTANCE_ID" @@ -980,7 +986,7 @@ function ExecTasks { # soft / hard execution time checks that needs to be a subfunction since it is called both from main loop and from parallelExec sub loop function _ExecTasksTimeCheck { - if [ $spinner == true ]; then + if [ $spinner == true ] && [ "$_OFUNCTIONS_SHOW_SPINNER" != false ]; then Spinner fi if [ $counting == true ]; then @@ -2477,16 +2483,8 @@ function FileMove () { mv -f "$source" "$dest" return $? elif [ -w "$source" ]; then - if [ -f "$dest" ]; then # for files we don't need recursive delete - rm -f "$dest" - elif [ -d "$dest" ]; then # for directories we need recursive delete - rm -rf "$dest" - fi - if [ -f "$source" ]; then - cp -p "$source" "$dest" && rm -f "$source" # for files we don't need recursive copy & delete - elif [ -d "$source" ]; then - cp -rp "$source" "$dest" && rm -rf "$source" # for directories we need recursive copy & delete - fi + [ -f "$dest" ] && rm -f "$dest" + cp -p "$source" "$dest" && rm -f "$source" return $? else return -1 @@ -2641,6 +2639,7 @@ _OFUNCTIONS_BOOTSTRAP=true [ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0) or build it with merge.sh" && exit 1 _LOGGER_PREFIX="time" +_LOGGER_WRITE_PARTIAL_LOGS=true ## Working directory. This directory exists in any replica and contains state files, backups, soft deleted files etc OSYNC_DIR=".osync_workdir" @@ -3056,7 +3055,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -3485,7 +3484,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -4118,7 +4117,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -4768,7 +4767,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -4873,16 +4872,8 @@ function FileMove () { mv -f "$source" "$dest" return $? elif [ -w "$source" ]; then - if [ -f "$dest" ]; then # for files we don't need recursive delete - rm -f "$dest" - elif [ -d "$dest" ]; then # for directories we need recursive delete - rm -rf "$dest" - fi - if [ -f "$source" ]; then - cp -p "$source" "$dest" && rm -f "$source" # for files we don't need recursive copy & delete - elif [ -d "$source" ]; then - cp -rp "$source" "$dest" && rm -rf "$source" # for directories we need recursive copy & delete - fi + [ -f "$dest" ] && rm -f "$dest" + cp -p "$source" "$dest" && rm -f "$source" return $? else return -1 @@ -5783,7 +5774,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -6071,7 +6062,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -6596,6 +6587,7 @@ function Usage { echo "--no-prefix Will suppress time / date suffix from output" echo "--silent Will run osync without any output to stdout, used for cron jobs" echo "--errors-only Output only errors (can be combined with silent or verbose)" + echo "--non-interactive Don't show running animation in cron / service mode" echo "--summary Outputs a list of transferred / deleted files at the end of the run" echo "--log-conflicts [EXPERIMENTAL] Outputs a list of conflicted files" echo "--alert-conflicts Send an email if conflictual files found (implies --log-conflicts)" @@ -6872,6 +6864,10 @@ function GetCommandlineArguments { opts=$opts" --errors-only" _LOGGER_ERR_ONLY=true ;; + --non-interactive) + opts$opts" --non-interactive" + _OFUNCTIONS_SHOW_SPINNER=false + ;; --summary) opts=$opts" --summary" _SUMMARY=true diff --git a/install.sh b/install.sh index 0b540be..803889a 100755 --- a/install.sh +++ b/install.sh @@ -17,8 +17,8 @@ INSTANCE_ID="installer-$SCRIPT_BUILD" ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11 ## Please adapt this to fit your distro needs -_OFUNCTIONS_VERSION=2.4.3 -_OFUNCTIONS_BUILD=2022050801 +_OFUNCTIONS_VERSION=2.5.0 +_OFUNCTIONS_BUILD=2023061001 _OFUNCTIONS_BOOTSTRAP=true if ! type "$BASH" > /dev/null; then @@ -41,6 +41,8 @@ _LOGGER_SILENT=false _LOGGER_VERBOSE=false _LOGGER_ERR_ONLY=false _LOGGER_PREFIX="date" +_LOGGER_WRITE_PARTIAL_LOGS=false # Writes partial log files to /tmp so sending logs via alerts can feed on them +_OFUNCTIONS_SHOW_SPINNER=true # Show spinner in ExecTasks function if [ "$KEEP_LOGGING" == "" ]; then KEEP_LOGGING=1801 fi @@ -145,7 +147,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -256,18 +258,18 @@ function Logger { if [ "$level" == "CRITICAL" ]; then _Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true ERROR_ALERT=true - # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. Need to keep this flag. - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. We need to create these flag files for ERROR_ALERT / WARN_ALERT to be picked up by Alert + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "ERROR" ]; then _Logger "$prefix($level):$value" "$prefix\e[91m$value\e[0m" true ERROR_ALERT=true - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "WARN" ]; then _Logger "$prefix($level):$value" "$prefix\e[33m$value\e[0m" true WARN_ALERT=true - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID.$TSTAMP" + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.WARN_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "NOTICE" ]; then if [ "$_LOGGER_ERR_ONLY" != true ]; then @@ -382,11 +384,11 @@ function GenericTrapQuit { local exitcode=0 # Get ERROR / WARN alert flags from subprocesses that call Logger - if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.WARN_ALERT.$SCRIPT_PID.$TSTAMP" ]; then WARN_ALERT=true exitcode=2 fi - if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" ]; then ERROR_ALERT=true exitcode=1 fi @@ -601,11 +603,11 @@ function GenericTrapQuit { local exitcode=0 # Get ERROR / WARN alert flags from subprocesses that call Logger - if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.WARN_ALERT.$SCRIPT_PID.$TSTAMP" ]; then WARN_ALERT=true exitcode=2 fi - if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" ]; then ERROR_ALERT=true exitcode=1 fi diff --git a/osync-batch.sh b/osync-batch.sh index 3d0083b..1d57374 100755 --- a/osync-batch.sh +++ b/osync-batch.sh @@ -54,7 +54,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -166,18 +166,18 @@ function Logger { if [ "$level" == "CRITICAL" ]; then _Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true ERROR_ALERT=true - # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. Need to keep this flag. - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. We need to create these flag files for ERROR_ALERT / WARN_ALERT to be picked up by Alert + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "ERROR" ]; then _Logger "$prefix($level):$value" "$prefix\e[91m$value\e[0m" true ERROR_ALERT=true - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "WARN" ]; then _Logger "$prefix($level):$value" "$prefix\e[33m$value\e[0m" true WARN_ALERT=true - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID.$TSTAMP" + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.WARN_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "NOTICE" ]; then if [ "$_LOGGER_ERR_ONLY" != true ]; then @@ -224,11 +224,11 @@ function GenericTrapQuit { local exitcode=0 # Get ERROR / WARN alert flags from subprocesses that call Logger - if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.WARN_ALERT.$SCRIPT_PID.$TSTAMP" ]; then WARN_ALERT=true exitcode=2 fi - if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" ]; then ERROR_ALERT=true exitcode=1 fi diff --git a/osync.sh b/osync.sh index 2631f41..adb6de7 100755 --- a/osync.sh +++ b/osync.sh @@ -6,15 +6,15 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2022 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" -PROGRAM_VERSION=1.3.0-rc3 -PROGRAM_BUILD=2021062901 +PROGRAM_VERSION=1.3.0-rc4-dev +PROGRAM_BUILD=2023061001 IS_STABLE=true CONFIG_FILE_REVISION_REQUIRED=1.3.0 -_OFUNCTIONS_VERSION=2.4.3 -_OFUNCTIONS_BUILD=2022050801 +_OFUNCTIONS_VERSION=2.5.0 +_OFUNCTIONS_BUILD=2023061001 _OFUNCTIONS_BOOTSTRAP=true if ! type "$BASH" > /dev/null; then @@ -37,6 +37,8 @@ _LOGGER_SILENT=false _LOGGER_VERBOSE=false _LOGGER_ERR_ONLY=false _LOGGER_PREFIX="date" +_LOGGER_WRITE_PARTIAL_LOGS=false # Writes partial log files to /tmp so sending logs via alerts can feed on them +_OFUNCTIONS_SHOW_SPINNER=true # Show spinner in ExecTasks function if [ "$KEEP_LOGGING" == "" ]; then KEEP_LOGGING=1801 fi @@ -141,7 +143,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -252,18 +254,18 @@ function Logger { if [ "$level" == "CRITICAL" ]; then _Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true ERROR_ALERT=true - # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. Need to keep this flag. - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. We need to create these flag files for ERROR_ALERT / WARN_ALERT to be picked up by Alert + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "ERROR" ]; then _Logger "$prefix($level):$value" "$prefix\e[91m$value\e[0m" true ERROR_ALERT=true - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "WARN" ]; then _Logger "$prefix($level):$value" "$prefix\e[33m$value\e[0m" true WARN_ALERT=true - echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID.$TSTAMP" + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.WARN_ALERT.$SCRIPT_PID.$TSTAMP" return elif [ "$level" == "NOTICE" ]; then if [ "$_LOGGER_ERR_ONLY" != true ]; then @@ -378,11 +380,11 @@ function GenericTrapQuit { local exitcode=0 # Get ERROR / WARN alert flags from subprocesses that call Logger - if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.WARN_ALERT.$SCRIPT_PID.$TSTAMP" ]; then WARN_ALERT=true exitcode=2 fi - if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then + if [ -f "$RUN_DIR/$PROGRAM.ERROR_ALERT.$SCRIPT_PID.$TSTAMP" ]; then ERROR_ALERT=true exitcode=1 fi @@ -441,7 +443,11 @@ function SendAlert { fi fi - body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of current log"$'\n\n'"$(tail -n 1000 "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP")" + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ]; then + body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of current log"$'\n\n'"$(tail -n 1000 "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP")" + else + body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of current log"$'\n\n'"$(tail -n 1000 "$LOG_FILE")" + fi if [ $ERROR_ALERT == true ]; then subject="Error alert for $INSTANCE_ID" @@ -912,7 +918,7 @@ function ExecTasks { # soft / hard execution time checks that needs to be a subfunction since it is called both from main loop and from parallelExec sub loop function _ExecTasksTimeCheck { - if [ $spinner == true ]; then + if [ $spinner == true ] && [ "$_OFUNCTIONS_SHOW_SPINNER" != false ]; then Spinner fi if [ $counting == true ]; then @@ -2327,16 +2333,8 @@ function FileMove () { mv -f "$source" "$dest" return $? elif [ -w "$source" ]; then - if [ -f "$dest" ]; then # for files we don't need recursive delete - rm -f "$dest" - elif [ -d "$dest" ]; then # for directories we need recursive delete - rm -rf "$dest" - fi - if [ -f "$source" ]; then - cp -p "$source" "$dest" && rm -f "$source" # for files we don't need recursive copy & delete - elif [ -d "$source" ]; then - cp -rp "$source" "$dest" && rm -rf "$source" # for directories we need recursive copy & delete - fi + [ -f "$dest" ] && rm -f "$dest" + cp -p "$source" "$dest" && rm -f "$source" return $? else return -1 @@ -2491,6 +2489,7 @@ _OFUNCTIONS_BOOTSTRAP=true [ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0) or build it with merge.sh" && exit 1 _LOGGER_PREFIX="time" +_LOGGER_WRITE_PARTIAL_LOGS=true ## Working directory. This directory exists in any replica and contains state files, backups, soft deleted files etc OSYNC_DIR=".osync_workdir" @@ -2897,7 +2896,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -3314,7 +3313,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -3930,7 +3929,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -4565,7 +4564,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -4665,16 +4664,8 @@ function FileMove () { mv -f "$source" "$dest" return $? elif [ -w "$source" ]; then - if [ -f "$dest" ]; then # for files we don't need recursive delete - rm -f "$dest" - elif [ -d "$dest" ]; then # for directories we need recursive delete - rm -rf "$dest" - fi - if [ -f "$source" ]; then - cp -p "$source" "$dest" && rm -f "$source" # for files we don't need recursive copy & delete - elif [ -d "$source" ]; then - cp -rp "$source" "$dest" && rm -rf "$source" # for directories we need recursive copy & delete - fi + [ -f "$dest" ] && rm -f "$dest" + cp -p "$source" "$dest" && rm -f "$source" return $? else return -1 @@ -5566,7 +5557,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -5842,7 +5833,7 @@ function _Logger { echo -e "$logValue" >> "$LOG_FILE" # Build current log file for alerts if we have a sufficient environment - if [ "$RUN_DIR/$PROGRAM" != "/" ]; then + if [ "$_LOGGER_WRITE_PARTIAL_LOGS" == true ] && [ "$RUN_DIR/$PROGRAM" != "/" ]; then echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" fi fi @@ -6354,6 +6345,7 @@ function Usage { echo "--no-prefix Will suppress time / date suffix from output" echo "--silent Will run osync without any output to stdout, used for cron jobs" echo "--errors-only Output only errors (can be combined with silent or verbose)" + echo "--non-interactive Don't show running animation in cron / service mode" echo "--summary Outputs a list of transferred / deleted files at the end of the run" echo "--log-conflicts [EXPERIMENTAL] Outputs a list of conflicted files" echo "--alert-conflicts Send an email if conflictual files found (implies --log-conflicts)" @@ -6629,6 +6621,10 @@ function GetCommandlineArguments { opts=$opts" --errors-only" _LOGGER_ERR_ONLY=true ;; + --non-interactive) + opts$opts" --non-interactive" + _OFUNCTIONS_SHOW_SPINNER=false + ;; --summary) opts=$opts" --summary" _SUMMARY=true From 833c16ca4363dbbccacab18ac2a3a8565dd1ba99 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sun, 11 Jun 2023 09:43:44 +0200 Subject: [PATCH 19/38] Move osync test dir to /tmp --- dev/tests/run_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tests/run_tests.sh b/dev/tests/run_tests.sh index 095427c..02a12a0 100755 --- a/dev/tests/run_tests.sh +++ b/dev/tests/run_tests.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# osync test suite 2023060901 +# osync test suite 2023061101 # Allows the following environment variables @@ -79,7 +79,7 @@ OSYNC_UPGRADE="upgrade-v1.0x-v1.3x.sh" TMP_FILE="$DEV_DIR/tmp" -OSYNC_TESTS_DIR="${homedir}/osync-tests" +OSYNC_TESTS_DIR="/tmp/osync-tests" INITIATOR_DIR="$OSYNC_TESTS_DIR/initiator" TARGET_DIR="$OSYNC_TESTS_DIR/target" OSYNC_WORKDIR=".osync_workdir" From 57ef91936406fa84247b31b78d0d1d6ffebe317c Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sun, 11 Jun 2023 11:04:44 +0200 Subject: [PATCH 20/38] Update sync paths. Remove log garbage --- dev/tests/run_tests.sh | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/dev/tests/run_tests.sh b/dev/tests/run_tests.sh index 02a12a0..2624990 100755 --- a/dev/tests/run_tests.sh +++ b/dev/tests/run_tests.sh @@ -79,7 +79,7 @@ OSYNC_UPGRADE="upgrade-v1.0x-v1.3x.sh" TMP_FILE="$DEV_DIR/tmp" -OSYNC_TESTS_DIR="/tmp/osync-tests" +OSYNC_TESTS_DIR="${homedir}/osync-tests" INITIATOR_DIR="$OSYNC_TESTS_DIR/initiator" TARGET_DIR="$OSYNC_TESTS_DIR/target" OSYNC_WORKDIR=".osync_workdir" @@ -224,6 +224,7 @@ function oneTimeSetUp () { SetConfFileValue "$CONF_DIR/$OLD_CONF" "REMOTE_3RD_PARTY_HOSTS" "" SetConfFileValue "$CONF_DIR/$OLD_CONF" "REMOTE_HOST_PING" false + else echo "Running with local settings" REMOTE_USER="root" @@ -236,6 +237,17 @@ function oneTimeSetUp () { SetConfFileValue "$CONF_DIR/$OLD_CONF" "REMOTE_HOST_PING" true fi + + # Fix test directories for Github actions + SetConfFileValue "$CONF_DIR/$LOCAL_CONF" INITIATOR_SYNC_DIR "\"${homedir}/osync-tests/initiator\"" + SetConfFileValue "$CONF_DIR/$LOCAL_CONF" TARGET_SYNC_DIR "\"${homedir}/osync-tests/target\"" + + SetConfFileValue "$CONF_DIR/$REMOTE_CONF" INITIATOR_SYNC_DIR "\"${homedir}/osync-tests/initiator\"" + + SetConfFileValue "$CONF_DIR/$OLD_CONF" MASTER_SYNC_DIR "\"${homedir}/osync-tests/initiator\"" + SetConfFileValue "$CONF_DIR/$OLD_CONF" SLAVE_SYNC_DIR "\"${homedir}/osync-tests/target\"" + + # Get default ssh port from env if [ "$SSH_PORT" == "" ]; then SSH_PORT=22 @@ -249,8 +261,8 @@ function oneTimeSetUp () { readonly __confRemote=3 osyncParameters=() - osyncParameters[$__quickLocal]="--initiator=$INITIATOR_DIR --target=$TARGET_DIR --instance-id=quicklocal" - osyncParameters[$__confLocal]="$CONF_DIR/$LOCAL_CONF" + osyncParameters[$__quickLocal]="--initiator=$INITIATOR_DIR --target=$TARGET_DIR --instance-id=quicklocal --non-interactive" + osyncParameters[$__confLocal]="$CONF_DIR/$LOCAL_CONF --non-interactive" osyncDaemonParameters=() @@ -261,8 +273,8 @@ function oneTimeSetUp () { # Do not check remote config on msys or cygwin since we don't have a local SSH server if [ "$LOCAL_OS" != "msys" ] && [ "$LOCAL_OS" != "Cygwin" ] && [ $SKIP_REMOTE != true ]; then - osyncParameters[$__quickRemote]="--initiator=$INITIATOR_DIR --target=ssh://localhost:$SSH_PORT/$TARGET_DIR --rsakey=${homedir}/.ssh/id_rsa_local_osync_tests --instance-id=quickremote --remote-token=SomeAlphaNumericToken9" - osyncParameters[$__confRemote]="$CONF_DIR/$REMOTE_CONF" + osyncParameters[$__quickRemote]="--initiator=$INITIATOR_DIR --target=ssh://localhost:$SSH_PORT/$TARGET_DIR --rsakey=${homedir}/.ssh/id_rsa_local_osync_tests --instance-id=quickremote --remote-token=SomeAlphaNumericToken9 --non-interactive" + osyncParameters[$__confRemote]="$CONF_DIR/$REMOTE_CONF --non-interactive" osyncDaemonParameters[$__remote]="$CONF_DIR/$REMOTE_CONF --on-changes" From 9b862ae49dfc41ad31080ed451faac1a3ce454cf Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sun, 11 Jun 2023 11:18:39 +0200 Subject: [PATCH 21/38] Fix typo --- dev/n_osync.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/n_osync.sh b/dev/n_osync.sh index 4aef0dc..d25b3cb 100644 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -7,7 +7,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2022 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.3.0-rc4-dev -PROGRAM_BUILD=2023061001 +PROGRAM_BUILD=2023061101 IS_STABLE=true CONFIG_FILE_REVISION_REQUIRED=1.3.0 @@ -3177,7 +3177,7 @@ function GetCommandlineArguments { _LOGGER_ERR_ONLY=true ;; --non-interactive) - opts$opts" --non-interactive" + opts=$opts" --non-interactive" _OFUNCTIONS_SHOW_SPINNER=false ;; --summary) From 9b9c1105eb93169382ddf54b0380589392ad89b3 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sun, 11 Jun 2023 11:19:25 +0200 Subject: [PATCH 22/38] Rebuilt targets --- dev/debug_osync.sh | 4 ++-- osync.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/debug_osync.sh b/dev/debug_osync.sh index fb72087..0dc0915 100755 --- a/dev/debug_osync.sh +++ b/dev/debug_osync.sh @@ -7,7 +7,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2022 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.3.0-rc4-dev -PROGRAM_BUILD=2023061001 +PROGRAM_BUILD=2023061101 IS_STABLE=true CONFIG_FILE_REVISION_REQUIRED=1.3.0 @@ -6865,7 +6865,7 @@ function GetCommandlineArguments { _LOGGER_ERR_ONLY=true ;; --non-interactive) - opts$opts" --non-interactive" + opts=$opts" --non-interactive" _OFUNCTIONS_SHOW_SPINNER=false ;; --summary) diff --git a/osync.sh b/osync.sh index adb6de7..cf1a852 100755 --- a/osync.sh +++ b/osync.sh @@ -7,7 +7,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2022 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.3.0-rc4-dev -PROGRAM_BUILD=2023061001 +PROGRAM_BUILD=2023061101 IS_STABLE=true CONFIG_FILE_REVISION_REQUIRED=1.3.0 @@ -6622,7 +6622,7 @@ function GetCommandlineArguments { _LOGGER_ERR_ONLY=true ;; --non-interactive) - opts$opts" --non-interactive" + opts=$opts" --non-interactive" _OFUNCTIONS_SHOW_SPINNER=false ;; --summary) From d0af0804a0c6994dd0024f1331f34e123e02a3d0 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sun, 11 Jun 2023 11:24:38 +0200 Subject: [PATCH 23/38] Fix systemd detection on windows --- dev/common_install.sh | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/dev/common_install.sh b/dev/common_install.sh index 6f57262..217b937 100644 --- a/dev/common_install.sh +++ b/dev/common_install.sh @@ -10,7 +10,7 @@ PROGRAM_BINARY=$PROGRAM".sh" PROGRAM_BATCH=$PROGRAM"-batch.sh" SSH_FILTER="ssh_filter.sh" -SCRIPT_BUILD=2020112901 +SCRIPT_BUILD=2023061101 INSTANCE_ID="installer-$SCRIPT_BUILD" ## osync / obackup / pmocr / zsnap install script @@ -80,15 +80,26 @@ function SetLocalOSSettings { } function GetInit { + init="none" if [ -f /sbin/openrc-run ]; then init="openrc" Logger "Detected openrc." "NOTICE" + elif [ -f /usr/lib/systemd/systemd ]; then + init="systemd" + Logger "Detected systemd." "NOTICE" elif [ -f /sbin/init ]; then - if file /sbin/init | grep systemd > /dev/null; then - init="systemd" - Logger "Detected systemd." "NOTICE" + if type -p file > /dev/null 2>&1; then + if file /sbin/init | grep systemd > /dev/null; then + init="systemd" + Logger "Detected systemd." "NOTICE" + else + init="initV" + fi else init="initV" + fi + + if [ $init == "initV" ]; then Logger "Detected initV." "NOTICE" fi else From d9e22b06e5daa7cb06c2737b2d858b052f13b8c7 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Sun, 11 Jun 2023 11:26:02 +0200 Subject: [PATCH 24/38] Add preflight merge --- dev/tests/run_tests.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dev/tests/run_tests.sh b/dev/tests/run_tests.sh index 2624990..9c6c584 100755 --- a/dev/tests/run_tests.sh +++ b/dev/tests/run_tests.sh @@ -200,6 +200,10 @@ function PrepareLocalDirs () { function oneTimeSetUp () { START_TIME=$SECONDS + echo "Running forced merge" + cd "${DEV_DIR}" + ./merge.sh osync + echo "Running install.sh from ${OSYNC_DIR}" ls -alh ${OSYNC_DIR} $SUDO_CMD ${OSYNC_DIR}/install.sh --prefix="${FAKEROOT}" From 9ade468ed8506a729c97e24b976ea7e511cc3432 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Wed, 14 Jun 2023 11:10:24 +0200 Subject: [PATCH 25/38] Add permission fixes for tests --- dev/tests/run_tests.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dev/tests/run_tests.sh b/dev/tests/run_tests.sh index 9c6c584..89db1b5 100755 --- a/dev/tests/run_tests.sh +++ b/dev/tests/run_tests.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# osync test suite 2023061101 +# osync test suite 2023061401 # Allows the following environment variables @@ -202,10 +202,13 @@ function oneTimeSetUp () { echo "Running forced merge" cd "${DEV_DIR}" - ./merge.sh osync + $SUDO_CMD ./merge.sh osync + echo "Setting security for files" + $SUDO_CMD find ${OSYNC_DIR} -exec chmod 755 {} \+ - echo "Running install.sh from ${OSYNC_DIR}" + echo "Show content of osync dir" ls -alh ${OSYNC_DIR} + echo "Running install.sh from ${OSYNC_DIR}" $SUDO_CMD ${OSYNC_DIR}/install.sh --prefix="${FAKEROOT}" source "$DEV_DIR/ofunctions.sh" From edfc292486126d793cbd40f399a4cb3fcdcf45cd Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Wed, 14 Jun 2023 11:25:12 +0200 Subject: [PATCH 26/38] Temporarily disable merge test --- dev/tests/run_tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/tests/run_tests.sh b/dev/tests/run_tests.sh index 89db1b5..6a21830 100755 --- a/dev/tests/run_tests.sh +++ b/dev/tests/run_tests.sh @@ -200,9 +200,9 @@ function PrepareLocalDirs () { function oneTimeSetUp () { START_TIME=$SECONDS - echo "Running forced merge" - cd "${DEV_DIR}" - $SUDO_CMD ./merge.sh osync + #echo "Running forced merge" + #cd "${DEV_DIR}" + #$SUDO_CMD ./merge.sh osync echo "Setting security for files" $SUDO_CMD find ${OSYNC_DIR} -exec chmod 755 {} \+ From 776c07386db6539f71b2a7fab27cef126e4cef25 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Wed, 14 Jun 2023 13:49:18 +0200 Subject: [PATCH 27/38] Fix tests --- dev/ofunctions.sh | 12 ++++++++++-- dev/tests/run_tests.sh | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) mode change 100644 => 100755 dev/ofunctions.sh diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh old mode 100644 new mode 100755 index 7d399bf..4f1f958 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -31,8 +31,8 @@ #### OFUNCTIONS FULL SUBSET #### #### OFUNCTIONS MINI SUBSET #### #### OFUNCTIONS MICRO SUBSET #### -_OFUNCTIONS_VERSION=2.5.0 -_OFUNCTIONS_BUILD=2023061001 +_OFUNCTIONS_VERSION=2.5.1 +_OFUNCTIONS_BUILD=2023061401 #### _OFUNCTIONS_BOOTSTRAP SUBSET #### _OFUNCTIONS_BOOTSTRAP=true #### _OFUNCTIONS_BOOTSTRAP SUBSET END #### @@ -2275,6 +2275,13 @@ function InitRemoteOSDependingSettings { ## Set rsync default arguments (complete with -r or -d depending on recursivity later) RSYNC_DEFAULT_ARGS="-ltD -8" + + ## NPF-MOD: Regarding #242, we need to add --old-args if rsync > 3.2.3 + rsync_version=$("${RSYNC_EXECUTABLE}" --version 2>/dev/null| head -1 | awk '{print $3}') + if [ $(Vercomp $rsync_version 3.2.3) -eq 1 ]; then + RSYNC_DEFAULT_ARGS="$RSYNC_DEFAULT_ARGS --old-args" + fi + if [ "$_DRYRUN" == true ]; then RSYNC_DRY_ARG="-n" DRY_WARNING="/!\ DRY RUN " @@ -2282,6 +2289,7 @@ function InitRemoteOSDependingSettings { RSYNC_DRY_ARG="" fi + RSYNC_ATTR_ARGS="" if [ "$PRESERVE_PERMISSIONS" != false ]; then RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -p" diff --git a/dev/tests/run_tests.sh b/dev/tests/run_tests.sh index 6a21830..e5d8b03 100755 --- a/dev/tests/run_tests.sh +++ b/dev/tests/run_tests.sh @@ -209,7 +209,7 @@ function oneTimeSetUp () { echo "Show content of osync dir" ls -alh ${OSYNC_DIR} echo "Running install.sh from ${OSYNC_DIR}" - $SUDO_CMD ${OSYNC_DIR}/install.sh --prefix="${FAKEROOT}" + $SUDO_CMD ${OSYNC_DIR}/install.sh --no-stats --prefix="${FAKEROOT}" source "$DEV_DIR/ofunctions.sh" # Fix default umask because of ACL test that expects 0022 when creating test files From 99f107fcb084fa52997b1a106825f54ae7fe793a Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Wed, 14 Jun 2023 13:49:36 +0200 Subject: [PATCH 28/38] Add general fix for #242 --- dev/ofunctions.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index 4f1f958..777aeed 100755 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -2277,10 +2277,13 @@ function InitRemoteOSDependingSettings { RSYNC_DEFAULT_ARGS="-ltD -8" ## NPF-MOD: Regarding #242, we need to add --old-args if rsync > 3.2.3 - rsync_version=$("${RSYNC_EXECUTABLE}" --version 2>/dev/null| head -1 | awk '{print $3}') - if [ $(Vercomp $rsync_version 3.2.3) -eq 1 ]; then - RSYNC_DEFAULT_ARGS="$RSYNC_DEFAULT_ARGS --old-args" - fi + #rsync_version=$("${RSYNC_EXECUTABLE}" --version 2>/dev/null| head -1 | awk '{print $3}') + #if [ $(VerComp $rsync_version 3.2.3) -eq 1 ]; then + # RSYNC_DEFAULT_ARGS="$RSYNC_DEFAULT_ARGS --old-args" + #fi + # NPF-MOD: Strangely enough, also happens on RHEL7 rsync 3.1.1 + # Let's resolve this easier + RSYNC_OLD_ARGS=1 if [ "$_DRYRUN" == true ]; then RSYNC_DRY_ARG="-n" From c4349bc94527ccc47e2ba71a4b6e3fa1b6e0330f Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Wed, 14 Jun 2023 13:50:04 +0200 Subject: [PATCH 29/38] Rebuilt targets --- dev/debug_osync.sh | 15 +++++++++++++-- install.sh | 23 +++++++++++++++++------ osync.sh | 15 +++++++++++++-- 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/dev/debug_osync.sh b/dev/debug_osync.sh index 0dc0915..2136dd7 100755 --- a/dev/debug_osync.sh +++ b/dev/debug_osync.sh @@ -42,8 +42,8 @@ CONFIG_FILE_REVISION_REQUIRED=1.3.0 # UnlockReplicas yes #__WITH_PARANOIA_DEBUG # CleanUp no #__WITH_PARANOIA_DEBUG -_OFUNCTIONS_VERSION=2.5.0 -_OFUNCTIONS_BUILD=2023061001 +_OFUNCTIONS_VERSION=2.5.1 +_OFUNCTIONS_BUILD=2023061401 _OFUNCTIONS_BOOTSTRAP=true if ! type "$BASH" > /dev/null; then @@ -2254,6 +2254,16 @@ function InitRemoteOSDependingSettings { ## Set rsync default arguments (complete with -r or -d depending on recursivity later) RSYNC_DEFAULT_ARGS="-ltD -8" + + ## NPF-MOD: Regarding #242, we need to add --old-args if rsync > 3.2.3 + #rsync_version=$("${RSYNC_EXECUTABLE}" --version 2>/dev/null| head -1 | awk '{print $3}') + #if [ $(VerComp $rsync_version 3.2.3) -eq 1 ]; then + # RSYNC_DEFAULT_ARGS="$RSYNC_DEFAULT_ARGS --old-args" + #fi + # NPF-MOD: Strangely enough, also happens on RHEL7 rsync 3.1.1 + # Let's resolve this easier + RSYNC_OLD_ARGS=1 + if [ "$_DRYRUN" == true ]; then RSYNC_DRY_ARG="-n" DRY_WARNING="/!\ DRY RUN " @@ -2261,6 +2271,7 @@ function InitRemoteOSDependingSettings { RSYNC_DRY_ARG="" fi + RSYNC_ATTR_ARGS="" if [ "$PRESERVE_PERMISSIONS" != false ]; then RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -p" diff --git a/install.sh b/install.sh index 803889a..f6a3556 100755 --- a/install.sh +++ b/install.sh @@ -10,15 +10,15 @@ PROGRAM_BINARY=$PROGRAM".sh" PROGRAM_BATCH=$PROGRAM"-batch.sh" SSH_FILTER="ssh_filter.sh" -SCRIPT_BUILD=2020112901 +SCRIPT_BUILD=2023061101 INSTANCE_ID="installer-$SCRIPT_BUILD" ## osync / obackup / pmocr / zsnap install script ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11 ## Please adapt this to fit your distro needs -_OFUNCTIONS_VERSION=2.5.0 -_OFUNCTIONS_BUILD=2023061001 +_OFUNCTIONS_VERSION=2.5.1 +_OFUNCTIONS_BUILD=2023061401 _OFUNCTIONS_BOOTSTRAP=true if ! type "$BASH" > /dev/null; then @@ -655,15 +655,26 @@ function SetLocalOSSettings { } function GetInit { + init="none" if [ -f /sbin/openrc-run ]; then init="openrc" Logger "Detected openrc." "NOTICE" + elif [ -f /usr/lib/systemd/systemd ]; then + init="systemd" + Logger "Detected systemd." "NOTICE" elif [ -f /sbin/init ]; then - if file /sbin/init | grep systemd > /dev/null; then - init="systemd" - Logger "Detected systemd." "NOTICE" + if type -p file > /dev/null 2>&1; then + if file /sbin/init | grep systemd > /dev/null; then + init="systemd" + Logger "Detected systemd." "NOTICE" + else + init="initV" + fi else init="initV" + fi + + if [ $init == "initV" ]; then Logger "Detected initV." "NOTICE" fi else diff --git a/osync.sh b/osync.sh index cf1a852..ca042ed 100755 --- a/osync.sh +++ b/osync.sh @@ -13,8 +13,8 @@ IS_STABLE=true CONFIG_FILE_REVISION_REQUIRED=1.3.0 -_OFUNCTIONS_VERSION=2.5.0 -_OFUNCTIONS_BUILD=2023061001 +_OFUNCTIONS_VERSION=2.5.1 +_OFUNCTIONS_BUILD=2023061401 _OFUNCTIONS_BOOTSTRAP=true if ! type "$BASH" > /dev/null; then @@ -2104,6 +2104,16 @@ function InitRemoteOSDependingSettings { ## Set rsync default arguments (complete with -r or -d depending on recursivity later) RSYNC_DEFAULT_ARGS="-ltD -8" + + ## NPF-MOD: Regarding #242, we need to add --old-args if rsync > 3.2.3 + #rsync_version=$("${RSYNC_EXECUTABLE}" --version 2>/dev/null| head -1 | awk '{print $3}') + #if [ $(VerComp $rsync_version 3.2.3) -eq 1 ]; then + # RSYNC_DEFAULT_ARGS="$RSYNC_DEFAULT_ARGS --old-args" + #fi + # NPF-MOD: Strangely enough, also happens on RHEL7 rsync 3.1.1 + # Let's resolve this easier + RSYNC_OLD_ARGS=1 + if [ "$_DRYRUN" == true ]; then RSYNC_DRY_ARG="-n" DRY_WARNING="/!\ DRY RUN " @@ -2111,6 +2121,7 @@ function InitRemoteOSDependingSettings { RSYNC_DRY_ARG="" fi + RSYNC_ATTR_ARGS="" if [ "$PRESERVE_PERMISSIONS" != false ]; then RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -p" From b2f221cb8675e285ed0446313075f02b4df02be6 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Wed, 14 Jun 2023 14:06:18 +0200 Subject: [PATCH 30/38] Make rsync old arguments fix permanent --- dev/ofunctions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index 777aeed..a1a3235 100755 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -2283,7 +2283,7 @@ function InitRemoteOSDependingSettings { #fi # NPF-MOD: Strangely enough, also happens on RHEL7 rsync 3.1.1 # Let's resolve this easier - RSYNC_OLD_ARGS=1 + export RSYNC_OLD_ARGS=1 if [ "$_DRYRUN" == true ]; then RSYNC_DRY_ARG="-n" From 988009a438a6b6455783d2933b8af65a80bbe72a Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Wed, 14 Jun 2023 14:06:40 +0200 Subject: [PATCH 31/38] =?UTF-8?q?Rebu=C3=83ilt=20targets?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dev/debug_osync.sh | 2 +- osync.sh | 245 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 245 insertions(+), 2 deletions(-) diff --git a/dev/debug_osync.sh b/dev/debug_osync.sh index 2136dd7..11424c1 100755 --- a/dev/debug_osync.sh +++ b/dev/debug_osync.sh @@ -2262,7 +2262,7 @@ function InitRemoteOSDependingSettings { #fi # NPF-MOD: Strangely enough, also happens on RHEL7 rsync 3.1.1 # Let's resolve this easier - RSYNC_OLD_ARGS=1 + export RSYNC_OLD_ARGS=1 if [ "$_DRYRUN" == true ]; then RSYNC_DRY_ARG="-n" diff --git a/osync.sh b/osync.sh index ca042ed..11424c1 100755 --- a/osync.sh +++ b/osync.sh @@ -12,6 +12,35 @@ IS_STABLE=true CONFIG_FILE_REVISION_REQUIRED=1.3.0 +##### Execution order #__WITH_PARANOIA_DEBUG +##### Function Name Is parallel #__WITH_PARANOIA_DEBUG +# GetLocalOS no #__WITH_PARANOIA_DEBUG +# InitLocalOSDependingSettings no #__WITH_PARANOIA_DEBUG +# CheckEnvironment no #__WITH_PARANOIA_DEBUG +# PreInit no #__WITH_PARANOIA_DEBUG +# Init no #__WITH_PARANOIA_DEBUG +# PostInit no #__WITH_PARANOIA_DEBUG +# GetRemoteOS no #__WITH_PARANOIA_DEBUG +# InitRemoteOSDependingSettings no #__WITH_PARANOIA_DEBUG +# CheckReplicas yes #__WITH_PARANOIA_DEBUG +# RunBeforeHook yes #__WITH_PARANOIA_DEBUG +# Main no #__WITH_PARANOIA_DEBUG +# HandleLocks yes #__WITH_PARANOIA_DEBUG +# Sync no #__WITH_PARANOIA_DEBUG +# treeList yes #__WITH_PARANOIA_DEBUG +# deleteList yes #__WITH_PARANOIA_DEBUG +# timestampList yes #__WITH_PARANOIA_DEBUG +# conflictList no #__WITH_PARANOIA_DEBUG +# syncAttrs no #__WITH_PARANOIA_DEBUG +# syncUpdate no #__WITH_PARANOIA_DEBUG +# syncUpdate no #__WITH_PARANOIA_DEBUG +# deletionPropagation yes #__WITH_PARANOIA_DEBUG +# treeList yes #__WITH_PARANOIA_DEBUG +# timestampList yes #__WITH_PARANOIA_DEBUG +# SoftDelete yes #__WITH_PARANOIA_DEBUG +# RunAfterHook yes #__WITH_PARANOIA_DEBUG +# UnlockReplicas yes #__WITH_PARANOIA_DEBUG +# CleanUp no #__WITH_PARANOIA_DEBUG _OFUNCTIONS_VERSION=2.5.1 _OFUNCTIONS_BUILD=2023061401 @@ -47,6 +76,10 @@ fi ERROR_ALERT=false WARN_ALERT=false +## allow function call checks #__WITH_PARANOIA_DEBUG +if [ "$_PARANOIA_DEBUG" == true ];then #__WITH_PARANOIA_DEBUG + _DEBUG=true #__WITH_PARANOIA_DEBUG +fi #__WITH_PARANOIA_DEBUG ## allow debugging from command line with _DEBUG=true if [ ! "$_DEBUG" == true ]; then @@ -211,6 +244,11 @@ function RemoteLogger { _Logger "" "$prefix$value" return fi + elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG + if [ "$_PARANOIA_DEBUG" == true ]; then #__WITH_PARANOIA_DEBUG + _Logger "" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG + return #__WITH_PARANOIA_DEBUG + fi #__WITH_PARANOIA_DEBUG else _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "" "Value was: $prefix$value" true @@ -285,6 +323,11 @@ function Logger { _Logger "$prefix$value" "$prefix$value" return fi + elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG + if [ "$_PARANOIA_DEBUG" == true ]; then #__WITH_PARANOIA_DEBUG + _Logger "$prefix$value" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG + return #__WITH_PARANOIA_DEBUG + fi #__WITH_PARANOIA_DEBUG else _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "Value was: $prefix$value" "Value was: $prefix$value" true @@ -329,6 +372,7 @@ function KillChilds { children="${children/$pid/}" fi for child in $children; do + Logger "Launching KillChilds \"$child\" true" "DEBUG" #__WITH_PARANOIA_DEBUG KillChilds "$child" true done fi @@ -363,6 +407,7 @@ function KillAllChilds { local pids="${1}" # List of parent pids to kill separated by semi-colon local self="${2:-false}" # Should parent be killed too ? + __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG local errorcount=0 @@ -419,6 +464,7 @@ function SendAlert { local runAlert="${1:-false}" # Specifies if current message is sent while running or at the end of a run local attachment="${2:-true}" # Should we send the log file as attachment + __CheckArguments 0-2 $# "$@" #__WITH_PARANOIA_DEBUG local attachmentFile local subject @@ -500,6 +546,7 @@ function SendEmail { local smtpUser="${9}" local smtpPassword="${10}" + __CheckArguments 3-10 $# "$@" #__WITH_PARANOIA_DEBUG local mail_no_attachment= local attachment_command= @@ -680,6 +727,7 @@ function LoadConfigFile { local configFile="${1}" local revisionRequired="${2}" + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local revisionPresent @@ -709,6 +757,22 @@ function LoadConfigFile { } # Quick and dirty performance logger only used for debugging +function _PerfProfiler { #__WITH_PARANOIA_DEBUG + local perfString #__WITH_PARANOIA_DEBUG + local i #__WITH_PARANOIA_DEBUG + #__WITH_PARANOIA_DEBUG + perfString=$(ps -p $$ -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan) #__WITH_PARANOIA_DEBUG + #__WITH_PARANOIA_DEBUG + for i in $(pgrep -P $$); do #__WITH_PARANOIA_DEBUG + perfString="$perfString\n"$(ps -p $i -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan | tail -1) #__WITH_PARANOIA_DEBUG + done #__WITH_PARANOIA_DEBUG + #__WITH_PARANOIA_DEBUG + if type iostat > /dev/null 2>&1; then #__WITH_PARANOIA_DEBUG + perfString="$perfString\n"$(iostat) #__WITH_PARANOIA_DEBUG + fi #__WITH_PARANOIA_DEBUG + #__WITH_PARANOIA_DEBUG + Logger "PerfProfiler:\n$perfString" "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG +} #__WITH_PARANOIA_DEBUG _OFUNCTIONS_SPINNER="|/-\\" function Spinner { @@ -811,9 +875,11 @@ function ExecTasks { local minTimeBetweenRetries="${17:-300}" # Time (in seconds) between postponed command retries local validExitCodes="${18:-0}" # Semi colon separated list of valid main command exit codes which will not trigger errors + __CheckArguments 1-18 $# "$@" #__WITH_PARANOIA_DEBUG local i + Logger "${FUNCNAME[0]} id [$id] called by [${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} < ${FUNCNAME[5]} < ${FUNCNAME[6]} ...]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG # Since ExecTasks takes up to 17 arguments, do a quick preflight check in DEBUG mode if [ "$_DEBUG" == true ]; then @@ -866,6 +932,7 @@ function ExecTasks { local newPidsArray # New array of currently running pids for next iteration local pidsTimeArray # Array containing execution begin time of pids local executeCommand # Boolean to check if currentCommand can be executed given a condition + local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG local functionMode local softAlert=false # Does a soft alert need to be triggered, if yes, send an alert once local failedPidsList # List containing failed pids with exit code separated by semicolons (eg : 2355:1;4534:2;2354:3) @@ -915,6 +982,7 @@ function ExecTasks { counter=$mainItemCount fi + Logger "Running ${FUNCNAME[0]} as [$functionMode] for [$mainItemCount] mainItems and [$auxItemCount] auxItems." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG # soft / hard execution time checks that needs to be a subfunction since it is called both from main loop and from parallelExec sub loop function _ExecTasksTimeCheck { @@ -1064,15 +1132,22 @@ function ExecTasks { Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG" fi fi + hasPids=true ##__WITH_PARANOIA_DEBUG fi done # hasPids can be false on last iteration in ParallelExec mode + if [ $hasPids == false ] && [ "$functionMode" = "WaitForTaskCompletion" ]; then ##__WITH_PARANOIA_DEBUG + Logger "No valable pids given." "ERROR" ##__WITH_PARANOIA_DEBUG + fi ##__WITH_PARANOIA_DEBUG pidsArray=("${newPidsArray[@]}") # Trivial wait time for bash to not eat up all CPU sleep $sleepTime + if [ "$_PERF_PROFILER" == true ]; then ##__WITH_PARANOIA_DEBUG + _PerfProfiler ##__WITH_PARANOIA_DEBUG + fi ##__WITH_PARANOIA_DEBUG } @@ -1210,6 +1285,7 @@ function ExecTasks { _ExecTasksPidsCheck done + Logger "${FUNCNAME[0]} ended for [$id] using [$mainItemCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG # Return exit code if only one process was monitored, else return number of errors # As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value @@ -1490,9 +1566,65 @@ function GetLocalOS { fi } +#__BEGIN_WITH_PARANOIA_DEBUG +function __CheckArguments { + # Checks the number of arguments of a function and raises an error if some are missing + + if [ "$_DEBUG" == true ]; then + local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments + local numberOfGivenArguments="${2}" # Number of arguments that have been passed + + local minArgs + local maxArgs + + # All arguments of the function to check are passed as array in ${3} (the function call waits for $@) + # If any of the arguments contains spaces, bash things there are two aguments + # In order to avoid this, we need to iterate over ${3} and count + + callerName="${FUNCNAME[1]}" + + local iterate=3 + local fetchArguments=true + local argList="" + local countedArguments + while [ $fetchArguments == true ]; do + cmd='argument=${'$iterate'}' + eval $cmd + if [ "$argument" == "" ]; then + fetchArguments=false + else + argList="$argList[Argument $((iterate-2)): $argument] " + iterate=$((iterate+1)) + fi + done + + countedArguments=$((iterate-3)) + + if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then + minArgs=$numberOfArguments + maxArgs=$numberOfArguments + else + IFS='-' read minArgs maxArgs <<< "$numberOfArguments" + fi + + Logger "Entering function [$callerName]." "PARANOIA_DEBUG" + + if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then + Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR" + Logger "$callerName arguments: $argList" "ERROR" + else + if [ ! -z "$argList" ]; then + Logger "$callerName arguments: $argList" "PARANOIA_DEBUG" + fi + fi + fi +} + +#__END_WITH_PARANOIA_DEBUG function GetRemoteOS { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG if [ "$REMOTE_OPERATION" != true ]; then return 0 @@ -1631,6 +1763,7 @@ ENDSSH function RunLocalCommand { local command="${1}" # Command to run local hardMaxTime="${2}" # Max time to wait for command to compleet + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG if [ $_DRYRUN == true ]; then Logger "Dryrun: Local command [$command] not run." "NOTICE" @@ -1658,6 +1791,7 @@ function RunLocalCommand { function RunRemoteCommand { local command="${1}" # Command to run local hardMaxTime="${2}" # Max time to wait for command to compleet + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG if [ "$REMOTE_OPERATION" != true ]; then @@ -1691,6 +1825,7 @@ function RunRemoteCommand { } function RunBeforeHook { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local pids @@ -1717,6 +1852,7 @@ function RunBeforeHook { } function RunAfterHook { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local pids @@ -1736,6 +1872,7 @@ function RunAfterHook { function TimeCheck { # Checks if more than deltatime seconds have passed since last check, which is stored in timefile + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local timefile="${1}" local deltatime="${2}" @@ -1770,9 +1907,11 @@ function Ping { function CheckConnectivityRemoteHost { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local retval + if [ "$_PARANOIA_DEBUG" != true ]; then # Do not loose time in paranoia debug #__WITH_PARANOIA_DEBUG if [ "$REMOTE_HOST_PING" != false ] && [ "$REMOTE_OPERATION" != false ]; then eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" & @@ -1783,6 +1922,7 @@ function CheckConnectivityRemoteHost { return $retval fi fi + fi #__WITH_PARANOIA_DEBUG } function CheckConnectivity3rdPartyHosts { @@ -1790,6 +1930,7 @@ function CheckConnectivity3rdPartyHosts { # third_party_hosts_ips=('1.1.1.1' '8.8.8.8' 'kernel.org' 'google.com') # CheckConnectivity3rdPartyHosts $third_party_hosts_ips + __CheckArguments 0-1 $# "$@" #__WITH_PARANOIA_DEBUG local remote_3rd_party_hosts="${1}" # Optional list of hosts to check @@ -1797,6 +1938,7 @@ function CheckConnectivity3rdPartyHosts { local retval local i + if [ "$_PARANOIA_DEBUG" != true ]; then # Do not loose time in paranoia debug #__WITH_PARANOIA_DEBUG if [ "$remote_3rd_party_hosts" == "" ]; then remote_3rd_party_hosts="$REMOTE_3RD_PARTY_HOSTS" @@ -1823,11 +1965,13 @@ function CheckConnectivity3rdPartyHosts { return 0 fi fi + fi #__WITH_PARANOIA_DEBUG } function RsyncPatternsAdd { local patternType="${1}" # exclude or include local pattern="${2}" + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local rest @@ -1857,6 +2001,7 @@ function RsyncPatternsAdd { function RsyncPatternsFromAdd { local patternType="${1}" local patternFrom="${2}" + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG ## Check if the exclude list has a full path, and if not, add the config file path if there is one if [ "$(basename "$patternFrom")" == "$patternFrom" ]; then @@ -1869,6 +2014,7 @@ function RsyncPatternsFromAdd { } function RsyncPatterns { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG if [ "$RSYNC_PATTERN_FIRST" == "exclude" ]; then if [ "$RSYNC_EXCLUDE_PATTERN" != "" ]; then @@ -1906,6 +2052,7 @@ function RsyncPatterns { } function PreInit { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local compressionString @@ -1960,6 +2107,7 @@ function PreInit { } function PostInit { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG # Define remote commands if [ -f "$SSH_RSA_PRIVATE_KEY" ]; then @@ -2026,6 +2174,7 @@ function SetCompression { } function InitLocalOSDependingSettings { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG ## If running under Msys, some commands do not run the same way ## Using mingw version of find instead of windows one @@ -2086,6 +2235,7 @@ function InitLocalOSDependingSettings { # Gets executed regardless of the need of remote connections. It is just that this code needs to get executed after we know if there is a remote os, and if yes, which one function InitRemoteOSDependingSettings { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG if [ "$REMOTE_OS" == "msys" ] || [ "$REMOTE_OS" == "Cygwin" ]; then REMOTE_FIND_CMD="$(dirname $BASH)/find" @@ -2112,7 +2262,7 @@ function InitRemoteOSDependingSettings { #fi # NPF-MOD: Strangely enough, also happens on RHEL7 rsync 3.1.1 # Let's resolve this easier - RSYNC_OLD_ARGS=1 + export RSYNC_OLD_ARGS=1 if [ "$_DRYRUN" == true ]; then RSYNC_DRY_ARG="-n" @@ -2575,6 +2725,7 @@ function TrapQuit { } function CheckEnvironment { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG if [ "$REMOTE_OPERATION" == true ]; then if ! type ssh > /dev/null 2>&1 ; then @@ -2624,6 +2775,7 @@ function CheckCurrentConfig { local booleans local num_vars + __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG # Full check is for initiator driven runs if [ $fullCheck == true ]; then @@ -2665,6 +2817,7 @@ function UpdateBooleans { # Gets checked in quicksync and config file mode function CheckCurrentConfigAll { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local tmp @@ -2712,6 +2865,7 @@ function _CheckReplicasLocal { local replicaType="${2}" local stateDir="${3}" + __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG local retval local diskSpace @@ -2788,6 +2942,7 @@ function _CheckReplicasRemote { local replicaType="${2}" local stateDir="${3}" + __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG local retval local cmd @@ -2819,6 +2974,10 @@ if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" RUN_DIR="$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" fi +## allow function call checks #__WITH_PARANOIA_DEBUG +if [ "$_PARANOIA_DEBUG" == true ];then #__WITH_PARANOIA_DEBUG + _DEBUG=true #__WITH_PARANOIA_DEBUG +fi #__WITH_PARANOIA_DEBUG ## allow debugging from command line with _DEBUG=true if [ ! "$_DEBUG" == true ]; then @@ -2975,6 +3134,11 @@ function RemoteLogger { _Logger "" "$prefix$value" return fi + elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG + if [ "$_PARANOIA_DEBUG" == true ]; then #__WITH_PARANOIA_DEBUG + _Logger "" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG + return #__WITH_PARANOIA_DEBUG + fi #__WITH_PARANOIA_DEBUG else _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "" "Value was: $prefix$value" true @@ -3089,6 +3253,7 @@ ENDSSH } function CheckReplicas { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local initiatorPid local targetPid @@ -3137,6 +3302,7 @@ function _HandleLocksLocal { local replicaType="${3}" local overwrite="${4:-false}" + __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG local retval local lockfileContent @@ -3205,6 +3371,7 @@ function _HandleLocksRemote { local replicaType="${3}" local overwrite="${4:-false}" + __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG local retval local initiatorRunningPids @@ -3248,6 +3415,10 @@ if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" RUN_DIR="$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" fi +## allow function call checks #__WITH_PARANOIA_DEBUG +if [ "$_PARANOIA_DEBUG" == true ];then #__WITH_PARANOIA_DEBUG + _DEBUG=true #__WITH_PARANOIA_DEBUG +fi #__WITH_PARANOIA_DEBUG ## allow debugging from command line with _DEBUG=true if [ ! "$_DEBUG" == true ]; then @@ -3392,6 +3563,11 @@ function RemoteLogger { _Logger "" "$prefix$value" return fi + elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG + if [ "$_PARANOIA_DEBUG" == true ]; then #__WITH_PARANOIA_DEBUG + _Logger "" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG + return #__WITH_PARANOIA_DEBUG + fi #__WITH_PARANOIA_DEBUG else _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "" "Value was: $prefix$value" true @@ -3505,6 +3681,7 @@ ENDSSH } function HandleLocks { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local retval local initiatorPid @@ -3565,6 +3742,7 @@ function _UnlockReplicasLocal { local lockfile="${1}" local replicaType="${2}" + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local retval @@ -3583,6 +3761,7 @@ function _UnlockReplicasRemote { local lockfile="${1}" local replicaType="${2}" + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local retval local cmd @@ -3609,6 +3788,7 @@ ENDSSH } function UnlockReplicas { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local initiatorPid=0 local targetPid=0 @@ -3656,6 +3836,7 @@ function treeList { local replicaType="${2}" # replica type: initiator, target local treeFilename="${3}" # filename to output tree (will be prefixed with $replicaType) + __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG local retval local rsyncCmd @@ -3707,6 +3888,7 @@ function treeList { function deleteList { local replicaType="${1}" # replica type: initiator, target + __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG local retval local subretval @@ -3780,6 +3962,7 @@ function _getFileCtimeMtimeLocal { local fileList="${3}" # Contains list of files to get time attrs local timestampFile="${4}" # Where to store the timestamp file + __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG echo -n "" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" @@ -3811,6 +3994,7 @@ function _getFileCtimeMtimeRemote { local fileList="${3}" local timestampFile="${4}" + __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG local retval local cmd @@ -3852,6 +4036,10 @@ if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" RUN_DIR="$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" fi +## allow function call checks #__WITH_PARANOIA_DEBUG +if [ "$_PARANOIA_DEBUG" == true ];then #__WITH_PARANOIA_DEBUG + _DEBUG=true #__WITH_PARANOIA_DEBUG +fi #__WITH_PARANOIA_DEBUG ## allow debugging from command line with _DEBUG=true if [ ! "$_DEBUG" == true ]; then @@ -4008,6 +4196,11 @@ function RemoteLogger { _Logger "" "$prefix$value" return fi + elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG + if [ "$_PARANOIA_DEBUG" == true ]; then #__WITH_PARANOIA_DEBUG + _Logger "" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG + return #__WITH_PARANOIA_DEBUG + fi #__WITH_PARANOIA_DEBUG else _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "" "Value was: $prefix$value" true @@ -4090,6 +4283,7 @@ function timestampList { local fileList="${3}" # List of files to get timestamps for local timestampFilename="${4}" # filename to output timestamp list (will be prefixed with $replicaType) + __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG local retval local rsyncCmd @@ -4111,6 +4305,7 @@ function conflictList { local timestampCurrentFilename="${1}" # filename of current timestamp list (will be prefixed with $replicaType) local timestampAfterFilename="${2}" # filename of previous timestamp list (will be prefixed with $replicaType) + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local retval @@ -4179,6 +4374,7 @@ function syncAttrs { local initiatorReplica="${1}" local targetReplica="${2}" + __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local initiatorPid local targetPid @@ -4312,6 +4508,7 @@ function syncUpdate { local sourceReplica="${1}" # Contains replica type of source: initiator, target local destinationReplica="${2}" # Contains replica type of destination: initiator, target local remoteDelete="${3:-false}" # Use rsnyc to delete remote files if not existent in source + __CheckArguments 2-3 $# "$@" #__WITH_PARANOIA_DEBUG local rsyncCmd local retval @@ -4381,6 +4578,7 @@ function _deleteLocal { local replicaType="${1}" # Replica type local replicaDir="${2}" # Full path to replica local deletionDir="${3}" # deletion dir in format .[workdir]/deleted + __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG local retval=0 local parentdir @@ -4471,6 +4669,7 @@ function _deleteRemote { local replicaType="${1}" # Replica type local replicaDir="${2}" # Full path to replica local deletionDir="${3}" # deletion dir in format .[workdir]/deleted + __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG local retval local rsyncCmd @@ -4536,6 +4735,10 @@ if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" RUN_DIR="$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" fi +## allow function call checks #__WITH_PARANOIA_DEBUG +if [ "$_PARANOIA_DEBUG" == true ];then #__WITH_PARANOIA_DEBUG + _DEBUG=true #__WITH_PARANOIA_DEBUG +fi #__WITH_PARANOIA_DEBUG ## allow debugging from command line with _DEBUG=true if [ ! "$_DEBUG" == true ]; then @@ -4643,6 +4846,11 @@ function RemoteLogger { _Logger "" "$prefix$value" return fi + elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG + if [ "$_PARANOIA_DEBUG" == true ]; then #__WITH_PARANOIA_DEBUG + _Logger "" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG + return #__WITH_PARANOIA_DEBUG + fi #__WITH_PARANOIA_DEBUG else _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "" "Value was: $prefix$value" true @@ -4793,6 +5001,7 @@ ENDSSH # delete_Propagation(replica type) function deletionPropagation { local replicaType="${1}" # Contains replica type: initiator, target where to delete + __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG local retval local replicaDir @@ -4836,6 +5045,7 @@ function deletionPropagation { } function Initialize { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG Logger "Initializing initiator and target file lists." "NOTICE" @@ -4898,6 +5108,7 @@ function Initialize { ###### Step 8a & 8b: Create after run ctime & mtime file list of replicas function Sync { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local resumeCount local resumeInitiator @@ -5388,6 +5599,7 @@ function _SoftDeleteLocal { local changeTime="${3}" # Delete files older than changeTime days local deletionType="${4}" # Trivial deletion type string + __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG local retval @@ -5440,6 +5652,7 @@ function _SoftDeleteRemote { local changeTime="${3}" # Delete files older than changeTime days local deletionType="${4}" # Trivial deletion type string + __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG local retval @@ -5480,6 +5693,10 @@ if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" RUN_DIR="$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" fi +## allow function call checks #__WITH_PARANOIA_DEBUG +if [ "$_PARANOIA_DEBUG" == true ];then #__WITH_PARANOIA_DEBUG + _DEBUG=true #__WITH_PARANOIA_DEBUG +fi #__WITH_PARANOIA_DEBUG ## allow debugging from command line with _DEBUG=true if [ ! "$_DEBUG" == true ]; then @@ -5636,6 +5853,11 @@ function RemoteLogger { _Logger "" "$prefix$value" return fi + elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG + if [ "$_PARANOIA_DEBUG" == true ]; then #__WITH_PARANOIA_DEBUG + _Logger "" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG + return #__WITH_PARANOIA_DEBUG + fi #__WITH_PARANOIA_DEBUG else _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "" "Value was: $prefix$value" true @@ -5717,6 +5939,7 @@ ENDSSH } function SoftDelete { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local initiatorPid local targetPid @@ -5759,6 +5982,7 @@ function SoftDelete { } function _TriggerInitiatorRunLocal { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local PUSH_FILE @@ -5781,6 +6005,7 @@ function _TriggerInitiatorRunLocal { } function _TriggerInitiatorRunRemote { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG $SSH_CMD env _REMOTE_TOKEN="$_REMOTE_TOKEN" \ env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ @@ -5805,6 +6030,10 @@ if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" RUN_DIR="$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" fi +## allow function call checks #__WITH_PARANOIA_DEBUG +if [ "$_PARANOIA_DEBUG" == true ];then #__WITH_PARANOIA_DEBUG + _DEBUG=true #__WITH_PARANOIA_DEBUG +fi #__WITH_PARANOIA_DEBUG ## allow debugging from command line with _DEBUG=true if [ ! "$_DEBUG" == true ]; then @@ -5912,6 +6141,11 @@ function RemoteLogger { _Logger "" "$prefix$value" return fi + elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG + if [ "$_PARANOIA_DEBUG" == true ]; then #__WITH_PARANOIA_DEBUG + _Logger "" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG + return #__WITH_PARANOIA_DEBUG + fi #__WITH_PARANOIA_DEBUG else _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "" "Value was: $prefix$value" true @@ -5961,6 +6195,7 @@ ENDSSH } function TriggerInitiatorRun { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG if [ "$REMOTE_OPERATION" != false ]; then _TriggerInitiatorRunRemote @@ -5974,6 +6209,7 @@ function _SummaryFromRsyncFile { local summaryFile="${2}" local direction="${3}" + __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG if [ -f "$summaryFile" ]; then while read -r file; do @@ -6010,6 +6246,7 @@ function _SummaryFromDeleteFile { local summaryFile="${2}" local direction="${3}" + __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG if [ -f "$summaryFile" ]; then while read -r file; do @@ -6024,6 +6261,7 @@ function _SummaryFromDeleteFile { } function Summary { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG ( _LOGGER_PREFIX="" @@ -6053,6 +6291,7 @@ function Summary { } function LogConflicts { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local subject local body @@ -6088,6 +6327,7 @@ function LogConflicts { } function Init { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG # Set error exit code if a piped command fails set -o pipefail @@ -6331,12 +6571,14 @@ function Init { } function Main { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG HandleLocks Sync } function Usage { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG if [ "$IS_STABLE" != true ]; then echo -e "\e[93mThis is an unstable dev build. Please use with caution.\e[0m" @@ -6394,6 +6636,7 @@ function Usage { function SyncOnChanges { local isTargetHelper="${1:-false}" # Is this service supposed to be run as target helper ? + __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG local watchDirectory local watchCmd From ea1b3b379ef327163396ff6e388c43b69eb7890a Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Wed, 14 Jun 2023 14:23:16 +0200 Subject: [PATCH 32/38] Fix misc option names --- sync.conf.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync.conf.example b/sync.conf.example index 439abc0..aa42c59 100644 --- a/sync.conf.example +++ b/sync.conf.example @@ -100,7 +100,7 @@ REMOTE_3RD_PARTY_HOSTS="www.kernel.org www.google.com" [MISC_OPTIONS] ## Optional arguments passed to rsync executable. The following are already managed by the program and shoul never be passed here -## -r -l -p -t -g -o -D -E - u- i- n --executability -A -X -L -K -H -8 -zz –skip-compress –checksum –bwlimit –partial –partial-dir –no-whole-file –whole-file –backup –backup-dir –suffix +## -r -l -p -t -g -o -D -E - u- i- n --executability -A -X -L -K -H -8 --zz -–skip-compress -–checksum –-bwlimit –-partial –-partial-dir –-no-whole-file –-whole-file –-backup –-backup-dir –-suffix ## --exclude --exclude-from --include --include-from --list-only --stats ## When dealing with different filesystems for sync, or using SMB mountpoints, try adding --modify-window=2 --omit-dir-times as optional arguments. RSYNC_OPTIONAL_ARGS="" From 4054f3ca7560be70eb19d75de2b516a910d5e33b Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Wed, 14 Jun 2023 14:30:28 +0200 Subject: [PATCH 33/38] Make sure we update remote.conf rsa key path --- dev/tests/run_tests.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/tests/run_tests.sh b/dev/tests/run_tests.sh index e5d8b03..bb1da19 100755 --- a/dev/tests/run_tests.sh +++ b/dev/tests/run_tests.sh @@ -123,8 +123,9 @@ function SetupSSH { ssh-keyscan -H localhost >> "${homedir}/.ssh/known_hosts" fi - # Update remote conf files with SSH port + # Update remote conf files with SSH port and file id location sed -i.tmp 's#ssh://.*@localhost:[0-9]*/${HOME}/osync-tests/target#ssh://'$REMOTE_USER'@localhost:'$SSH_PORT'/'${homedir}'/osync-tests/target#' "$CONF_DIR/$REMOTE_CONF" + sed -i.tmp2 's#SSH_RSA_PRIVATE_KEY="${HOME}/.ssh/id_rsa_local_osync_tests"#SSH_RSA_PRIVATE_KEY="'${homedir}'/.ssh/id_rsa_local_osync_tests"#' "$CONF_DIR/$REMOTE_CONF" echo "ls -alh ${homedir}/.ssh" ls -alh "${homedir}/.ssh" From cef25b646f4570856e3b0174404e9b8dccfafb71 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Wed, 14 Jun 2023 19:05:58 +0200 Subject: [PATCH 34/38] Prepare v1.3 release --- README.md | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 624ac84..06af294 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,12 @@ -# osync [![Build Status](https://travis-ci.org/deajan/osync.svg?branch=master)](https://travis-ci.org/deajan/osync) [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) [![GitHub Release](https://img.shields.io/github/release/deajan/osync.svg?label=Latest)](https://github.com/deajan/osync/releases/latest) [![Percentage of issues still open](http://isitmaintained.com/badge/open/deajan/osync.svg)](http://isitmaintained.com/project/deajan/osync "Percentage of issues still open") [![Codacy Badge](https://api.codacy.com/project/badge/Grade/651acb2fd64642eb91078ba523b7f887)](https://www.codacy.com/app/ozy/osync?utm_source=github.com&utm_medium=referral&utm_content=deajan/osync&utm_campaign=Badge_Grade) +# osync +[![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) +[![GitHub Release](https://img.shields.io/github/release/deajan/osync.svg?label=Latest)](https://github.com/deajan/osync/releases/latest) +[![Percentage of issues still open](http://isitmaintained.com/badge/open/deajan/osync.svg)](http://isitmaintained.com/project/deajan/osync "Percentage of issues still open") +[![Codacy Badge](https://api.codacy.com/project/badge/Grade/651acb2fd64642eb91078ba523b7f887)](https://www.codacy.com/app/ozy/osync?utm_source=github.com&utm_medium=referral&utm_content=deajan/osync&utm_campaign=Badge_Grade) +[![linux tests](https://github.com/deajan/osync/actions/workflows/linux.yml/badge.svg)](https://github.com/deajan/osync/actions/workflows/linux.yml/badge.svg) +[![windows tests](https://github.com/deajan/osync/actions/workflows/windows.yml/badge.svg)](https://github.com/deajan/osync/actions/workflows/windows.yml/badge.svg) +[![macos tests](https://github.com/deajan/osync/actions/workflows/macos.yml/badge.svg)](https://github.com/deajan/osync/actions/workflows/macos.yml/badge.svg) + A two way filesync script running on bash Linux, BSD, Android, MacOSX, Cygwin, MSYS2, Win10 bash and virtually any system supporting bash). File synchronization is bidirectional, and can be run manually, as scheduled task, or triggered on file changes in daemon mode. @@ -30,8 +38,8 @@ osync uses pidlocks to prevent multiple concurrent sync processes on/to the same You may launch concurrent sync processes on the same system but as long as the replicas to synchronize are different. Multiple osync tasks may be launched sequentially by osync osync-batch tool. -Currently, it has been tested on CentOS 5.x, 6.x, 7.x, Fedora 22-25, Debian 6-8, Linux Mint 14-18, Ubuntu 12.04-12.10, FreeBSD 8.3-11, Mac OS X, QTS 4.5.1(x86) and pfSense 2.3.x. -Microsoft Windows is supported via MSYS or Cygwin and now via Windows 10 bash. +Currently, it has been tested on CentOS 5.x, 6.x, 7.x, AlmaLinux 9, Fedora 22-25, Debian 6-11, Linux Mint 14-18, Ubuntu 12.04-22.04, FreeBSD 8.3-11, Mac OS X, QTS 4.5.1(x86) and pfSense 2.3.x. +Microsoft Windows is supported via MSYS, Cygwin and or via WSL. Android support works via Termux. Some users also have successfully used osync on Gentoo and created an openRC init scriptt for it. @@ -40,12 +48,6 @@ Some users also have successfully used osync on Gentoo and created an openRC ini osync has been designed to not delete any data, but rather make backups of conflictual files or soft deletes. Nevertheless, you should always have a neat backup of your data before trying a new sync tool. -You may get osync on github (stable or latest dev snapshot) or on the author's site (stable version) -Getting osync via author's site on - - $ wget http://www.netpower.fr/projects/osync/osync.v1.2.tar.gz - $ tar xvf osync.v1.2.tar.gz - Getting osync via github (remove the -b "stable" if you want latest dev snapshot) $ git clone -b "stable" https://github.com/deajan/osync @@ -74,11 +76,11 @@ Archlinux packages are available at Since osync v1.1 the config file format has changed in semantics and adds new config options. Also, master is now called initiator and slave is now called target. -osync v1.2 also added multiple new configuration options. +osync v1.3 also added multiple new configuration options. -You can upgrade all v1.0x-v1.2-dev config files by running the upgrade script +You can upgrade all v1.0x-v1.3-dev config files by running the upgrade script - $ ./upgrade-v1.0x-v1.2x.sh /etc/osync/your-config-file.conf + $ ./upgrade-v1.0x-v1.3x.sh /etc/osync/your-config-file.conf The script will backup your config file, update it's content and try to connect to initiator and target replicas to update the state dir. From e47c1d7f58e495aa6ed055c8dba594955d8e13b8 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Wed, 14 Jun 2023 19:09:05 +0200 Subject: [PATCH 35/38] Bump version --- dev/n_osync.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 dev/n_osync.sh diff --git a/dev/n_osync.sh b/dev/n_osync.sh old mode 100644 new mode 100755 index d25b3cb..ef68e14 --- a/dev/n_osync.sh +++ b/dev/n_osync.sh @@ -4,10 +4,10 @@ #Check dryruns with nosuffix mode for timestampList PROGRAM="osync" # Rsync based two way sync engine with fault tolerance -AUTHOR="(C) 2013-2022 by Orsiris de Jong" +AUTHOR="(C) 2013-2023 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" -PROGRAM_VERSION=1.3.0-rc4-dev -PROGRAM_BUILD=2023061101 +PROGRAM_VERSION=1.3.0 +PROGRAM_BUILD=2023061401 IS_STABLE=true CONFIG_FILE_REVISION_REQUIRED=1.3.0 From 98a4d6bb0c7b0af4e470f9d4dbaf897a78188d11 Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Wed, 14 Jun 2023 19:09:33 +0200 Subject: [PATCH 36/38] Rebuilt targets --- dev/debug_osync.sh | 6 +- osync.sh | 249 +-------------------------------------------- 2 files changed, 6 insertions(+), 249 deletions(-) diff --git a/dev/debug_osync.sh b/dev/debug_osync.sh index 11424c1..9d37b56 100755 --- a/dev/debug_osync.sh +++ b/dev/debug_osync.sh @@ -4,10 +4,10 @@ #Check dryruns with nosuffix mode for timestampList PROGRAM="osync" # Rsync based two way sync engine with fault tolerance -AUTHOR="(C) 2013-2022 by Orsiris de Jong" +AUTHOR="(C) 2013-2023 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" -PROGRAM_VERSION=1.3.0-rc4-dev -PROGRAM_BUILD=2023061101 +PROGRAM_VERSION=1.3.0 +PROGRAM_BUILD=2023061401 IS_STABLE=true CONFIG_FILE_REVISION_REQUIRED=1.3.0 diff --git a/osync.sh b/osync.sh index 11424c1..0d486f5 100755 --- a/osync.sh +++ b/osync.sh @@ -4,43 +4,14 @@ #Check dryruns with nosuffix mode for timestampList PROGRAM="osync" # Rsync based two way sync engine with fault tolerance -AUTHOR="(C) 2013-2022 by Orsiris de Jong" +AUTHOR="(C) 2013-2023 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" -PROGRAM_VERSION=1.3.0-rc4-dev -PROGRAM_BUILD=2023061101 +PROGRAM_VERSION=1.3.0 +PROGRAM_BUILD=2023061401 IS_STABLE=true CONFIG_FILE_REVISION_REQUIRED=1.3.0 -##### Execution order #__WITH_PARANOIA_DEBUG -##### Function Name Is parallel #__WITH_PARANOIA_DEBUG -# GetLocalOS no #__WITH_PARANOIA_DEBUG -# InitLocalOSDependingSettings no #__WITH_PARANOIA_DEBUG -# CheckEnvironment no #__WITH_PARANOIA_DEBUG -# PreInit no #__WITH_PARANOIA_DEBUG -# Init no #__WITH_PARANOIA_DEBUG -# PostInit no #__WITH_PARANOIA_DEBUG -# GetRemoteOS no #__WITH_PARANOIA_DEBUG -# InitRemoteOSDependingSettings no #__WITH_PARANOIA_DEBUG -# CheckReplicas yes #__WITH_PARANOIA_DEBUG -# RunBeforeHook yes #__WITH_PARANOIA_DEBUG -# Main no #__WITH_PARANOIA_DEBUG -# HandleLocks yes #__WITH_PARANOIA_DEBUG -# Sync no #__WITH_PARANOIA_DEBUG -# treeList yes #__WITH_PARANOIA_DEBUG -# deleteList yes #__WITH_PARANOIA_DEBUG -# timestampList yes #__WITH_PARANOIA_DEBUG -# conflictList no #__WITH_PARANOIA_DEBUG -# syncAttrs no #__WITH_PARANOIA_DEBUG -# syncUpdate no #__WITH_PARANOIA_DEBUG -# syncUpdate no #__WITH_PARANOIA_DEBUG -# deletionPropagation yes #__WITH_PARANOIA_DEBUG -# treeList yes #__WITH_PARANOIA_DEBUG -# timestampList yes #__WITH_PARANOIA_DEBUG -# SoftDelete yes #__WITH_PARANOIA_DEBUG -# RunAfterHook yes #__WITH_PARANOIA_DEBUG -# UnlockReplicas yes #__WITH_PARANOIA_DEBUG -# CleanUp no #__WITH_PARANOIA_DEBUG _OFUNCTIONS_VERSION=2.5.1 _OFUNCTIONS_BUILD=2023061401 @@ -76,10 +47,6 @@ fi ERROR_ALERT=false WARN_ALERT=false -## allow function call checks #__WITH_PARANOIA_DEBUG -if [ "$_PARANOIA_DEBUG" == true ];then #__WITH_PARANOIA_DEBUG - _DEBUG=true #__WITH_PARANOIA_DEBUG -fi #__WITH_PARANOIA_DEBUG ## allow debugging from command line with _DEBUG=true if [ ! "$_DEBUG" == true ]; then @@ -244,11 +211,6 @@ function RemoteLogger { _Logger "" "$prefix$value" return fi - elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG - if [ "$_PARANOIA_DEBUG" == true ]; then #__WITH_PARANOIA_DEBUG - _Logger "" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG - return #__WITH_PARANOIA_DEBUG - fi #__WITH_PARANOIA_DEBUG else _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "" "Value was: $prefix$value" true @@ -323,11 +285,6 @@ function Logger { _Logger "$prefix$value" "$prefix$value" return fi - elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG - if [ "$_PARANOIA_DEBUG" == true ]; then #__WITH_PARANOIA_DEBUG - _Logger "$prefix$value" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG - return #__WITH_PARANOIA_DEBUG - fi #__WITH_PARANOIA_DEBUG else _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "Value was: $prefix$value" "Value was: $prefix$value" true @@ -372,7 +329,6 @@ function KillChilds { children="${children/$pid/}" fi for child in $children; do - Logger "Launching KillChilds \"$child\" true" "DEBUG" #__WITH_PARANOIA_DEBUG KillChilds "$child" true done fi @@ -407,7 +363,6 @@ function KillAllChilds { local pids="${1}" # List of parent pids to kill separated by semi-colon local self="${2:-false}" # Should parent be killed too ? - __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG local errorcount=0 @@ -464,7 +419,6 @@ function SendAlert { local runAlert="${1:-false}" # Specifies if current message is sent while running or at the end of a run local attachment="${2:-true}" # Should we send the log file as attachment - __CheckArguments 0-2 $# "$@" #__WITH_PARANOIA_DEBUG local attachmentFile local subject @@ -546,7 +500,6 @@ function SendEmail { local smtpUser="${9}" local smtpPassword="${10}" - __CheckArguments 3-10 $# "$@" #__WITH_PARANOIA_DEBUG local mail_no_attachment= local attachment_command= @@ -727,7 +680,6 @@ function LoadConfigFile { local configFile="${1}" local revisionRequired="${2}" - __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local revisionPresent @@ -757,22 +709,6 @@ function LoadConfigFile { } # Quick and dirty performance logger only used for debugging -function _PerfProfiler { #__WITH_PARANOIA_DEBUG - local perfString #__WITH_PARANOIA_DEBUG - local i #__WITH_PARANOIA_DEBUG - #__WITH_PARANOIA_DEBUG - perfString=$(ps -p $$ -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan) #__WITH_PARANOIA_DEBUG - #__WITH_PARANOIA_DEBUG - for i in $(pgrep -P $$); do #__WITH_PARANOIA_DEBUG - perfString="$perfString\n"$(ps -p $i -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan | tail -1) #__WITH_PARANOIA_DEBUG - done #__WITH_PARANOIA_DEBUG - #__WITH_PARANOIA_DEBUG - if type iostat > /dev/null 2>&1; then #__WITH_PARANOIA_DEBUG - perfString="$perfString\n"$(iostat) #__WITH_PARANOIA_DEBUG - fi #__WITH_PARANOIA_DEBUG - #__WITH_PARANOIA_DEBUG - Logger "PerfProfiler:\n$perfString" "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG -} #__WITH_PARANOIA_DEBUG _OFUNCTIONS_SPINNER="|/-\\" function Spinner { @@ -875,11 +811,9 @@ function ExecTasks { local minTimeBetweenRetries="${17:-300}" # Time (in seconds) between postponed command retries local validExitCodes="${18:-0}" # Semi colon separated list of valid main command exit codes which will not trigger errors - __CheckArguments 1-18 $# "$@" #__WITH_PARANOIA_DEBUG local i - Logger "${FUNCNAME[0]} id [$id] called by [${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} < ${FUNCNAME[5]} < ${FUNCNAME[6]} ...]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG # Since ExecTasks takes up to 17 arguments, do a quick preflight check in DEBUG mode if [ "$_DEBUG" == true ]; then @@ -932,7 +866,6 @@ function ExecTasks { local newPidsArray # New array of currently running pids for next iteration local pidsTimeArray # Array containing execution begin time of pids local executeCommand # Boolean to check if currentCommand can be executed given a condition - local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG local functionMode local softAlert=false # Does a soft alert need to be triggered, if yes, send an alert once local failedPidsList # List containing failed pids with exit code separated by semicolons (eg : 2355:1;4534:2;2354:3) @@ -982,7 +915,6 @@ function ExecTasks { counter=$mainItemCount fi - Logger "Running ${FUNCNAME[0]} as [$functionMode] for [$mainItemCount] mainItems and [$auxItemCount] auxItems." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG # soft / hard execution time checks that needs to be a subfunction since it is called both from main loop and from parallelExec sub loop function _ExecTasksTimeCheck { @@ -1132,22 +1064,15 @@ function ExecTasks { Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG" fi fi - hasPids=true ##__WITH_PARANOIA_DEBUG fi done # hasPids can be false on last iteration in ParallelExec mode - if [ $hasPids == false ] && [ "$functionMode" = "WaitForTaskCompletion" ]; then ##__WITH_PARANOIA_DEBUG - Logger "No valable pids given." "ERROR" ##__WITH_PARANOIA_DEBUG - fi ##__WITH_PARANOIA_DEBUG pidsArray=("${newPidsArray[@]}") # Trivial wait time for bash to not eat up all CPU sleep $sleepTime - if [ "$_PERF_PROFILER" == true ]; then ##__WITH_PARANOIA_DEBUG - _PerfProfiler ##__WITH_PARANOIA_DEBUG - fi ##__WITH_PARANOIA_DEBUG } @@ -1285,7 +1210,6 @@ function ExecTasks { _ExecTasksPidsCheck done - Logger "${FUNCNAME[0]} ended for [$id] using [$mainItemCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG # Return exit code if only one process was monitored, else return number of errors # As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value @@ -1566,65 +1490,9 @@ function GetLocalOS { fi } -#__BEGIN_WITH_PARANOIA_DEBUG -function __CheckArguments { - # Checks the number of arguments of a function and raises an error if some are missing - - if [ "$_DEBUG" == true ]; then - local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments - local numberOfGivenArguments="${2}" # Number of arguments that have been passed - - local minArgs - local maxArgs - - # All arguments of the function to check are passed as array in ${3} (the function call waits for $@) - # If any of the arguments contains spaces, bash things there are two aguments - # In order to avoid this, we need to iterate over ${3} and count - - callerName="${FUNCNAME[1]}" - - local iterate=3 - local fetchArguments=true - local argList="" - local countedArguments - while [ $fetchArguments == true ]; do - cmd='argument=${'$iterate'}' - eval $cmd - if [ "$argument" == "" ]; then - fetchArguments=false - else - argList="$argList[Argument $((iterate-2)): $argument] " - iterate=$((iterate+1)) - fi - done - - countedArguments=$((iterate-3)) - - if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then - minArgs=$numberOfArguments - maxArgs=$numberOfArguments - else - IFS='-' read minArgs maxArgs <<< "$numberOfArguments" - fi - - Logger "Entering function [$callerName]." "PARANOIA_DEBUG" - - if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then - Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR" - Logger "$callerName arguments: $argList" "ERROR" - else - if [ ! -z "$argList" ]; then - Logger "$callerName arguments: $argList" "PARANOIA_DEBUG" - fi - fi - fi -} - -#__END_WITH_PARANOIA_DEBUG function GetRemoteOS { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG if [ "$REMOTE_OPERATION" != true ]; then return 0 @@ -1763,7 +1631,6 @@ ENDSSH function RunLocalCommand { local command="${1}" # Command to run local hardMaxTime="${2}" # Max time to wait for command to compleet - __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG if [ $_DRYRUN == true ]; then Logger "Dryrun: Local command [$command] not run." "NOTICE" @@ -1791,7 +1658,6 @@ function RunLocalCommand { function RunRemoteCommand { local command="${1}" # Command to run local hardMaxTime="${2}" # Max time to wait for command to compleet - __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG if [ "$REMOTE_OPERATION" != true ]; then @@ -1825,7 +1691,6 @@ function RunRemoteCommand { } function RunBeforeHook { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local pids @@ -1852,7 +1717,6 @@ function RunBeforeHook { } function RunAfterHook { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local pids @@ -1872,7 +1736,6 @@ function RunAfterHook { function TimeCheck { # Checks if more than deltatime seconds have passed since last check, which is stored in timefile - __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local timefile="${1}" local deltatime="${2}" @@ -1907,11 +1770,9 @@ function Ping { function CheckConnectivityRemoteHost { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local retval - if [ "$_PARANOIA_DEBUG" != true ]; then # Do not loose time in paranoia debug #__WITH_PARANOIA_DEBUG if [ "$REMOTE_HOST_PING" != false ] && [ "$REMOTE_OPERATION" != false ]; then eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" & @@ -1922,7 +1783,6 @@ function CheckConnectivityRemoteHost { return $retval fi fi - fi #__WITH_PARANOIA_DEBUG } function CheckConnectivity3rdPartyHosts { @@ -1930,7 +1790,6 @@ function CheckConnectivity3rdPartyHosts { # third_party_hosts_ips=('1.1.1.1' '8.8.8.8' 'kernel.org' 'google.com') # CheckConnectivity3rdPartyHosts $third_party_hosts_ips - __CheckArguments 0-1 $# "$@" #__WITH_PARANOIA_DEBUG local remote_3rd_party_hosts="${1}" # Optional list of hosts to check @@ -1938,7 +1797,6 @@ function CheckConnectivity3rdPartyHosts { local retval local i - if [ "$_PARANOIA_DEBUG" != true ]; then # Do not loose time in paranoia debug #__WITH_PARANOIA_DEBUG if [ "$remote_3rd_party_hosts" == "" ]; then remote_3rd_party_hosts="$REMOTE_3RD_PARTY_HOSTS" @@ -1965,13 +1823,11 @@ function CheckConnectivity3rdPartyHosts { return 0 fi fi - fi #__WITH_PARANOIA_DEBUG } function RsyncPatternsAdd { local patternType="${1}" # exclude or include local pattern="${2}" - __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local rest @@ -2001,7 +1857,6 @@ function RsyncPatternsAdd { function RsyncPatternsFromAdd { local patternType="${1}" local patternFrom="${2}" - __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG ## Check if the exclude list has a full path, and if not, add the config file path if there is one if [ "$(basename "$patternFrom")" == "$patternFrom" ]; then @@ -2014,7 +1869,6 @@ function RsyncPatternsFromAdd { } function RsyncPatterns { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG if [ "$RSYNC_PATTERN_FIRST" == "exclude" ]; then if [ "$RSYNC_EXCLUDE_PATTERN" != "" ]; then @@ -2052,7 +1906,6 @@ function RsyncPatterns { } function PreInit { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local compressionString @@ -2107,7 +1960,6 @@ function PreInit { } function PostInit { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG # Define remote commands if [ -f "$SSH_RSA_PRIVATE_KEY" ]; then @@ -2174,7 +2026,6 @@ function SetCompression { } function InitLocalOSDependingSettings { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG ## If running under Msys, some commands do not run the same way ## Using mingw version of find instead of windows one @@ -2235,7 +2086,6 @@ function InitLocalOSDependingSettings { # Gets executed regardless of the need of remote connections. It is just that this code needs to get executed after we know if there is a remote os, and if yes, which one function InitRemoteOSDependingSettings { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG if [ "$REMOTE_OS" == "msys" ] || [ "$REMOTE_OS" == "Cygwin" ]; then REMOTE_FIND_CMD="$(dirname $BASH)/find" @@ -2725,7 +2575,6 @@ function TrapQuit { } function CheckEnvironment { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG if [ "$REMOTE_OPERATION" == true ]; then if ! type ssh > /dev/null 2>&1 ; then @@ -2775,7 +2624,6 @@ function CheckCurrentConfig { local booleans local num_vars - __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG # Full check is for initiator driven runs if [ $fullCheck == true ]; then @@ -2817,7 +2665,6 @@ function UpdateBooleans { # Gets checked in quicksync and config file mode function CheckCurrentConfigAll { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local tmp @@ -2865,7 +2712,6 @@ function _CheckReplicasLocal { local replicaType="${2}" local stateDir="${3}" - __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG local retval local diskSpace @@ -2942,7 +2788,6 @@ function _CheckReplicasRemote { local replicaType="${2}" local stateDir="${3}" - __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG local retval local cmd @@ -2974,10 +2819,6 @@ if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" RUN_DIR="$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" fi -## allow function call checks #__WITH_PARANOIA_DEBUG -if [ "$_PARANOIA_DEBUG" == true ];then #__WITH_PARANOIA_DEBUG - _DEBUG=true #__WITH_PARANOIA_DEBUG -fi #__WITH_PARANOIA_DEBUG ## allow debugging from command line with _DEBUG=true if [ ! "$_DEBUG" == true ]; then @@ -3134,11 +2975,6 @@ function RemoteLogger { _Logger "" "$prefix$value" return fi - elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG - if [ "$_PARANOIA_DEBUG" == true ]; then #__WITH_PARANOIA_DEBUG - _Logger "" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG - return #__WITH_PARANOIA_DEBUG - fi #__WITH_PARANOIA_DEBUG else _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "" "Value was: $prefix$value" true @@ -3253,7 +3089,6 @@ ENDSSH } function CheckReplicas { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local initiatorPid local targetPid @@ -3302,7 +3137,6 @@ function _HandleLocksLocal { local replicaType="${3}" local overwrite="${4:-false}" - __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG local retval local lockfileContent @@ -3371,7 +3205,6 @@ function _HandleLocksRemote { local replicaType="${3}" local overwrite="${4:-false}" - __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG local retval local initiatorRunningPids @@ -3415,10 +3248,6 @@ if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" RUN_DIR="$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" fi -## allow function call checks #__WITH_PARANOIA_DEBUG -if [ "$_PARANOIA_DEBUG" == true ];then #__WITH_PARANOIA_DEBUG - _DEBUG=true #__WITH_PARANOIA_DEBUG -fi #__WITH_PARANOIA_DEBUG ## allow debugging from command line with _DEBUG=true if [ ! "$_DEBUG" == true ]; then @@ -3563,11 +3392,6 @@ function RemoteLogger { _Logger "" "$prefix$value" return fi - elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG - if [ "$_PARANOIA_DEBUG" == true ]; then #__WITH_PARANOIA_DEBUG - _Logger "" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG - return #__WITH_PARANOIA_DEBUG - fi #__WITH_PARANOIA_DEBUG else _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "" "Value was: $prefix$value" true @@ -3681,7 +3505,6 @@ ENDSSH } function HandleLocks { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local retval local initiatorPid @@ -3742,7 +3565,6 @@ function _UnlockReplicasLocal { local lockfile="${1}" local replicaType="${2}" - __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local retval @@ -3761,7 +3583,6 @@ function _UnlockReplicasRemote { local lockfile="${1}" local replicaType="${2}" - __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local retval local cmd @@ -3788,7 +3609,6 @@ ENDSSH } function UnlockReplicas { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local initiatorPid=0 local targetPid=0 @@ -3836,7 +3656,6 @@ function treeList { local replicaType="${2}" # replica type: initiator, target local treeFilename="${3}" # filename to output tree (will be prefixed with $replicaType) - __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG local retval local rsyncCmd @@ -3888,7 +3707,6 @@ function treeList { function deleteList { local replicaType="${1}" # replica type: initiator, target - __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG local retval local subretval @@ -3962,7 +3780,6 @@ function _getFileCtimeMtimeLocal { local fileList="${3}" # Contains list of files to get time attrs local timestampFile="${4}" # Where to store the timestamp file - __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG echo -n "" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP" @@ -3994,7 +3811,6 @@ function _getFileCtimeMtimeRemote { local fileList="${3}" local timestampFile="${4}" - __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG local retval local cmd @@ -4036,10 +3852,6 @@ if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" RUN_DIR="$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" fi -## allow function call checks #__WITH_PARANOIA_DEBUG -if [ "$_PARANOIA_DEBUG" == true ];then #__WITH_PARANOIA_DEBUG - _DEBUG=true #__WITH_PARANOIA_DEBUG -fi #__WITH_PARANOIA_DEBUG ## allow debugging from command line with _DEBUG=true if [ ! "$_DEBUG" == true ]; then @@ -4196,11 +4008,6 @@ function RemoteLogger { _Logger "" "$prefix$value" return fi - elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG - if [ "$_PARANOIA_DEBUG" == true ]; then #__WITH_PARANOIA_DEBUG - _Logger "" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG - return #__WITH_PARANOIA_DEBUG - fi #__WITH_PARANOIA_DEBUG else _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "" "Value was: $prefix$value" true @@ -4283,7 +4090,6 @@ function timestampList { local fileList="${3}" # List of files to get timestamps for local timestampFilename="${4}" # filename to output timestamp list (will be prefixed with $replicaType) - __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG local retval local rsyncCmd @@ -4305,7 +4111,6 @@ function conflictList { local timestampCurrentFilename="${1}" # filename of current timestamp list (will be prefixed with $replicaType) local timestampAfterFilename="${2}" # filename of previous timestamp list (will be prefixed with $replicaType) - __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local retval @@ -4374,7 +4179,6 @@ function syncAttrs { local initiatorReplica="${1}" local targetReplica="${2}" - __CheckArguments 2 $# "$@" #__WITH_PARANOIA_DEBUG local initiatorPid local targetPid @@ -4508,7 +4312,6 @@ function syncUpdate { local sourceReplica="${1}" # Contains replica type of source: initiator, target local destinationReplica="${2}" # Contains replica type of destination: initiator, target local remoteDelete="${3:-false}" # Use rsnyc to delete remote files if not existent in source - __CheckArguments 2-3 $# "$@" #__WITH_PARANOIA_DEBUG local rsyncCmd local retval @@ -4578,7 +4381,6 @@ function _deleteLocal { local replicaType="${1}" # Replica type local replicaDir="${2}" # Full path to replica local deletionDir="${3}" # deletion dir in format .[workdir]/deleted - __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG local retval=0 local parentdir @@ -4669,7 +4471,6 @@ function _deleteRemote { local replicaType="${1}" # Replica type local replicaDir="${2}" # Full path to replica local deletionDir="${3}" # deletion dir in format .[workdir]/deleted - __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG local retval local rsyncCmd @@ -4735,10 +4536,6 @@ if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" RUN_DIR="$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" fi -## allow function call checks #__WITH_PARANOIA_DEBUG -if [ "$_PARANOIA_DEBUG" == true ];then #__WITH_PARANOIA_DEBUG - _DEBUG=true #__WITH_PARANOIA_DEBUG -fi #__WITH_PARANOIA_DEBUG ## allow debugging from command line with _DEBUG=true if [ ! "$_DEBUG" == true ]; then @@ -4846,11 +4643,6 @@ function RemoteLogger { _Logger "" "$prefix$value" return fi - elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG - if [ "$_PARANOIA_DEBUG" == true ]; then #__WITH_PARANOIA_DEBUG - _Logger "" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG - return #__WITH_PARANOIA_DEBUG - fi #__WITH_PARANOIA_DEBUG else _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "" "Value was: $prefix$value" true @@ -5001,7 +4793,6 @@ ENDSSH # delete_Propagation(replica type) function deletionPropagation { local replicaType="${1}" # Contains replica type: initiator, target where to delete - __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG local retval local replicaDir @@ -5045,7 +4836,6 @@ function deletionPropagation { } function Initialize { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG Logger "Initializing initiator and target file lists." "NOTICE" @@ -5108,7 +4898,6 @@ function Initialize { ###### Step 8a & 8b: Create after run ctime & mtime file list of replicas function Sync { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local resumeCount local resumeInitiator @@ -5599,7 +5388,6 @@ function _SoftDeleteLocal { local changeTime="${3}" # Delete files older than changeTime days local deletionType="${4}" # Trivial deletion type string - __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG local retval @@ -5652,7 +5440,6 @@ function _SoftDeleteRemote { local changeTime="${3}" # Delete files older than changeTime days local deletionType="${4}" # Trivial deletion type string - __CheckArguments 4 $# "$@" #__WITH_PARANOIA_DEBUG local retval @@ -5693,10 +5480,6 @@ if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" RUN_DIR="$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" fi -## allow function call checks #__WITH_PARANOIA_DEBUG -if [ "$_PARANOIA_DEBUG" == true ];then #__WITH_PARANOIA_DEBUG - _DEBUG=true #__WITH_PARANOIA_DEBUG -fi #__WITH_PARANOIA_DEBUG ## allow debugging from command line with _DEBUG=true if [ ! "$_DEBUG" == true ]; then @@ -5853,11 +5636,6 @@ function RemoteLogger { _Logger "" "$prefix$value" return fi - elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG - if [ "$_PARANOIA_DEBUG" == true ]; then #__WITH_PARANOIA_DEBUG - _Logger "" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG - return #__WITH_PARANOIA_DEBUG - fi #__WITH_PARANOIA_DEBUG else _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "" "Value was: $prefix$value" true @@ -5939,7 +5717,6 @@ ENDSSH } function SoftDelete { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local initiatorPid local targetPid @@ -5982,7 +5759,6 @@ function SoftDelete { } function _TriggerInitiatorRunLocal { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local PUSH_FILE @@ -6005,7 +5781,6 @@ function _TriggerInitiatorRunLocal { } function _TriggerInitiatorRunRemote { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG $SSH_CMD env _REMOTE_TOKEN="$_REMOTE_TOKEN" \ env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \ @@ -6030,10 +5805,6 @@ if [ "$_REMOTE_EXECUTION" == true ]; then mkdir -p "$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" RUN_DIR="$RUN_DIR/$PROGRAM.remote.$SCRIPT_PID.$TSTAMP" fi -## allow function call checks #__WITH_PARANOIA_DEBUG -if [ "$_PARANOIA_DEBUG" == true ];then #__WITH_PARANOIA_DEBUG - _DEBUG=true #__WITH_PARANOIA_DEBUG -fi #__WITH_PARANOIA_DEBUG ## allow debugging from command line with _DEBUG=true if [ ! "$_DEBUG" == true ]; then @@ -6141,11 +5912,6 @@ function RemoteLogger { _Logger "" "$prefix$value" return fi - elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG - if [ "$_PARANOIA_DEBUG" == true ]; then #__WITH_PARANOIA_DEBUG - _Logger "" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG - return #__WITH_PARANOIA_DEBUG - fi #__WITH_PARANOIA_DEBUG else _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "" "Value was: $prefix$value" true @@ -6195,7 +5961,6 @@ ENDSSH } function TriggerInitiatorRun { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG if [ "$REMOTE_OPERATION" != false ]; then _TriggerInitiatorRunRemote @@ -6209,7 +5974,6 @@ function _SummaryFromRsyncFile { local summaryFile="${2}" local direction="${3}" - __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG if [ -f "$summaryFile" ]; then while read -r file; do @@ -6246,7 +6010,6 @@ function _SummaryFromDeleteFile { local summaryFile="${2}" local direction="${3}" - __CheckArguments 3 $# "$@" #__WITH_PARANOIA_DEBUG if [ -f "$summaryFile" ]; then while read -r file; do @@ -6261,7 +6024,6 @@ function _SummaryFromDeleteFile { } function Summary { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG ( _LOGGER_PREFIX="" @@ -6291,7 +6053,6 @@ function Summary { } function LogConflicts { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG local subject local body @@ -6327,7 +6088,6 @@ function LogConflicts { } function Init { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG # Set error exit code if a piped command fails set -o pipefail @@ -6571,14 +6331,12 @@ function Init { } function Main { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG HandleLocks Sync } function Usage { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG if [ "$IS_STABLE" != true ]; then echo -e "\e[93mThis is an unstable dev build. Please use with caution.\e[0m" @@ -6636,7 +6394,6 @@ function Usage { function SyncOnChanges { local isTargetHelper="${1:-false}" # Is this service supposed to be run as target helper ? - __CheckArguments 1 $# "$@" #__WITH_PARANOIA_DEBUG local watchDirectory local watchCmd From 63cbe3786e1e8837ea7ca8e9813aa5dad2bc535a Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Fri, 16 Jun 2023 00:20:54 +0200 Subject: [PATCH 37/38] Update changelog --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) mode change 100644 => 100755 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md old mode 100644 new mode 100755 index 0c9058a..cffa9a5 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,14 @@ ## RECENT CHANGES -### dd Mmm YYYY: To be done +### Current master - Make --log-conflicts non experimental (randomly fails) - ! new option FORCE_CONFLICT_PREVALANCE which will always use Initiator or Target, regardless of best time - ! target-helper: destination mails etc on target, also, no cmd after on configs -### dd Mmm YYYY: osync v1.3 release (for full changelog since v1.2 branch see all v1.3-beta/RC entries) +### 16 June 2023: osync v1.3 release (for full changelog since v1.2 branch see all v1.3-beta/RC entries) +- Fix for new RSYNC protocol - New options ALWAYS_SEND_MAILS to allow sending logs regardless of execution states ### 29 June 2020: osync v1.3-RC1 release From f6fd288cfaf9e83dd54f99d6ad253be128c8410b Mon Sep 17 00:00:00 2001 From: Noah Zalev Date: Tue, 4 Jul 2023 14:33:04 -0400 Subject: [PATCH 38/38] Typo fix --- dev/debug_osync.sh | 2 +- dev/ofunctions.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/debug_osync.sh b/dev/debug_osync.sh index 9d37b56..7702c4e 100755 --- a/dev/debug_osync.sh +++ b/dev/debug_osync.sh @@ -504,7 +504,7 @@ function SendAlert { fi if [ $runAlert == true ]; then - subject="Currently runing - $subject" + subject="Currently running - $subject" else subject="Finished run - $subject" fi diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index a1a3235..205803d 100755 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -512,7 +512,7 @@ function SendAlert { fi if [ $runAlert == true ]; then - subject="Currently runing - $subject" + subject="Currently running - $subject" else subject="Finished run - $subject" fi