Compare commits

...

52 Commits

Author SHA1 Message Date
deajan 58282b5fc0 Rebuilt targets 8 years ago
deajan 5f5ac1850d Fixed logging issue with multiple users 8 years ago
deajan 132ae6ec8b Make travis happy 8 years ago
deajan 36f85ce6db Added basic travis file 8 years ago
deajan 9e2f07153c Updated changelog 8 years ago
deajan 21e98e474c Rebuilt targets 8 years ago
deajan 2b14648e14 Fixed EscapeSpaces with bash >= 4.3 8 years ago
Orsiris de Jong 1f5a510b6e Fixed os detection (again after PR62) 8 years ago
Orsiris de Jong e4acfc029d Re introduced os detection fix that disappeared with PR2 8 years ago
Orsiris de Jong 44c341a7af Merge pull request #63 from deajan/stable
Merged PR60 / PR62
8 years ago
Orsiris de Jong 528d35c64e Merge pull request #62 from Shadowigor/stable-fakeroot
Add fakeroot support for Arch packaging in stable branch
8 years ago
Shadowigor 1ddd051033 Added fakeroot support for Arch packaging 8 years ago
deajan cd2b4b6154 Updated changelog 8 years ago
deajan e5813e6741 Fixed os detection 8 years ago
deajan eab6a05e4f Updated readme for new release 8 years ago
deajan 250d8a4b18 Renamed sync.conf to sync.conf.example 8 years ago
deajan ea0a1770db mend
Updated Changelog
8 years ago
deajan 0e37289fdc Updated changeelog 8 years ago
deajan 66c2735564 Rebuilt targets 8 years ago
deajan ef3153cbcd Fixed RunAfterHook double execution 8 years ago
deajan f70d8cda6a Rebuilt target 8 years ago
deajan 04585c720b Updated changelog 8 years ago
deajan 05975c3fa5 Rebuilt targets 8 years ago
deajan 3fb6bbb98c Added common installer template 8 years ago
deajan 4a60e64ca1 Fixed soft deletion when SUDO_EXEC enabled 8 years ago
Orsiris de Jong c906f001a4 Update README.md 8 years ago
deajan 665bc67af6 Updated install details 8 years ago
deajan 57b32d28fa Rebuilt targets 8 years ago
deajan 9bc8232071 Fixed double pattern declaration 8 years ago
deajan 3bd39e19c6 Updated changelog 8 years ago
deajan e8779625ce Rebuild targets 8 years ago
deajan 008b74200d Fixed soft deletion always enabled remotely 8 years ago
deajan fcfb26b19b Rebuild targets 8 years ago
deajan a76df153f0 Updated changelog 8 years ago
deajan 108bf7d316 Fixed attribute list for ctimes 8 years ago
deajan fc31fe4f5f Target rebuilds 8 years ago
deajan 8c10e2f8ef Fixed verbose warning for missing deletion dirs 8 years ago
deajan 8e3b5bf762 Revert "Fix for missing state subdirs"
This reverts commit 021b354929.
8 years ago
deajan 2abe5c36c1 Revert "missing state subdirs"
This reverts commit 2de50b413c.
8 years ago
deajan 2de50b413c Target rebuild 8 years ago
deajan 021b354929 Fix for missing state subdirs 8 years ago
deajan 0d7aa043a2 Rebuild targets 8 years ago
deajan 010e171f50 More remote deletion love 8 years ago
deajan 9b01a7533f Prevent remote deletion to fail on time mismatch between systems 8 years ago
deajan bdeb7fe907 Re-rebuild for v1.1.1 8 years ago
deajan 2d3e945817 Rebuilt osync for v1.1.1 8 years ago
deajan c234bdf107 More fixes for remote deletion from PR55 8 years ago
deajan c209fdd116 Fixes for remote deletion from PR55 8 years ago
deajan 6196ce6b31 More misc fixes 8 years ago
deajan 13537c5768 Fixed doubled path 8 years ago
deajan 05d2a33b7e Fix typo 8 years ago
deajan 98024c824e Misc fix 8 years ago

@ -0,0 +1,9 @@
language:
bash
os:
linux
osx
script:
dev/tests/run_tests.sh

@ -7,6 +7,23 @@ KNOWN ISSUES
RECENT CHANGES
--------------
02 Sep 2016:
- Fixed directories containing spaces with bash >= 4.3
- Fixed installer for CYGWIN / MSYS environment
28 Aug 2016: osync v1.1.2 released
- Renamed sync.conf to sync.conf.example (thanks to https://github.com/hortimech)
- Fixed RunAfterHook may be executed twice
- Fixed soft deletion when SUDO_EXEC is enabled
06 Aug 2016: osync v1.1.1 released
- Fixed bogus rsync pattern additions
- Fixed soft deletion always enabled on target
- Fixed problem with attributes file list function
- Fixed deletion propagation code
- Fixed missing deletion / backup diretories message in verbose mode
27 Jul 2016: osync v1.1 released
- More msys and cygwin compatibility
- Logging begins now before any remote checks

@ -33,11 +33,12 @@ Microsoft Windows is supported via MSYS or Cygwin.
Osync has been designed to not delete any data, but rather make backups of conflictual files or soft deletes.
Nevertheless, you should always have a neat backup of your data before trying a new sync tool.
You can download the latest stable release of Osync at www.netpower.fr/osync or https://github.com/deajan/osync/archive/v1.1.tar.gz
You can download the latest stable release of Osync at https://github.com/deajan/osync/archive/v1.1.2.tar.gz
You may also get the last development version at https://github.com/deajan/osync with the following command
$ git clone -b "v1.1" https://github.com/deajan/osync
$ git clone -b "v1.2-maint" https://github.com/deajan/osync
$ cd osync
$ sh install.sh
Osync will install itself to /usr/local/bin and an example configuration file will be installed to /etc/osync

@ -0,0 +1,315 @@
#!/usr/bin/env bash
PROGRAM=[prgname]
PROGRAM_VERSION=[version]
PROGRAM_BINARY=$PROGRAM".sh"
PROGRAM_BATCH=$PROGRAM"-batch.sh"
SCRIPT_BUILD=2016082902
## osync / obackup / pmocr / zsnap install script
## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8 & 10
## Please adapt this to fit your distro needs
#TODO: silent mode and no stats mode
CONF_DIR=$FAKEROOT/etc/$PROGRAM
BIN_DIR="$FAKEROOT/usr/local/bin"
SERVICE_DIR_INIT=$FAKEROOT/etc/init.d
# Should be /usr/lib/systemd/system, but /lib/systemd/system exists on debian & rhel / fedora
SERVICE_DIR_SYSTEMD_SYSTEM=$FAKEROOT/lib/systemd/system
SERVICE_DIR_SYSTEMD_USER=$FAKEROOT/etc/systemd/user
## osync specific code
OSYNC_SERVICE_FILE_INIT="osync-srv"
OSYNC_SERVICE_FILE_SYSTEMD_SYSTEM="osync-srv@.service"
OSYNC_SERVICE_FILE_SYSTEMD_USER="osync-srv@.service.user"
## pmocr specfic code
PMOCR_SERVICE_FILE_INIT="pmocr-srv"
PMOCR_SERVICE_FILE_SYSTEMD_SYSTEM="pmocr-srv.service"
## Generic code
## Default log file
if [ -w $FAKEROOT/var/log ]; then
LOG_FILE="$FAKEROOT/var/log/$PROGRAM-install.log"
elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
LOG_FILE="$HOME/$PROGRAM-install.log"
else
LOG_FILE="./$PROGRAM-install.log"
fi
# Generic quick logging function
function _QuickLogger {
local value="${1}"
local destination="${2}" # Destination: stdout, log, both
if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then
echo -e "$(date) - $value" >> "$LOG_FILE"
elif ([ "$destination" == "stdout" ] || [ "$destination" == "both" ]); then
echo -e "$value"
fi
}
function QuickLogger {
local value="${1}"
if [ "$_SILENT" -eq 1 ]; then
_QuickLogger "$value" "log"
else
_QuickLogger "$value" "stdout"
fi
}
function urlencode() {
# urlencode <string>
local LANG=C
local length="${#1}"
for (( i = 0; i < length; i++ )); do
local c="${1:i:1}"
case $c in
[a-zA-Z0-9.~_-]) printf "$c" ;;
*) printf '%%%02X' "'$c" ;;
esac
done
}
function SetOSSettings {
USER=root
local local_os_var
local_os_var="$(uname -spio 2>&1)"
if [ $? != 0 ]; then
local_os_var="$(uname -v 2>&1)"
if [ $? != 0 ]; then
local_os_var="$(uname)"
fi
fi
case $local_os_var in
*"BSD"*)
GROUP=wheel
;;
*"Darwin"*)
GROUP=admin
;;
*"MINGW32"*|*"CYGWIN"*)
USER=""
GROUP=""
;;
*)
GROUP=root
;;
esac
if ([ "$USER" != "" ] && [ "$(whoami)" != "$USER" ] && [ "$FAKEROOT" == "" ]); then
QuickLogger "Must be run as $USER."
exit 1
fi
OS=$(urlencode "$local_os_var")
}
function GetInit {
if [ -f /sbin/init ]; then
if file /sbin/init | grep systemd > /dev/null; then
init="systemd"
else
init="initV"
fi
else
QuickLogger "Can't detect initV or systemd. Service files won't be installed. You can still run $PROGRAM manually or via cron."
init="none"
fi
}
function CreateConfDir {
if [ ! -d "$CONF_DIR" ]; then
mkdir "$CONF_DIR"
if [ $? == 0 ]; then
QuickLogger "Created directory [$CONF_DIR]."
else
QuickLogger "Cannot create directory [$CONF_DIR]."
exit 1
fi
else
QuickLogger "Config directory [$CONF_DIR] exists."
fi
}
function CopyExampleFiles {
if [ -f "./sync.conf.example" ]; then
cp "./sync.conf.example" "$FAKEROOT/etc/$PROGRAM/sync.conf.example"
fi
if [ -f "./host_backup.conf.example" ]; then
cp "./host_backup.conf.example" "$FAKEROOT/etc/$PROGRAM/host_backup.conf.example"
fi
if [ -f "./exlude.list.example" ]; then
cp "./exclude.list.example" "$FAKEROOT/etc/$PROGRAM"
fi
if [ -f "./snapshot.conf.example" ]; then
cp "./snapshot.conf.example" "$FAKEROOT/etc/$PROGRAM/snapshot.conf.example"
fi
}
function CopyProgram {
cp "./$PROGRAM_BINARY" "$BIN_DIR"
if [ $? != 0 ]; then
QuickLogger "Cannot copy $PROGRAM_BINARY to [$BIN_DIR]. Make sure to run install script in the directory containing all other files."
QuickLogger "Also make sure you have permissions to write to [$BIN_DIR]."
exit 1
else
chmod 755 "$BIN_DIR/$PROGRAM_BINARY"
QuickLogger "Copied $PROGRAM_BINARY to [$BIN_DIR]."
fi
if [ -f "./$PROGRAM_BATCH" ]; then
cp "./$PROGRAM_BATCH" "$BIN_DIR"
if [ $? != 0 ]; then
QuickLogger "Cannot copy $PROGRAM_BATCH to [$BIN_DIR]."
else
chmod 755 "$BIN_DIR/$PROGRAM_BATCH"
QuickLogger "Copied $PROGRAM_BATCH to [$BIN_DIR]."
fi
fi
if [ -f "./ssh_filter.sh" ]; then
cp "./ssh_filter.sh" "$BIN_DIR"
if [ $? != 0 ]; then
QuickLogger "Cannot copy ssh_filter.sh to [$BIN_DIR]."
else
chmod 755 "$BIN_DIR/ssh_filter.sh"
if ([ "$USER" != "" ] && [ "$GROUP" != "" ] && [ "$FAKEROOT" == "" ]); then
chown $USER:$GROUP "$BIN_DIR/ssh_filter.sh"
fi
QuickLogger "Copied ssh_filter.sh to [$BIN_DIR]."
fi
fi
}
function CopyServiceFiles {
# OSYNC SPECIFIC
if ([ "$init" == "systemd" ] && [ -f "./$OSYNC_SERVICE_FILE_SYSTEMD_SYSTEM" ]); then
cp "./$OSYNC_SERVICE_FILE_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_SYSTEM" && cp "./$OSYNC_SERVICE_FILE_SYSTEMD_USER" "$SERVICE_DIR_SYSTEMD_USER/$SERVICE_FILE_SYSTEMD_SYSTEM"
if [ $? != 0 ]; then
QuickLogger "Cannot copy the systemd file to [$SERVICE_DIR_SYSTEMD_SYSTEM] or [$SERVICE_DIR_SYSTEMD_USER]."
else
QuickLogger "Created osync-srv service in [$SERVICE_DIR_SYSTEMD_SYSTEM] and [$SERVICE_DIR_SYSTEMD_USER]."
QuickLogger "Can be activated with [systemctl start osync-srv@instance.conf] where instance.conf is the name of the config file in /etc/osync."
QuickLogger "Can be enabled on boot with [systemctl enable osync-srv@instance.conf]."
QuickLogger "In userland, active with [systemctl --user start osync-srv@instance.conf]."
fi
elif ([ "$init" == "initV" ] && [ -f "./$OSYNC_SERVICE_FILE_INIT" ]); then
cp "./$OSYNC_SERVICE_FILE_INIT" "$SERVICE_DIR_INIT"
if [ $? != 0 ]; then
QuickLogger "Cannot copy osync-srv to [$SERVICE_DIR_INIT]."
else
chmod 755 "$SERVICE_DIR_INIT/$OSYNC_SERVICE_FILE_INIT"
QuickLogger "Created osync-srv service in [$SERVICE_DIR_INIT]."
QuickLogger "Can be activated with [service $OSYNC_SERVICE_FILE_INIT start]."
QuickLogger "Can be enabled on boot with [chkconfig $OSYNC_SERVICE_FILE_INIT on]."
fi
fi
# PMOCR SPECIFIC
if ([ "$init" == "systemd" ] && [ -f "./$PMOCR_SERVICE_FILE_SYSTEMD_SYSTEM" ]); then
cp "./$PMOCR_SERVICE_FILE_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_SYSTEM"
if [ $? != 0 ]; then
QuickLogger "Cannot copy the systemd file to [$SERVICE_DIR_SYSTEMD_SYSTEM] or [$SERVICE_DIR_SYSTEMD_USER]."
else
QuickLogger "Created pmocr-srv service in [$SERVICE_DIR_SYSTEMD_SYSTEM] and [$SERVICE_DIR_SYSTEMD_USER]."
QuickLogger "Can be activated with [systemctl start pmocr-srv] after configuring file options in [$BIN_DIR/$PROGRAM]."
QuickLogger "Can be enabled on boot with [systemctl enable pmocr-srv]."
fi
elif ([ "$init" == "initV" ] && [ -f "./$PMOCR_SERVICE_FILE_INIT" ]); then
cp "./$PMOCR_SERVICE_FILE_INIT" "$SERVICE_DIR_INIT"
if [ $? != 0 ]; then
QuickLogger "Cannot copy pmoct-srv to [$SERVICE_DIR_INIT]."
else
chmod 755 "$SERVICE_DIR_INIT/$PMOCR_SERVICE_FILE_INIT"
QuickLogger "Created osync-srv service in [$SERVICE_DIR_INIT]."
QuickLogger "Can be activated with [service $PMOCR_SERVICE_FILE_INIT start]."
QuickLogger "Can be enabled on boot with [chkconfig $PMOCR_SERVICE_FILE_INIT on]."
fi
fi
}
function Statistics {
if type wget > /dev/null; then
wget -qO- "$STATS_LINK" > /dev/null 2>&1
if [ $? == 0 ]; then
return 0
fi
fi
if type curl > /dev/null; then
curl "$STATS_LINK" -o /dev/null > /dev/null 2>&1
if [ $? == 0 ]; then
return 0
fi
fi
QuickLogger "Neiter wget nor curl could be used for. Cannot run statistics. Use the provided link please."
return 1
}
function Usage {
echo "Installs $PROGRAM into $BIN_DIR"
echo "options:"
echo "--silent Will log and bypass user interaction."
echo "--no-stats Used with --silent in order to refuse sending anonymous install stats."
exit 127
}
_SILENT=0
_STATS=1
for i in "$@"
do
case $i in
--silent)
_SILENT=1
;;
--no-stats)
_STATS=0
;;
--help|-h|-?)
Usage
esac
done
if [ "$FAKEROOT" != "" ]; then
mkdir -p "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_USER" "$BIN_DIR"
fi
SetOSSettings
CreateConfDir
CopyExampleFiles
CopyProgram
GetInit
CopyServiceFiles
STATS_LINK="http://instcount.netpower.fr?program=$PROGRAM&version=$PROGRAM_VERSION&os=$OS"
QuickLogger "$PROGRAM installed. Use with $BIN_DIR/$PROGRAM"
if [ $_STATS -eq 1 ]; then
if [ $_SILENT -eq 1 ]; then
Statistics
else
QuickLogger "In order to make install statistics, the script would like to connect to $STATS_LINK"
read -r -p "No data except those in the url will be send. Allow [Y/n]" response
case $response in
[nN])
exit
;;
*)
Statistics
exit $?
;;
esac
fi
fi

@ -3,11 +3,11 @@
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.1
PROGRAM_BUILD=2016072701
PROGRAM_VERSION=1.1.4
PROGRAM_BUILD=2016090201
IS_STABLE=yes
## FUNC_BUILD=2016071902
## FUNC_BUILD=2016071902-d
## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
@ -80,7 +80,7 @@ fi
# Default alert attachment filename
ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.last.log"
ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.$$.last.log"
# Set error exit code if a piped command fails
set -o pipefail
@ -351,7 +351,7 @@ function SendAlert {
# Delete tmp log file
if [ -f "$ALERT_LOG_FILE" ]; then
rm "$ALERT_LOG_FILE"
rm -f "$ALERT_LOG_FILE"
fi
}
@ -386,7 +386,7 @@ function SendEmail {
local auth_string=
if [ ! -f "$attachment" ]; then
attachment_command="-a $ALERT_LOG_FILE"
attachment_command="-a $attachment"
mail_no_attachment=1
else
mail_no_attachment=0
@ -577,7 +577,7 @@ function StripQuotes {
function EscapeSpaces {
local string="${1}" # String on which spaces will be escaped
echo "${string// /\ }"
echo "${string// /\\ }"
}
function IsNumeric {
@ -1041,8 +1041,6 @@ function RsyncPatternsFromAdd {
local pattern_from="${2}"
__CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local pattern_from=
## Check if the exclude list has a full path, and if not, add the config file path if there is one
if [ "$(basename $pattern_from)" == "$pattern_from" ]; then
pattern_from="$(dirname $CONFIG_FILE)/$pattern_from"
@ -1308,9 +1306,7 @@ function TrapQuit {
exitcode=240 # Special exit code for daemon mode not stopping on warnings
else
UnlockReplicas
if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then
RunAfterHook
fi
RunAfterHook
CleanUp
Logger "$PROGRAM finished." "NOTICE"
exitcode=0
@ -1826,9 +1822,8 @@ function _get_file_ctime_mtime_local {
local file_list="${3}" # Contains list of files to get time attrs
__CheckArguments 3 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
#cat "$file_list" | xargs -I {} stat -c '%n;%Z;%Y' "$replica_path{}" | sort > "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"
echo -n "" > "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"
while read file; do $STAT_CTIME_MTIME_CMD "$replica_path$file" | sort > "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"; done < "$file_list"
while read file; do $STAT_CTIME_MTIME_CMD "$replica_path$file" | sort >> "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"; done < "$file_list"
}
function _get_file_ctime_mtime_remote {
@ -1837,7 +1832,8 @@ function _get_file_ctime_mtime_remote {
local file_list="${3}"
__CheckArguments 3 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local cmd=
local cmd
cmd='cat "'$file_list'" | '$SSH_CMD' "while read file; do '$REMOTE_STAT_CTIME_MTIME_CMD' \"'$replica_path'\$file\"; done | sort" > "'$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID'"'
Logger "CMD: $cmd" "DEBUG"
eval $cmd
@ -2041,18 +2037,19 @@ function _delete_local {
do
if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then
if [ "$SOFT_DELETE" != "no" ]; then
if [ ! -d "$replica_dir$deletion_dir" ]; then
mkdir -p "$replica_dir$deletion_dir"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ $_VERBOSE -eq 1 ]; then
Logger "Soft deleting $replica_dir$files" "NOTICE"
fi
if [ $_DRYRUN -ne 1 ]; then
if [ ! -d "$replica_dir$deletion_dir" ]; then
mkdir -p "$replica_dir$deletion_dir"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ -e "$replica_dir$deletion_dir/$files" ]; then
rm -rf "${replica_dir:?}$deletion_dir/$files"
fi
@ -2108,7 +2105,7 @@ function _delete_remote {
# Additionnaly, we need to copy the deletetion list to the remote state folder
esc_dest_dir="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}")"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" \"${INITIATOR[1]}${INITIATOR[3]}/$2\" $REMOTE_USER@$REMOTE_HOST:\"$esc_dest_dir/\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID 2>&1"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"${INITIATOR[1]}${INITIATOR[3]}/$deleted_list_file\" $REMOTE_USER@$REMOTE_HOST:\"$esc_dest_dir/\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID 2>&1"
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd" 2>> "$LOG_FILE"
if [ $? != 0 ]; then
@ -2119,12 +2116,12 @@ function _delete_remote {
exit 1
fi
$SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=$_DEBUG _DRYRUN=$_DRYRUN _VERBOSE=$_VERBOSE COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "$TARGET_STATE_DIR/$deleted_list_file")" REPLICA_DIR="$(EscapeSpaces "$replica_dir")" DELETE_DIR="$(EscapeSpaces "$deletion_dir")" FAILED_DELETE_LIST="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" 2>&1 &
$SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=$_DEBUG _DRYRUN=$_DRYRUN _VERBOSE=$_VERBOSE COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_list_file")" REPLICA_DIR="$(EscapeSpaces "$replica_dir")" SOFT_DELETE=$SOFT_DELETE DELETE_DIR="$(EscapeSpaces "$deletion_dir")" FAILED_DELETE_LIST="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" 2>&1 &
## The following lines are executed remotely
function _logger {
local value="${1}" # What to log
echo -e "$value" >> "$LOG_FILE"
echo -e "$value" >&2 # Log to STDERR
if [ $_SILENT -eq 0 ]; then
echo -e "$value"
@ -2168,32 +2165,36 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=
IFS=$'\r\n'
for files in $(cat "$FILE_LIST")
do
Logger "Processing file [$file]." "DEBUG"
if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then
if [ ! -d "$REPLICA_DIR$DELETE_DIR" ]; then
$COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ "$SOFT_DELETE" != "no" ]; then
if [ $_VERBOSE -eq 1 ]; then
Logger "Soft deleting $REPLICA_DIR$files" "NOTICE"
fi
Logger "Full path for deletion is [$REPLICA_DIR$DELETE_DIR/$files]." "DEBUG"
if [ ! -d "$REPLICA_DIR$DELETE_DIR" ]; then
$COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ $_DRYRUN -ne 1 ]; then
if [ -e "$REPLICA_DIR$DELETE_DIR/$files" ]; then
$COMMAND_SUDO rm -rf "$REPLICA_DIR$DELETE_DIR/$files"
fi
if [ -e "$$REPLICA_DIR$files" ]; then
if [ -e "$REPLICA_DIR$files" ]; then
# In order to keep full path on soft deletion, create parent directories before move
parentdir="$(dirname "$files")"
if [ "$parentdir" != "." ]; then
$COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR/$parentdir"
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR/$parentdir"
else
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR"1
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR"
fi
if [ $? != 0 ]; then
Logger "Cannot move $REPLICA_DIR$files to deletion directory." "ERROR"
@ -2211,7 +2212,7 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=
$COMMAND_SUDO rm -rf "$REPLICA_DIR$files"
if [ $? != 0 ]; then
Logger "Cannot delete $REPLICA_DIR$files" "ERROR"
echo "$files" >> "$TARGET_STATE_DIR/$FAILED_DELETE_LIST"
echo "$files" >> "$FAILED_DELETE_LIST"
fi
fi
fi
@ -2227,7 +2228,7 @@ ENDSSH
## Copy back the deleted failed file list
esc_source_file="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $REMOTE_USER@$REMOTE_HOST:\"$esc_source_file\" \"${INITIATOR[1]}${INITIATOR[3]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID\""
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" $REMOTE_USER@$REMOTE_HOST:\"$esc_source_file\" \"${INITIATOR[1]}${INITIATOR[3]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID\""
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd" 2>> "$LOG_FILE"
if [ $? != 0 ]; then
@ -2488,13 +2489,13 @@ function _SoftDeleteLocal {
fi
if [ $_VERBOSE -eq 1 ]; then
# Cannot launch log function from xargs, ugly hack
$FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete file {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
$COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete file {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
$FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete directory {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
$COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete directory {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
fi
if [ $_DRYRUN -ne 1 ]; then
$FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} rm -f "{}" && $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} rm -rf "{}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 &
$COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} $COMMAND_SUDO rm -f "{}" && $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} $COMMAND_SUDO rm -rf "{}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 &
else
Dummy &
fi
@ -2523,14 +2524,14 @@ function _SoftDeleteRemote {
CheckConnectivityRemoteHost
if [ $_DRYRUN -eq 1 ]; then
Logger "Listing files older than $change_time days on target replica. Does not remove anything." "NOTICE"
Logger "Listing files older than $change_time days on $replica_type replica. Does not remove anything." "NOTICE"
else
Logger "Removing files older than $change_time days on target replica." "NOTICE"
Logger "Removing files older than $change_time days on $replica_type replica." "NOTICE"
fi
if [ $_VERBOSE -eq 1 ]; then
# Cannot launch log function from xargs, ugly hack
cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} echo Will delete file {} && '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"No remote backup/deletion directory.\"; exit 1; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} echo Will delete file {} && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
@ -2538,7 +2539,7 @@ function _SoftDeleteRemote {
fi
if [ $_DRYRUN -ne 1 ]; then
cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} rm -f \"{}\" && '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} rm -rf \"{}\"; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} '$COMMAND_SUDO' rm -f \"{}\" && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} '$COMMAND_SUDO' rm -rf \"{}\"; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
@ -2548,10 +2549,10 @@ function _SoftDeleteRemote {
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
retval=$?
if [ $retval -ne 0 ]; then
Logger "Error while executing cleanup on remote target replica." "ERROR"
Logger "Error while executing cleanup on remote $replica_type replica." "ERROR"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
else
Logger "Cleanup complete on target replica." "NOTICE"
Logger "Cleanup complete on $replica_type replica." "NOTICE"
fi
}
@ -3020,5 +3021,4 @@ opts="${opts# *}"
if [ $? == 0 ]; then
SoftDelete
fi
RunAfterHook
fi

@ -1,26 +1,38 @@
#!/usr/bin/env bash
## Merges ofunctions.sh and $PROGRAM
## MERGE 2016080601
## Merges ofunctions.sh and n_program.sh into program.sh
## Adds installer
PROGRAM=osync
VERSION=$(grep "PROGRAM_VERSION=" n_$PROGRAM.sh)
VERSION=${VERSION#*=}
FUNC_PATH=/home/git/common
PARANOIA_DEBUG_LINE="#__WITH_PARANOIA_DEBUG"
PARANOIA_DEBUG_LINE="__WITH_PARANOIA_DEBUG"
PARANOIA_DEBUG_BEGIN="#__BEGIN_WITH_PARANOIA_DEBUG"
PARANOIA_DEBUG_END="#__END_WITH_PARANOIA_DEBUG"
MINIMUM_FUNCTION_BEGIN="#### MINIMAL-FUNCTION-SET BEGIN ####"
MINIMUM_FUNCTION_END="#### MINIMAL-FUNCTION-SET END ####"
function Unexpand {
unexpand n_$PROGRAM.sh > tmp_$PROGRAM.sh
unexpand n_$PROGRAM.sh > tmp_$PROGRAM.sh
}
function Merge {
function MergeAll {
sed "/source \"\.\/ofunctions.sh\"/r ofunctions.sh" tmp_$PROGRAM.sh | grep -v 'source "./ofunctions.sh"' > debug_$PROGRAM.sh
chmod +x debug_$PROGRAM.sh
}
sed "/source \"\.\/ofunctions.sh\"/r /home/git/common/ofunctions.sh" tmp_$PROGRAM.sh | grep -v 'source "./ofunctions.sh"' > debug_$PROGRAM.sh
function MergeMinimum {
sed -n "/$MINIMUM_FUNCTION_BEGIN/,/$MINIMUM_FUNCTION_END/p" ofunctions.sh > tmp_minimal.sh
sed "/source \"\.\/ofunctions.sh\"/r tmp_minimal.sh" tmp_$PROGRAM.sh | grep -v 'source "./ofunctions.sh"' | grep -v "$PARANOIA_DEBUG_LINE" > debug_$PROGRAM.sh
rm -f tmp_minimal.sh
chmod +x debug_$PROGRAM.sh
}
function CleanDebug {
# sed explanation
@ -38,16 +50,22 @@ function CleanDebug {
}
function CopyCommons {
sed "s/\[prgname\]/$PROGRAM/g" /home/git/common/common_install.sh > ../tmp_install.sh
sed "s/\[version\]/$VERSION/g" ../tmp_install.sh > ../install.sh
sed "s/\[prgname\]/$PROGRAM/g" /home/git/common/common_batch.sh > ../$PROGRAM-batch.sh
chmod +x ../install.sh
chmod +x ../$PROGRAM-batch.sh
sed "s/\[prgname\]/$PROGRAM/g" common_install.sh > ../tmp_install.sh
sed "s/\[version\]/$VERSION/g" ../tmp_install.sh > ../install.sh
if [ -f "common_batch.sh" ]; then
sed "s/\[prgname\]/$PROGRAM/g" common_batch.sh > ../$PROGRAM-batch.sh
fi
chmod +x ../install.sh
chmod +x ../$PROGRAM-batch.sh
rm -f ../tmp_install.sh
}
Unexpand
Merge
if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
MergeAll
else
MergeMinimum
fi
CleanDebug
rm -f tmp_$PROGRAM.sh
rm -f ../tmp_install.sh
CopyCommons
rm -f tmp_$PROGRAM.sh

@ -3,8 +3,8 @@
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.1
PROGRAM_BUILD=2016072701
PROGRAM_VERSION=1.1.4
PROGRAM_BUILD=2016090201
IS_STABLE=yes
source "./ofunctions.sh"
@ -60,9 +60,7 @@ function TrapQuit {
exitcode=240 # Special exit code for daemon mode not stopping on warnings
else
UnlockReplicas
if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then
RunAfterHook
fi
RunAfterHook
CleanUp
Logger "$PROGRAM finished." "NOTICE"
exitcode=0
@ -578,9 +576,8 @@ function _get_file_ctime_mtime_local {
local file_list="${3}" # Contains list of files to get time attrs
__CheckArguments 3 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
#cat "$file_list" | xargs -I {} stat -c '%n;%Z;%Y' "$replica_path{}" | sort > "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"
echo -n "" > "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"
while read file; do $STAT_CTIME_MTIME_CMD "$replica_path$file" | sort > "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"; done < "$file_list"
while read file; do $STAT_CTIME_MTIME_CMD "$replica_path$file" | sort >> "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"; done < "$file_list"
}
function _get_file_ctime_mtime_remote {
@ -589,7 +586,8 @@ function _get_file_ctime_mtime_remote {
local file_list="${3}"
__CheckArguments 3 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local cmd=
local cmd
cmd='cat "'$file_list'" | '$SSH_CMD' "while read file; do '$REMOTE_STAT_CTIME_MTIME_CMD' \"'$replica_path'\$file\"; done | sort" > "'$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID'"'
Logger "CMD: $cmd" "DEBUG"
eval $cmd
@ -793,18 +791,19 @@ function _delete_local {
do
if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then
if [ "$SOFT_DELETE" != "no" ]; then
if [ ! -d "$replica_dir$deletion_dir" ]; then
mkdir -p "$replica_dir$deletion_dir"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ $_VERBOSE -eq 1 ]; then
Logger "Soft deleting $replica_dir$files" "NOTICE"
fi
if [ $_DRYRUN -ne 1 ]; then
if [ ! -d "$replica_dir$deletion_dir" ]; then
mkdir -p "$replica_dir$deletion_dir"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ -e "$replica_dir$deletion_dir/$files" ]; then
rm -rf "${replica_dir:?}$deletion_dir/$files"
fi
@ -860,7 +859,7 @@ function _delete_remote {
# Additionnaly, we need to copy the deletetion list to the remote state folder
esc_dest_dir="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}")"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" \"${INITIATOR[1]}${INITIATOR[3]}/$2\" $REMOTE_USER@$REMOTE_HOST:\"$esc_dest_dir/\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID 2>&1"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"${INITIATOR[1]}${INITIATOR[3]}/$deleted_list_file\" $REMOTE_USER@$REMOTE_HOST:\"$esc_dest_dir/\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID 2>&1"
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd" 2>> "$LOG_FILE"
if [ $? != 0 ]; then
@ -871,12 +870,12 @@ function _delete_remote {
exit 1
fi
$SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=$_DEBUG _DRYRUN=$_DRYRUN _VERBOSE=$_VERBOSE COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "$TARGET_STATE_DIR/$deleted_list_file")" REPLICA_DIR="$(EscapeSpaces "$replica_dir")" DELETE_DIR="$(EscapeSpaces "$deletion_dir")" FAILED_DELETE_LIST="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" 2>&1 &
$SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=$_DEBUG _DRYRUN=$_DRYRUN _VERBOSE=$_VERBOSE COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_list_file")" REPLICA_DIR="$(EscapeSpaces "$replica_dir")" SOFT_DELETE=$SOFT_DELETE DELETE_DIR="$(EscapeSpaces "$deletion_dir")" FAILED_DELETE_LIST="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" 2>&1 &
## The following lines are executed remotely
function _logger {
local value="${1}" # What to log
echo -e "$value" >> "$LOG_FILE"
echo -e "$value" >&2 # Log to STDERR
if [ $_SILENT -eq 0 ]; then
echo -e "$value"
@ -920,32 +919,36 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=
IFS=$'\r\n'
for files in $(cat "$FILE_LIST")
do
Logger "Processing file [$file]." "DEBUG"
if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then
if [ ! -d "$REPLICA_DIR$DELETE_DIR" ]; then
$COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ "$SOFT_DELETE" != "no" ]; then
if [ $_VERBOSE -eq 1 ]; then
Logger "Soft deleting $REPLICA_DIR$files" "NOTICE"
fi
Logger "Full path for deletion is [$REPLICA_DIR$DELETE_DIR/$files]." "DEBUG"
if [ ! -d "$REPLICA_DIR$DELETE_DIR" ]; then
$COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ $_DRYRUN -ne 1 ]; then
if [ -e "$REPLICA_DIR$DELETE_DIR/$files" ]; then
$COMMAND_SUDO rm -rf "$REPLICA_DIR$DELETE_DIR/$files"
fi
if [ -e "$$REPLICA_DIR$files" ]; then
if [ -e "$REPLICA_DIR$files" ]; then
# In order to keep full path on soft deletion, create parent directories before move
parentdir="$(dirname "$files")"
if [ "$parentdir" != "." ]; then
$COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR/$parentdir"
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR/$parentdir"
else
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR"1
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR"
fi
if [ $? != 0 ]; then
Logger "Cannot move $REPLICA_DIR$files to deletion directory." "ERROR"
@ -963,7 +966,7 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=
$COMMAND_SUDO rm -rf "$REPLICA_DIR$files"
if [ $? != 0 ]; then
Logger "Cannot delete $REPLICA_DIR$files" "ERROR"
echo "$files" >> "$TARGET_STATE_DIR/$FAILED_DELETE_LIST"
echo "$files" >> "$FAILED_DELETE_LIST"
fi
fi
fi
@ -979,7 +982,7 @@ ENDSSH
## Copy back the deleted failed file list
esc_source_file="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $REMOTE_USER@$REMOTE_HOST:\"$esc_source_file\" \"${INITIATOR[1]}${INITIATOR[3]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID\""
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" $REMOTE_USER@$REMOTE_HOST:\"$esc_source_file\" \"${INITIATOR[1]}${INITIATOR[3]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID\""
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd" 2>> "$LOG_FILE"
if [ $? != 0 ]; then
@ -1240,13 +1243,13 @@ function _SoftDeleteLocal {
fi
if [ $_VERBOSE -eq 1 ]; then
# Cannot launch log function from xargs, ugly hack
$FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete file {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
$COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete file {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
$FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete directory {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
$COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete directory {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
fi
if [ $_DRYRUN -ne 1 ]; then
$FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} rm -f "{}" && $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} rm -rf "{}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 &
$COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} $COMMAND_SUDO rm -f "{}" && $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} $COMMAND_SUDO rm -rf "{}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 &
else
Dummy &
fi
@ -1275,14 +1278,14 @@ function _SoftDeleteRemote {
CheckConnectivityRemoteHost
if [ $_DRYRUN -eq 1 ]; then
Logger "Listing files older than $change_time days on target replica. Does not remove anything." "NOTICE"
Logger "Listing files older than $change_time days on $replica_type replica. Does not remove anything." "NOTICE"
else
Logger "Removing files older than $change_time days on target replica." "NOTICE"
Logger "Removing files older than $change_time days on $replica_type replica." "NOTICE"
fi
if [ $_VERBOSE -eq 1 ]; then
# Cannot launch log function from xargs, ugly hack
cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} echo Will delete file {} && '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"No remote backup/deletion directory.\"; exit 1; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} echo Will delete file {} && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
@ -1290,7 +1293,7 @@ function _SoftDeleteRemote {
fi
if [ $_DRYRUN -ne 1 ]; then
cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} rm -f \"{}\" && '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} rm -rf \"{}\"; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} '$COMMAND_SUDO' rm -f \"{}\" && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} '$COMMAND_SUDO' rm -rf \"{}\"; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
@ -1300,10 +1303,10 @@ function _SoftDeleteRemote {
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
retval=$?
if [ $retval -ne 0 ]; then
Logger "Error while executing cleanup on remote target replica." "ERROR"
Logger "Error while executing cleanup on remote $replica_type replica." "ERROR"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
else
Logger "Cleanup complete on target replica." "NOTICE"
Logger "Cleanup complete on $replica_type replica." "NOTICE"
fi
}
@ -1772,5 +1775,4 @@ opts="${opts# *}"
if [ $? == 0 ]; then
SoftDelete
fi
RunAfterHook
fi

@ -1,4 +1,4 @@
## FUNC_BUILD=2016071902
## FUNC_BUILD=2016071902-d
## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
@ -71,7 +71,7 @@ fi
# Default alert attachment filename
ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.last.log"
ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.$$.last.log"
# Set error exit code if a piped command fails
set -o pipefail
@ -342,7 +342,7 @@ function SendAlert {
# Delete tmp log file
if [ -f "$ALERT_LOG_FILE" ]; then
rm "$ALERT_LOG_FILE"
rm -f "$ALERT_LOG_FILE"
fi
}
@ -377,7 +377,7 @@ function SendEmail {
local auth_string=
if [ ! -f "$attachment" ]; then
attachment_command="-a $ALERT_LOG_FILE"
attachment_command="-a $attachment"
mail_no_attachment=1
else
mail_no_attachment=0
@ -568,7 +568,7 @@ function StripQuotes {
function EscapeSpaces {
local string="${1}" # String on which spaces will be escaped
echo "${string// /\ }"
echo "${string// /\\ }"
}
function IsNumeric {
@ -1032,8 +1032,6 @@ function RsyncPatternsFromAdd {
local pattern_from="${2}"
__CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local pattern_from=
## Check if the exclude list has a full path, and if not, add the config file path if there is one
if [ "$(basename $pattern_from)" == "$pattern_from" ]; then
pattern_from="$(dirname $CONFIG_FILE)/$pattern_from"

@ -0,0 +1,8 @@
#!/usr/bin/env bash
# Basic run test to make travis happy
mkdir t1
mkdir t2
./osync.sh --initiator=t1 --target=t2

@ -1,10 +1,10 @@
#!/usr/bin/env bash
PROGRAM=osync
PROGRAM_VERSION=1.1
PROGRAM_VERSION=1.1.4
PROGRAM_BINARY=$PROGRAM".sh"
PROGRAM_BATCH=$PROGRAM"-batch.sh"
SCRIPT_BUILD=2016052601
SCRIPT_BUILD=2016082902
## osync / obackup / pmocr / zsnap install script
## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8 & 10
@ -12,12 +12,12 @@ SCRIPT_BUILD=2016052601
#TODO: silent mode and no stats mode
CONF_DIR=/etc/$PROGRAM
BIN_DIR=/usr/local/bin
SERVICE_DIR_INIT=/etc/init.d
CONF_DIR=$FAKEROOT/etc/$PROGRAM
BIN_DIR="$FAKEROOT/usr/local/bin"
SERVICE_DIR_INIT=$FAKEROOT/etc/init.d
# Should be /usr/lib/systemd/system, but /lib/systemd/system exists on debian & rhel / fedora
SERVICE_DIR_SYSTEMD_SYSTEM=/lib/systemd/system
SERVICE_DIR_SYSTEMD_USER=/etc/systemd/user
SERVICE_DIR_SYSTEMD_SYSTEM=$FAKEROOT/lib/systemd/system
SERVICE_DIR_SYSTEMD_USER=$FAKEROOT/etc/systemd/user
## osync specific code
OSYNC_SERVICE_FILE_INIT="osync-srv"
@ -31,8 +31,8 @@ PMOCR_SERVICE_FILE_SYSTEMD_SYSTEM="pmocr-srv.service"
## Generic code
## Default log file
if [ -w /var/log ]; then
LOG_FILE="/var/log/$PROGRAM-install.log"
if [ -w $FAKEROOT/var/log ]; then
LOG_FILE="$FAKEROOT/var/log/$PROGRAM-install.log"
elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
LOG_FILE="$HOME/$PROGRAM-install.log"
else
@ -95,16 +95,16 @@ function SetOSSettings {
*"Darwin"*)
GROUP=admin
;;
*)
GROUP=root
;;
*"MINGW32"*|*"CYGWIN"*)
USER=""
GROUP=""
;;
*)
GROUP=root
;;
esac
if ([ "$USER" != "" ] && [ "$(whoami)" != "$USER" ]); then
if ([ "$USER" != "" ] && [ "$(whoami)" != "$USER" ] && [ "$FAKEROOT" == "" ]); then
QuickLogger "Must be run as $USER."
exit 1
fi
@ -141,19 +141,19 @@ function CreateConfDir {
function CopyExampleFiles {
if [ -f "./sync.conf.example" ]; then
cp "./sync.conf.example" "/etc/$PROGRAM/sync.conf.example"
cp "./sync.conf.example" "$FAKEROOT/etc/$PROGRAM/sync.conf.example"
fi
if [ -f "./host_backup.conf.example" ]; then
cp "./host_backup.conf.example" "/etc/$PROGRAM/host_backup.conf.example"
cp "./host_backup.conf.example" "$FAKEROOT/etc/$PROGRAM/host_backup.conf.example"
fi
if [ -f "./exlude.list.example" ]; then
cp "./exclude.list.example" "/etc/$PROGRAM"
cp "./exclude.list.example" "$FAKEROOT/etc/$PROGRAM"
fi
if [ -f "./snapshot.conf.example" ]; then
cp "./snapshot.conf.example" "/etc/$PROGRAM/snapshot.conf.example"
cp "./snapshot.conf.example" "$FAKEROOT/etc/$PROGRAM/snapshot.conf.example"
fi
}
@ -184,7 +184,7 @@ function CopyProgram {
QuickLogger "Cannot copy ssh_filter.sh to [$BIN_DIR]."
else
chmod 755 "$BIN_DIR/ssh_filter.sh"
if ([ "$USER" != "" ] && [ "$GROUP" != "" ]); then
if ([ "$USER" != "" ] && [ "$GROUP" != "" ] && [ "$FAKEROOT" == "" ]); then
chown $USER:$GROUP "$BIN_DIR/ssh_filter.sh"
fi
QuickLogger "Copied ssh_filter.sh to [$BIN_DIR]."
@ -282,6 +282,10 @@ do
esac
done
if [ "$FAKEROOT" != "" ]; then
mkdir -p "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_USER" "$BIN_DIR"
fi
SetOSSettings
CreateConfDir
CopyExampleFiles

@ -3,11 +3,11 @@
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.1
PROGRAM_BUILD=2016072701
PROGRAM_VERSION=1.1.4
PROGRAM_BUILD=2016090201
IS_STABLE=yes
## FUNC_BUILD=2016071902
## FUNC_BUILD=2016071902-d
## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
@ -76,7 +76,7 @@ fi
# Default alert attachment filename
ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.last.log"
ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.$$.last.log"
# Set error exit code if a piped command fails
set -o pipefail
@ -339,7 +339,7 @@ function SendAlert {
# Delete tmp log file
if [ -f "$ALERT_LOG_FILE" ]; then
rm "$ALERT_LOG_FILE"
rm -f "$ALERT_LOG_FILE"
fi
}
@ -373,7 +373,7 @@ function SendEmail {
local auth_string=
if [ ! -f "$attachment" ]; then
attachment_command="-a $ALERT_LOG_FILE"
attachment_command="-a $attachment"
mail_no_attachment=1
else
mail_no_attachment=0
@ -563,7 +563,7 @@ function StripQuotes {
function EscapeSpaces {
local string="${1}" # String on which spaces will be escaped
echo "${string// /\ }"
echo "${string// /\\ }"
}
function IsNumeric {
@ -972,8 +972,6 @@ function RsyncPatternsFromAdd {
local pattern_type="${1}"
local pattern_from="${2}"
local pattern_from=
## Check if the exclude list has a full path, and if not, add the config file path if there is one
if [ "$(basename $pattern_from)" == "$pattern_from" ]; then
pattern_from="$(dirname $CONFIG_FILE)/$pattern_from"
@ -1234,9 +1232,7 @@ function TrapQuit {
exitcode=240 # Special exit code for daemon mode not stopping on warnings
else
UnlockReplicas
if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then
RunAfterHook
fi
RunAfterHook
CleanUp
Logger "$PROGRAM finished." "NOTICE"
exitcode=0
@ -1729,9 +1725,8 @@ function _get_file_ctime_mtime_local {
local replica_type="${2}" # Initiator / Target
local file_list="${3}" # Contains list of files to get time attrs
#cat "$file_list" | xargs -I {} stat -c '%n;%Z;%Y' "$replica_path{}" | sort > "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"
echo -n "" > "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"
while read file; do $STAT_CTIME_MTIME_CMD "$replica_path$file" | sort > "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"; done < "$file_list"
while read file; do $STAT_CTIME_MTIME_CMD "$replica_path$file" | sort >> "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"; done < "$file_list"
}
function _get_file_ctime_mtime_remote {
@ -1739,7 +1734,8 @@ function _get_file_ctime_mtime_remote {
local replica_type="${2}"
local file_list="${3}"
local cmd=
local cmd
cmd='cat "'$file_list'" | '$SSH_CMD' "while read file; do '$REMOTE_STAT_CTIME_MTIME_CMD' \"'$replica_path'\$file\"; done | sort" > "'$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID'"'
Logger "CMD: $cmd" "DEBUG"
eval $cmd
@ -1940,18 +1936,19 @@ function _delete_local {
do
if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then
if [ "$SOFT_DELETE" != "no" ]; then
if [ ! -d "$replica_dir$deletion_dir" ]; then
mkdir -p "$replica_dir$deletion_dir"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ $_VERBOSE -eq 1 ]; then
Logger "Soft deleting $replica_dir$files" "NOTICE"
fi
if [ $_DRYRUN -ne 1 ]; then
if [ ! -d "$replica_dir$deletion_dir" ]; then
mkdir -p "$replica_dir$deletion_dir"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ -e "$replica_dir$deletion_dir/$files" ]; then
rm -rf "${replica_dir:?}$deletion_dir/$files"
fi
@ -2006,7 +2003,7 @@ function _delete_remote {
# Additionnaly, we need to copy the deletetion list to the remote state folder
esc_dest_dir="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}")"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" \"${INITIATOR[1]}${INITIATOR[3]}/$2\" $REMOTE_USER@$REMOTE_HOST:\"$esc_dest_dir/\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID 2>&1"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" \"${INITIATOR[1]}${INITIATOR[3]}/$deleted_list_file\" $REMOTE_USER@$REMOTE_HOST:\"$esc_dest_dir/\" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID 2>&1"
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd" 2>> "$LOG_FILE"
if [ $? != 0 ]; then
@ -2017,12 +2014,12 @@ function _delete_remote {
exit 1
fi
$SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=$_DEBUG _DRYRUN=$_DRYRUN _VERBOSE=$_VERBOSE COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "$TARGET_STATE_DIR/$deleted_list_file")" REPLICA_DIR="$(EscapeSpaces "$replica_dir")" DELETE_DIR="$(EscapeSpaces "$deletion_dir")" FAILED_DELETE_LIST="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" 2>&1 &
$SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=$_DEBUG _DRYRUN=$_DRYRUN _VERBOSE=$_VERBOSE COMMAND_SUDO=$COMMAND_SUDO FILE_LIST="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_list_file")" REPLICA_DIR="$(EscapeSpaces "$replica_dir")" SOFT_DELETE=$SOFT_DELETE DELETE_DIR="$(EscapeSpaces "$deletion_dir")" FAILED_DELETE_LIST="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" 2>&1 &
## The following lines are executed remotely
function _logger {
local value="${1}" # What to log
echo -e "$value" >> "$LOG_FILE"
echo -e "$value" >&2 # Log to STDERR
if [ $_SILENT -eq 0 ]; then
echo -e "$value"
@ -2066,32 +2063,36 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=
IFS=$'\r\n'
for files in $(cat "$FILE_LIST")
do
Logger "Processing file [$file]." "DEBUG"
if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then
if [ ! -d "$REPLICA_DIR$DELETE_DIR" ]; then
$COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ "$SOFT_DELETE" != "no" ]; then
if [ $_VERBOSE -eq 1 ]; then
Logger "Soft deleting $REPLICA_DIR$files" "NOTICE"
fi
Logger "Full path for deletion is [$REPLICA_DIR$DELETE_DIR/$files]." "DEBUG"
if [ ! -d "$REPLICA_DIR$DELETE_DIR" ]; then
$COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR"
if [ $? != 0 ]; then
Logger "Cannot create replica deletion directory." "ERROR"
fi
fi
if [ $_DRYRUN -ne 1 ]; then
if [ -e "$REPLICA_DIR$DELETE_DIR/$files" ]; then
$COMMAND_SUDO rm -rf "$REPLICA_DIR$DELETE_DIR/$files"
fi
if [ -e "$$REPLICA_DIR$files" ]; then
if [ -e "$REPLICA_DIR$files" ]; then
# In order to keep full path on soft deletion, create parent directories before move
parentdir="$(dirname "$files")"
if [ "$parentdir" != "." ]; then
$COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR/$parentdir"
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR/$parentdir"
else
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR"1
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR"
fi
if [ $? != 0 ]; then
Logger "Cannot move $REPLICA_DIR$files to deletion directory." "ERROR"
@ -2109,7 +2110,7 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=
$COMMAND_SUDO rm -rf "$REPLICA_DIR$files"
if [ $? != 0 ]; then
Logger "Cannot delete $REPLICA_DIR$files" "ERROR"
echo "$files" >> "$TARGET_STATE_DIR/$FAILED_DELETE_LIST"
echo "$files" >> "$FAILED_DELETE_LIST"
fi
fi
fi
@ -2125,7 +2126,7 @@ ENDSSH
## Copy back the deleted failed file list
esc_source_file="$(EscapeSpaces "${TARGET[1]}${TARGET[3]}/$deleted_failed_list_file")"
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $REMOTE_USER@$REMOTE_HOST:\"$esc_source_file\" \"${INITIATOR[1]}${INITIATOR[3]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID\""
rsync_cmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" $REMOTE_USER@$REMOTE_HOST:\"$esc_source_file\" \"${INITIATOR[1]}${INITIATOR[3]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID\""
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
eval "$rsync_cmd" 2>> "$LOG_FILE"
if [ $? != 0 ]; then
@ -2383,13 +2384,13 @@ function _SoftDeleteLocal {
fi
if [ $_VERBOSE -eq 1 ]; then
# Cannot launch log function from xargs, ugly hack
$FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete file {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
$COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete file {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
$FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete directory {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
$COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete directory {}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
fi
if [ $_DRYRUN -ne 1 ]; then
$FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} rm -f "{}" && $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} rm -rf "{}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 &
$COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} $COMMAND_SUDO rm -f "{}" && $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} $COMMAND_SUDO rm -rf "{}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 &
else
Dummy &
fi
@ -2417,14 +2418,14 @@ function _SoftDeleteRemote {
CheckConnectivityRemoteHost
if [ $_DRYRUN -eq 1 ]; then
Logger "Listing files older than $change_time days on target replica. Does not remove anything." "NOTICE"
Logger "Listing files older than $change_time days on $replica_type replica. Does not remove anything." "NOTICE"
else
Logger "Removing files older than $change_time days on target replica." "NOTICE"
Logger "Removing files older than $change_time days on $replica_type replica." "NOTICE"
fi
if [ $_VERBOSE -eq 1 ]; then
# Cannot launch log function from xargs, ugly hack
cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} echo Will delete file {} && '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"No remote backup/deletion directory.\"; exit 1; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} echo Will delete file {} && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
@ -2432,7 +2433,7 @@ function _SoftDeleteRemote {
fi
if [ $_DRYRUN -ne 1 ]; then
cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} rm -f \"{}\" && '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} rm -rf \"{}\"; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} '$COMMAND_SUDO' rm -f \"{}\" && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} '$COMMAND_SUDO' rm -rf \"{}\"; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
@ -2442,10 +2443,10 @@ function _SoftDeleteRemote {
WaitForCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]}
retval=$?
if [ $retval -ne 0 ]; then
Logger "Error while executing cleanup on remote target replica." "ERROR"
Logger "Error while executing cleanup on remote $replica_type replica." "ERROR"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
else
Logger "Cleanup complete on target replica." "NOTICE"
Logger "Cleanup complete on $replica_type replica." "NOTICE"
fi
}
@ -2909,5 +2910,4 @@ opts="${opts# *}"
if [ $? == 0 ]; then
SoftDelete
fi
RunAfterHook
fi

Loading…
Cancel
Save