Archives
/
osync
mirror of https://github.com/deajan/osync
2
0
Fork 0
Code Issues Releases Wiki Activity

Added a lot of functionnality

Browse Source
pull/2/head
deajan 11 years ago
parent 1f9b84dae3
commit 8d8b1fab36
3 changed files with 143 additions and 101 deletions
Show Stats Download Patch File Download Diff File

10
CHANGELOG.md
Unescape Escape View File

@ -3,14 +3,16 @@ TODO LIST
! Tag as v1.0 beta ! Tag as v1.0 beta
! verify spaces in sync dir names ! verify spaces in sync dir names
! sudo support for rsync and generic cmds ! exit code line 627
! add osync support in ssh_filter.sh
! minimum space checks
! different rsync executable
RECENT CHANGES RECENT CHANGES
-------------- --------------
- Added minimum disk space checks
- Added osync support in ssh_filter.sh
- Added support for sudo exec on remote slave
- Added support for alternative rsync executable
- Added support for spaces in sync directories names
- Added support for ACL and xattr - Added support for ACL and xattr
- Added --force-unlock parameter to bypass any existing locks on replicas - Added --force-unlock parameter to bypass any existing locks on replicas
- Added full remote support for slave replica - Added full remote support for slave replica

228
osync.sh
Unescape Escape View File

@ -3,10 +3,10 @@
###### Osync - Rsync based two way sync engine with fault tolerance ###### Osync - Rsync based two way sync engine with fault tolerance
###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr) ###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr)
OSYNC_VERSION=0.8 OSYNC_VERSION=0.9
OSYNC_BUILD=2207201301 OSYNC_BUILD=2207201303
DEBUG=no DEBUG=yes
SCRIPT_PID=$$ SCRIPT_PID=$$
LOCAL_USER=$(whoami) LOCAL_USER=$(whoami)
@ -44,14 +44,14 @@ function LogDebug
fi fi
} }
function TrapError function TrapError {
{
local JOB="$0" local JOB="$0"
local LINE="$1" local LINE="$1"
local CODE="${2:-1}" local CODE="${2:-1}"
local CMD="$3"
if [ $silent -eq 0 ] if [ $silent -eq 0 ]
then then
echo " /!\ Error in ${JOB}: Near line ${LINE}, exit code ${CODE}" echo -e " /!\ Error in ${JOB}: Near line ${LINE}, exit code ${CODE}\nCommand $CMD"
fi fi
} }
@ -67,21 +67,36 @@ function TrapStop
if [ $soft_stop -eq 1 ] if [ $soft_stop -eq 1 ]
then then
LogError " /!\ WARNING: CTRL+C hit twice. Quitting osync." LogError " /!\ WARNING: CTRL+C hit twice. Quitting osync. Please wait..."
TrapQuit soft_stop=2
exit 1
fi fi
if [ $soft_stop -eq 2 ]
then
LogError " /!\ WARNING: CTRL+C hit three times. Quitting osync right now without any trap execution."
soft_stop=3
exit
fi
} }
function TrapQuit function TrapQuit
{ {
if [ $soft_stop -eq 3 ]
then
exit 1
fi
if [ $error_alert -ne 0 ] if [ $error_alert -ne 0 ]
then then
SendAlert SendAlert
UnlockDirectories UnlockDirectories
CleanUp
LogError "Osync finished with errros." LogError "Osync finished with errros."
exit 1 exit 1
else else
UnlockDirectories UnlockDirectories
CleanUp
Log "Osync finished." Log "Osync finished."
exit 0 exit 0
fi fi
@ -143,6 +158,7 @@ function CleanUp
rm -f /dev/shm/osync_deletition_on_master_$SCRIPT_PID rm -f /dev/shm/osync_deletition_on_master_$SCRIPT_PID
rm -f /dev/shm/osync_deletition_on_slave_$SCRIPT_PID rm -f /dev/shm/osync_deletition_on_slave_$SCRIPT_PID
rm -f /dev/shm/osync_remote_slave_lock_$SCRIPT_PID rm -f /dev/shm/osync_remote_slave_lock_$SCRIPT_PID
rm -f /dev/shm/osync_slave_space_$SCRIPT_PIDx
fi fi
} }
@ -418,9 +434,9 @@ function CheckConnectivity3rdPartyHosts
function CreateOsyncDirs function CreateOsyncDirs
{ {
if ! [ -d $MASTER_STATE_DIR ] if ! [ -d "$MASTER_STATE_DIR" ]
then then
mkdir --parents $MASTER_STATE_DIR mkdir --parents "$MASTER_STATE_DIR"
if [ $? != 0 ] if [ $? != 0 ]
then then
LogError "Cannot create master replica state dir [$MASTER_STATE_DIR]." LogError "Cannot create master replica state dir [$MASTER_STATE_DIR]."
@ -430,10 +446,10 @@ function CreateOsyncDirs
if [ "$REMOTE_SYNC" == "yes" ] if [ "$REMOTE_SYNC" == "yes" ]
then then
eval "$SSH_CMD \"if ! [ -d $SLAVE_STATE_DIR ]; then mkdir --parents $SLAVE_STATE_DIR; fi\"" & eval "$SSH_CMD \"if ! [ -d $SLAVE_STATE_DIR ]; then $COMMAND_SUDO mkdir --parents $SLAVE_STATE_DIR; fi\"" &
WaitForTaskCompletion $! 0 1800 WaitForTaskCompletion $! 0 1800
else else
if ! [ -d $SLAVE_STATE_DIR ]; then mkdir --parents $SLAVE_STATE_DIR; fi if ! [ -d "$SLAVE_STATE_DIR" ]; then mkdir --parents "$SLAVE_STATE_DIR"; fi
fi fi
if [ $? != 0 ] if [ $? != 0 ]
@ -445,11 +461,11 @@ function CreateOsyncDirs
function CheckMasterSlaveDirs function CheckMasterSlaveDirs
{ {
if ! [ -d $MASTER_SYNC_DIR ] if ! [ -d "$MASTER_SYNC_DIR" ]
then then
if [ "$CREATE_DIRS" == "yes" ] if [ "$CREATE_DIRS" == "yes" ]
then then
mkdir --parents $MASTER_SYNC_DIR mkdir --parents "$MASTER_SYNC_DIR"
if [ $? != 0 ] if [ $? != 0 ]
then then
LogError "Cannot create master directory [$MASTER_SYNC_DIR]." LogError "Cannot create master directory [$MASTER_SYNC_DIR]."
@ -467,7 +483,7 @@ function CheckMasterSlaveDirs
then then
if [ "$CREATE_DIRS" == "yes" ] if [ "$CREATE_DIRS" == "yes" ]
then then
eval "$SSH_CMD \"if ! [ -d $SLAVE_SYNC_DIR ]; then mkdir --parents $SLAVE_SYNC_DIR; fi"\" & eval "$SSH_CMD \"if ! [ -d $SLAVE_SYNC_DIR ]; then $COMMAND_SUDO mkdir --parents $SLAVE_SYNC_DIR; fi"\" &
WaitForTaskCompletion $! 0 1800 WaitForTaskCompletion $! 0 1800
if [ $? != 0 ] if [ $? != 0 ]
then then
@ -484,11 +500,11 @@ function CheckMasterSlaveDirs
fi fi
fi fi
else else
if [ ! -d $SLAVE_SYNC_DIR ] if [ ! -d "$SLAVE_SYNC_DIR" ]
then then
if [ "$CREATE_DIRS" == "yes" ] if [ "$CREATE_DIRS" == "yes" ]
then then
mkdir --parents $SLAVE_SYNC_DIR mkdir --parents "$SLAVE_SYNC_DIR"
if [ $? != 0 ] if [ $? != 0 ]
then then
LogError "Cannot create slave directory [$SLAVE_SYNC_DIR]." LogError "Cannot create slave directory [$SLAVE_SYNC_DIR]."
@ -504,6 +520,31 @@ function CheckMasterSlaveDirs
fi fi
} }
function CheckMinimumSpace
{
Log "Checking minimum disk space on master and slave."
MASTER_SPACE=$(df -P "$MASTER_SYNC_DIR" | tail -1 | awk '{print $4}')
if [ $MASTER_SPACE -lt $MINIMUM_SPACE ]
then
LogError "There is not enough free space on master [$MASTER_SPACE KB]."
fi
if [ "$REMOTE_SYNC" == "yes" ]
then
eval "$SSH_CMD \"$COMMAND_SUDO df -P $SLAVE_SYNC_DIR\"" > /dev/shm/osync_slave_space_$SCRIPT_PID &
WaitForTaskCompletion $! 0 1800
SLAVE_SPACE=$(cat /dev/shm/osync_slave_space_$SCRIPT_PID | tail -1 | awk '{print $4}')
else
SLAVE_SPACE=$(df -P "$SLAVE_SYNC_DIR" | tail -1 | awk '{print $4}')
fi
if [ $SLAVE_SPACE -lt $MINIMUM_SPACE ]
then
LogError "There is not enough free space on slave [$SLAVE_SPACE KB]."
fi
}
function RsyncExcludePattern function RsyncExcludePattern
{ {
OLD_IFS=$IFS OLD_IFS=$IFS
@ -522,7 +563,7 @@ function RsyncExcludePattern
function WriteLockFiles function WriteLockFiles
{ {
echo $SCRIPT_PID > $MASTER_STATE_DIR/lock echo $SCRIPT_PID > "$MASTER_STATE_DIR/lock"
if [ $? != 0 ] if [ $? != 0 ]
then then
LogError "Could not set lock on master replica." LogError "Could not set lock on master replica."
@ -533,7 +574,7 @@ function WriteLockFiles
if [ "$REMOTE_SYNC" == "yes" ] if [ "$REMOTE_SYNC" == "yes" ]
then then
eval "$SSH_CMD \"echo $SCRIPT_PID@$SYNC_ID > $SLAVE_STATE_DIR/lock\"" & eval "$SSH_CMD \"$COMMAND_SUDO echo $SCRIPT_PID@$SYNC_ID > $SLAVE_STATE_DIR/lock\"" &
WaitForTaskCompletion $! 0 1800 WaitForTaskCompletion $! 0 1800
if [ $? != 0 ] if [ $? != 0 ]
then then
@ -543,7 +584,7 @@ function WriteLockFiles
Log "Locked remote slave replica." Log "Locked remote slave replica."
fi fi
else else
echo "$SCRIPT_PID@$SYNC_ID" > $SLAVE_STATE_DIR/lock echo "$SCRIPT_PID@$SYNC_ID" > "$SLAVE_STATE_DIR/lock"
if [ $? != 0 ] if [ $? != 0 ]
then then
LogError "Couuld not set lock on local slave replica." LogError "Couuld not set lock on local slave replica."
@ -567,7 +608,7 @@ function LockDirectories
Log "Checking for replica locks." Log "Checking for replica locks."
if [ -f $MASTER_STATE_DIR/lock ] if [ -f "$MASTER_STATE_DIR/lock" ]
then then
master_lock_pid=$(cat $MASTER_STATE_DIR/lock) master_lock_pid=$(cat $MASTER_STATE_DIR/lock)
LogDebug "Master lock pid: $master_lock_pid" LogDebug "Master lock pid: $master_lock_pid"
@ -588,10 +629,10 @@ function LockDirectories
slave_lock_pid=$(cat /dev/shm/osync_remote_slave_lock_$SCRIPT_PID | cut -d'@' -f1) slave_lock_pid=$(cat /dev/shm/osync_remote_slave_lock_$SCRIPT_PID | cut -d'@' -f1)
slave_lock_id=$(cat /dev/shm/osync_remote_slave_lock_$SCRIPT_PID | cut -d'@' -f2) slave_lock_id=$(cat /dev/shm/osync_remote_slave_lock_$SCRIPT_PID | cut -d'@' -f2)
else else
if [ -f $SLAVE_STATE_DIR/lock ] if [ -f "$SLAVE_STATE_DIR/lock" ]
then then
slave_lock_pid=$(cat $SLAVE_STATE_DIR/lock | cut -d'@' -f1) slave_lock_pid=$(cat "$SLAVE_STATE_DIR/lock" | cut -d'@' -f1)
slave_lock_id=$(cat $SLAVE_STATE_DIR/lock | cut -d'@' -f2) slave_lock_id=$(cat "$SLAVE_STATE_DIR/lock" | cut -d'@' -f2)
fi fi
fi fi
@ -628,10 +669,10 @@ function UnlockDirectories
{ {
if [ "$REMOTE_SYNC" == "yes" ] if [ "$REMOTE_SYNC" == "yes" ]
then then
eval "$SSH_CMD \"if [ -f $SLAVE_STATE_DIR/lock ]; then rm $SLAVE_STATE_DIR/lock; fi\"" & eval "$SSH_CMD \"if [ -f $SLAVE_STATE_DIR/lock ]; then $COMMAND_SUDO rm $SLAVE_STATE_DIR/lock; fi\"" &
WaitForTaskCompletion $! 0 1800 WaitForTaskCompletion $! 0 1800
else else
if [ -f $SLAVE_STATE_DIR/lock ];then rm $SLAVE_STATE_DIR/lock; fi if [ -f "$SLAVE_STATE_DIR/lock" ];then rm "$SLAVE_STATE_DIR/lock"; fi
fi fi
if [ $? != 0 ] if [ $? != 0 ]
@ -641,9 +682,9 @@ function UnlockDirectories
Log "Removed slave replica lock." Log "Removed slave replica lock."
fi fi
if [ -f $MASTER_STATE_DIR/lock ] if [ -f "$MASTER_STATE_DIR/lock" ]
then then
rm $MASTER_STATE_DIR/lock rm "$MASTER_STATE_DIR/lock"
if [ $? != 0 ] if [ $? != 0 ]
then then
LogError "Could not unlock master replica." LogError "Could not unlock master replica."
@ -658,17 +699,16 @@ function UnlockDirectories
function master_tree_current function master_tree_current
{ {
Log "Creating master replica file list." Log "Creating master replica file list."
rsync_cmd="$(which $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -rlptgodE --exclude \"$OSYNC_DIR\" --list-only \"$MASTER_SYNC_DIR/\" | grep \"^-\|^d\" | awk '{print $5}' | (grep -v \"^\.$\" || :) > /dev/shm/osync_master-tree-current_$SCRIPT_PID &" $(which $RSYNC_EXECUTABLE) --rsync-path="$RSYNC_PATH" -rlptgodE --exclude "$OSYNC_DIR" --list-only "$MASTER_SYNC_DIR/" | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_master-tree-current_$SCRIPT_PID &
eval $rsync_cmd
child_pid=$! child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
if [ $? == 0 ] && [ -f /dev/shm/osync_master-tree-current_$SCRIPT_PID ] if [ $? == 0 ] && [ -f /dev/shm/osync_master-tree-current_$SCRIPT_PID ]
then then
mv /dev/shm/osync_master-tree-current_$SCRIPT_PID $MASTER_STATE_DIR/master-tree-current mv /dev/shm/osync_master-tree-current_$SCRIPT_PID "$MASTER_STATE_DIR/master-tree-current"
echo "master-replica-tree.success" > $MASTER_STATE_DIR/last-action echo "master-replica-tree.success" > "$MASTER_STATE_DIR/last-action"
else else
LogError "Cannot create master file list." LogError "Cannot create master file list."
echo "master-replica-tree.fail" > $MASTER_STATE_DIR/last-action echo "master-replica-tree.fail" > "$MASTER_STATE_DIR/last-action"
exit 1 exit 1
fi fi
} }
@ -678,20 +718,20 @@ function slave_tree_current
Log "Creating slave replica file list." Log "Creating slave replica file list."
if [ "$REMOTE_SYNC" == "yes" ] if [ "$REMOTE_SYNC" == "yes" ]
then then
$(which $RSYNC_EXECUTABLE) -rlptgodE --exclude "$OSYNC_DIR" -e "$RSYNC_SSH_CMD" --list-only $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-current_$SCRIPT_PID & $(which $RSYNC_EXECUTABLE) --rsync-path="$RSYNC_PATH" -rlptgodE --exclude "$OSYNC_DIR" -e "$RSYNC_SSH_CMD" --list-only "$REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/" | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-current_$SCRIPT_PID &
else else
$(which $RSYNC_EXECUTABLe) -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-current_$SCRIPT_PID & $(which $RSYNC_EXECUTABLE) --rsync-path="$RSYNC_PATH" -rlptgodE --exclude "$OSYNC_DIR" --list-only "$SLAVE_SYNC_DIR/" | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-current_$SCRIPT_PID &
fi fi
child_pid=$! child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
retval=$? retval=$?
if [ $retval == 0 ] && [ -f /dev/shm/osync_slave-tree-current_$SCRIPT_PID ] if [ $retval == 0 ] && [ -f /dev/shm/osync_slave-tree-current_$SCRIPT_PID ]
then then
mv /dev/shm/osync_slave-tree-current_$SCRIPT_PID $MASTER_STATE_DIR/slave-tree-current mv /dev/shm/osync_slave-tree-current_$SCRIPT_PID "$MASTER_STATE_DIR/slave-tree-current"
echo "slave-replica-tree.-success" > $MASTER_STATE_DIR/last-action echo "slave-replica-tree.-success" > "$MASTER_STATE_DIR/last-action"
else else
LogError "Cannot create slave file list." LogError "Cannot create slave file list."
echo "slave-replica-tree.fail" > $MASTER_STATE_DIR/last-action echo "slave-replica-tree.fail" > "$MASTER_STATE_DIR/last-action"
exit 1 exit 1
fi fi
} }
@ -699,26 +739,26 @@ function slave_tree_current
function master_delete_list function master_delete_list
{ {
Log "Creating master replica deleted file list." Log "Creating master replica deleted file list."
if [ -f $MASTER_STATE_DIR/master-tree-after ] if [ -f "$MASTER_STATE_DIR/master-tree-after" ]
then then
comm --nocheck-order -23 $MASTER_STATE_DIR/master-tree-after $MASTER_STATE_DIR/master-tree-current > $MASTER_STATE_DIR/master-deleted-list comm --nocheck-order -23 "$MASTER_STATE_DIR/master-tree-after" "$MASTER_STATE_DIR/master-tree-current" > "$MASTER_STATE_DIR/master-deleted-list"
echo "master-replica-deleted-list.success" > $MASTER_STATE_DIR/last-action echo "master-replica-deleted-list.success" > "$MASTER_STATE_DIR/last-action"
else else
touch $MASTER_STATE_DIR/master-deleted-list touch "$MASTER_STATE_DIR/master-deleted-list"
echo "master-replica-deleted-list.empty" > $MASTER_STATE_DIR/last-action echo "master-replica-deleted-list.empty" > "$MASTER_STATE_DIR/last-action"
fi fi
} }
function slave_delete_list function slave_delete_list
{ {
Log "Creating slave replica deleted file list." Log "Creating slave replica deleted file list."
if [ -f $MASTER_STATE_DIR/slave-tree-after ] if [ -f "$MASTER_STATE_DIR/slave-tree-after" ]
then then
comm --nocheck-order -23 $MASTER_STATE_DIR/slave-tree-after $MASTER_STATE_DIR/slave-tree-current > $MASTER_STATE_DIR/slave-deleted-list comm --nocheck-order -23 "$MASTER_STATE_DIR/slave-tree-after" "$MASTER_STATE_DIR/slave-tree-current" > "$MASTER_STATE_DIR/slave-deleted-list"
echo "slave-replica-deleted-list.success" > $MASTER_STATE_DIR/last-action echo "slave-replica-deleted-list.success" > "$MASTER_STATE_DIR/last-action"
else else
touch $MASTER_STATE_DIR/slave-deleted-list touch "$MASTER_STATE_DIR/slave-deleted-list"
echo "slave-replica-deleted-list.empty" > $MASTER_STATE_DIR/last-action echo "slave-replica-deleted-list.empty" > "$MASTER_STATE_DIR/last-action"
fi fi
} }
@ -727,9 +767,9 @@ function sync_update_slave
Log "Updating slave replica." Log "Updating slave replica."
if [ "$REMOTE_SYNC" == "yes" ] if [ "$REMOTE_SYNC" == "yes" ]
then then
rsync $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $SLAVE_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 & $(which $RSYNC_EXECUTABLE) --rsync-path="$RSYNC_PATH" $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $SLAVE_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" "$MASTER_SYNC_DIR/" "$REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/" > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 &
else else
rsync $RSYNC_ARGS -rlptgodEui $SLAVE_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 & $(which $RSYNC_EXECUTABLE) --rsync-path="$RSYNC_PATH" -rlptgodEui $SLAVE_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" "$MASTER_SYNC_DIR/" "$SLAVE_SYNC_DIR/" > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 &
fi fi
child_pid=$! child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
@ -742,11 +782,11 @@ function sync_update_slave
if [ $retval != 0 ] if [ $retval != 0 ]
then then
LogError "Updating slave replica failed. Stopping execution." LogError "Updating slave replica failed. Stopping execution."
echo "update-slave-replica.fail" > $MASTER_STATE_DIR/last-action echo "update-slave-replica.fail" > "$MASTER_STATE_DIR/last-action"
exit 1 exit 1
else else
Log "Updating slave replica succeded." Log "Updating slave replica succeded."
echo "update-slave-replica.success" > $MASTER_STATE_DIR/last-action echo "update-slave-replica.success" > "$MASTER_STATE_DIR/last-action"
fi fi
} }
@ -755,9 +795,9 @@ function sync_update_master
Log "Updating master replica." Log "Updating master replica."
if [ "$REMOTE_SYNC" == "yes" ] if [ "$REMOTE_SYNC" == "yes" ]
then then
rsync $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $MASTER_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --exclude-from "$MASTER_STATE_DIR/master-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 & $(which $RSYNC_EXECUTABLE) --rsync-path="$RSYNC_PATH" $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $MASTER_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --exclude-from "$MASTER_STATE_DIR/master-deleted-list" "$REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/" "$MASTER_SYNC_DIR/" > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 &
else else
rsync $RSYNC_ARGS -rlptgodEui $MASTER_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --exclude-from "$MASTER_STATE_DIR/master-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 & $(which $RSYNC_EXECUTABLE) --rsync-path="$RSYNC_PATH" $RSYNC_ARGS -rlptgodEui $MASTER_BACKUP --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --exclude-from "$MASTER_STATE_DIR/master-deleted-list" "$SLAVE_SYNC_DIR/" "$MASTER_SYNC_DIR/" > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 &
fi fi
child_pid=$! child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
@ -770,11 +810,11 @@ function sync_update_master
if [ $retval != 0 ] if [ $retval != 0 ]
then then
LogError "Updating master replica failed. Stopping execution." LogError "Updating master replica failed. Stopping execution."
echo "update-master-replica.fail" > $MASTER_STATE_DIR/last-action echo "update-master-replica.fail" > "$MASTER_STATE_DIR/last-action"
exit 1 exit 1
else else
Log "Updating master replica succeded." Log "Updating master replica succeded."
echo "update-master-replica.success" > $MASTER_STATE_DIR/last-action echo "update-master-replica.success" > "$MASTER_STATE_DIR/last-action"
fi fi
} }
@ -783,9 +823,9 @@ function delete_on_slave
Log "Propagating deletitions to slave replica." Log "Propagating deletitions to slave replica."
if [ "$REMOTE_SYNC" == "yes" ] if [ "$REMOTE_SYNC" == "yes" ]
then then
rsync $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $SLAVE_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --include-from "$MASTER_STATE_DIR/master-deleted-list" $MASTER_SYNC_DIR/ $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ > /dev/shm/osync_deletition_on_slave_$SCRIPT_PID 2>&1 & $(which $RSYNC_EXECUTABLE) --rsync-path="$RSYNC_PATH" $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $SLAVE_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --include-from "$MASTER_STATE_DIR/master-deleted-list" "$MASTER_SYNC_DIR/" "$REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/" > /dev/shm/osync_deletition_on_slave_$SCRIPT_PID 2>&1 &
else else
rsync $RSYNC_ARGS -rlptgodEui $SLAVE_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --include-from "$MASTER_STATE_DIR/master-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ > /dev/shm/osync_deletition_on_slave_$SCRIPT_PID 2>&1 & $(which $RSYNC_EXECUTABLE) --rsync-path="$RSYNC_PATH" $RSYNC_ARGS -rlptgodEui $SLAVE_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/slave-deleted-list" --include-from "$MASTER_STATE_DIR/master-deleted-list" "$MASTER_SYNC_DIR/" "$SLAVE_SYNC_DIR/" > /dev/shm/osync_deletition_on_slave_$SCRIPT_PID 2>&1 &
fi fi
child_pid=$! child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0 WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
@ -798,10 +838,10 @@ function delete_on_slave
if [ $retval != 0 ] if [ $retval != 0 ]
then then
LogError "Deletition on slave failed." LogError "Deletition on slave failed."
echo "delete-propagation-slave.fail" > $MASTER_STATE_DIR/last-action echo "delete-propagation-slave.fail" > "$MASTER_STATE_DIR/last-action"
exit 1 exit 1
else else
echo "delete-propagation-slave.success" > $MASTER_STATE_DIR/last-action echo "delete-propagation-slave.success" > "$MASTER_STATE_DIR/last-action"
fi fi
} }
@ -810,9 +850,9 @@ function delete_on_master
Log "Propagating deletitions to master replica." Log "Propagating deletitions to master replica."
if [ "$REMOTE_SYNC" == "yes" ] if [ "$REMOTE_SYNC" == "yes" ]
then then
rsync $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $MASTER_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --include-from "$MASTER_STATE_DIR/slave-deleted-list" $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_deletition_on_master_$SCRIPT_PID 2>&1 & $(which $RSYNC_EXECUTABLE) --rsync-path="$RSYNC_PATH" $RSYNC_ARGS -rlptgodEui -e "$RSYNC_SSH_CMD" $MASTER_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --include-from "$MASTER_STATE_DIR/slave-deleted-list" "$REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/" "$MASTER_SYNC_DIR/" > /dev/shm/osync_deletition_on_master_$SCRIPT_PID 2>&1 &
else else
rsync $RSYNC_ARGS -rlptgodEui $MASTER_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --include-from "$MASTER_STATE_DIR/slave-deleted-list" $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_deletition_on_master_$SCRIPT_PID 2>&1 & $(which $RSYNC_EXECUTABLE) --rsync-path="$RSYNC_PATH" $RSYNC_ARGS -rlptgodEui $MASTER_DELETE --delete --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --exclude-from "$MASTER_STATE_DIR/master-deleted-list" --include-from "$MASTER_STATE_DIR/slave-deleted-list" "$SLAVE_SYNC_DIR/" "$MASTER_SYNC_DIR/" > /dev/shm/osync_deletition_on_master_$SCRIPT_PID 2>&1 &
fi fi
child_pid=$! child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0 WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
@ -825,27 +865,27 @@ function delete_on_master
if [ $retval != 0 ] if [ $retval != 0 ]
then then
LogError "Deletition on master failed." LogError "Deletition on master failed."
echo "delete-propagation-master.fail" > $MASTER_STATE_DIR/last-action echo "delete-propagation-master.fail" > "$MASTER_STATE_DIR/last-action"
exit 1 exit 1
else else
echo "delete-propagation-master.success" > $MASTER_STATE_DIR/last-action echo "delete-propagation-master.success" > "$MASTER_STATE_DIR/last-action"
fi fi
} }
function master_tree_after function master_tree_after
{ {
Log "Creating after run master replica file list." Log "Creating after run master replica file list."
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $MASTER_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :)> /dev/shm/osync_master-tree-after_$SCRIPT_PID & $(which $RSYNC_EXECUTABLE) --rsync-path="$RSYNC_PATH" -rlptgodE --exclude "$OSYNC_DIR" --list-only "$MASTER_SYNC_DIR/" | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :)> /dev/shm/osync_master-tree-after_$SCRIPT_PID &
child_pid=$! child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0 WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
retval=$? retval=$?
if [ $retval == 0 ] && [ -f /dev/shm/osync_master-tree-after_$SCRIPT_PID ] if [ $retval == 0 ] && [ -f /dev/shm/osync_master-tree-after_$SCRIPT_PID ]
then then
mv /dev/shm/osync_master-tree-after_$SCRIPT_PID $MASTER_STATE_DIR/master-tree-after mv /dev/shm/osync_master-tree-after_$SCRIPT_PID "$MASTER_STATE_DIR/master-tree-after"
echo "master-replica-tree-after.success" > $MASTER_STATE_DIR/last-action echo "master-replica-tree-after.success" > "$MASTER_STATE_DIR/last-action"
else else
LogError "Cannot create slave file list." LogError "Cannot create slave file list."
echo "master-replica-tree-after.fail" > $MASTER_STATE_DIR/last-action echo "master-replica-tree-after.fail" > "$MASTER_STATE_DIR/last-action"
exit 1 exit 1
fi fi
} }
@ -855,20 +895,20 @@ function slave_tree_after
Log "Creating after run slave replica file list." Log "Creating after run slave replica file list."
if [ "$REMOTE_SYNC" == "yes" ] if [ "$REMOTE_SYNC" == "yes" ]
then then
rsync -rlptgodE -e "$RSYNC_SSH_CMD" --exclude "$OSYNC_DIR" --list-only $REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-after_$SCRIPT_PID & $(which $RSYNC_EXECUTABLE) --rsync-path="$RSYNC_PATH" -rlptgodE -e "$RSYNC_SSH_CMD" --exclude "$OSYNC_DIR" --list-only "$REMOTE_USER@$REMOTE_HOST:$SLAVE_SYNC_DIR/" | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-after_$SCRIPT_PID &
else else
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-after_$SCRIPT_PID & $(which $RSYNC_EXECUTABLE) --rsync-path="$RSYNC_PATH" -rlptgodE --exclude "$OSYNC_DIR" --list-only "$SLAVE_SYNC_DIR/" | grep "^-\|^d" | awk '{print $5}' | (grep -v "^\.$" || :) > /dev/shm/osync_slave-tree-after_$SCRIPT_PID &
fi fi
child_pid=$! child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0 WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
retval=$? retval=$?
if [ $retval == 0 ] && [ -f /dev/shm/osync_slave-tree-after_$SCRIPT_PID ] if [ $retval == 0 ] && [ -f /dev/shm/osync_slave-tree-after_$SCRIPT_PID ]
then then
mv /dev/shm/osync_slave-tree-after_$SCRIPT_PID $MASTER_STATE_DIR/slave-tree-after mv /dev/shm/osync_slave-tree-after_$SCRIPT_PID "$MASTER_STATE_DIR/slave-tree-after"
echo "slave-replica-tree-after.success" > $MASTER_STATE_DIR/last-action echo "slave-replica-tree-after.success" > "$MASTER_STATE_DIR/last-action"
else else
LogError "Cannot create slave file list." LogError "Cannot create slave file list."
echo "slave-replica-tree-after.fail" > $MASTER_STATE_DIR/last-action echo "slave-replica-tree-after.fail" > "$MASTER_STATE_DIR/last-action"
exit 1 exit 1
fi fi
} }
@ -878,12 +918,12 @@ function Sync
{ {
Log "Starting synchronization task." Log "Starting synchronization task."
if [ -f $MASTER_STATE_DIR/last-action ] && [ "$RESUME_SYNC" == "yes" ] if [ -f "$MASTER_STATE_DIR/last-action" ] && [ "$RESUME_SYNC" == "yes" ]
then then
resume_sync=$(cat $MASTER_STATE_DIR/last-action) resume_sync=$(cat "$MASTER_STATE_DIR/last-action")
if [ -f $MASTER_STATE_DIR/resume-count ] if [ -f "$MASTER_STATE_DIR/resume-count" ]
then then
resume_count=$(cat $MASTER_STATE_DIR/resume-count) resume_count=$(cat "$MASTER_STATE_DIR/resume-count")
else else
resume_count=0 resume_count=0
fi fi
@ -893,14 +933,14 @@ function Sync
if [ "$resume_sync" != "sync.success" ] if [ "$resume_sync" != "sync.success" ]
then then
Log "WARNING: Trying to resume aborted osync execution on $(stat --format %y $MASTER_STATE_DIR/last-action) at task [$resume_sync]." Log "WARNING: Trying to resume aborted osync execution on $(stat --format %y $MASTER_STATE_DIR/last-action) at task [$resume_sync]."
echo $(($resume_count+1)) > $MASTER_STATE_DIR/resume-count echo $(($resume_count+1)) > "$MASTER_STATE_DIR/resume-count"
else else
resume_sync=none resume_sync=none
fi fi
else else
Log "Will not resume aborted osync execution. Too much resume tries [$resume_count]." Log "Will not resume aborted osync execution. Too much resume tries [$resume_count]."
echo "noresume" > $MASTER_STATE_DIR/last-action echo "noresume" > "$MASTER_STATE_DIR/last-action"
echo "0" > $MASTER_STATE_DIR/resume-count echo "0" > "$MASTER_STATE_DIR/resume-count"
resume_sync=none resume_sync=none
fi fi
else else
@ -964,50 +1004,50 @@ function Sync
esac esac
Log "Finished synchronization task." Log "Finished synchronization task."
echo "sync.success" > $MASTER_STATE_DIR/last-action echo "sync.success" > "$MASTER_STATE_DIR/last-action"
echo "0" > $MASTER_STATE_DIR/resume-count echo "0" > "$MASTER_STATE_DIR/resume-count"
} }
function SoftDelete function SoftDelete
{ {
if [ "$CONFLICT_BACKUP" != "no" ] if [ "$CONFLICT_BACKUP" != "no" ]
then then
if [ -d $MASTER_BACKUP_DIR ] if [ -d "$MASTER_BACKUP_DIR" ]
then then
Log "Removing backups older than $CONFLICT_BACKUP_DAYS days on master replica." Log "Removing backups older than $CONFLICT_BACKUP_DAYS days on master replica."
find $MASTER_BACKUP_DIR/ -ctime +$CONFLICT_BACKUP_DAYS | xargs rm -rf find "$MASTER_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS | xargs rm -rf
fi fi
if [ "$REMOTE_SYNC" == "yes" ] if [ "$REMOTE_SYNC" == "yes" ]
then then
Log "Removing backups older than $CONFLICT_BACKUP_DAYS days on remote slave replica." Log "Removing backups older than $CONFLICT_BACKUP_DAYS days on remote slave replica."
eval "$SSH_CMD \"if [ -d $SLAVE_BACKUP_DIR ]; then find $SLAVE_BACKUP_DIR/ -ctime +$CONFLICT_BACKUP_DAYS | xargs rm -rf; fi\"" eval "$SSH_CMD \"if [ -d \"$SLAVE_BACKUP_DIR\" ]; then $COMMAND_SUDO find \"$SLAVE_BACKUP_DIR/\" -ctime +$CONFLICT_BACKUP_DAYS | xargs rm -rf; fi\""
else else
if [ -d $SLAVE_BACKUP_DIR ] if [ -d "$SLAVE_BACKUP_DIR" ]
then then
Log "Removing backups older than $CONFLICT_BACKUP_DAYS days on slave replica." Log "Removing backups older than $CONFLICT_BACKUP_DAYS days on slave replica."
find $SLAVE_BACKUP_DIR/ -ctime +$CONFLICT_BACKUP_DAYS | xargs rm -rf find "$SLAVE_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS | xargs rm -rf
fi fi
fi fi
fi fi
if [ "$SOFT_DELETE" != "no" ] if [ "$SOFT_DELETE" != "no" ]
then then
if [ -d $MASTER_DELETE_DIR ] if [ -d "$MASTER_DELETE_DIR" ]
then then
Log "Removing soft deleted items older than $SOFT_DELETE_DAYS days on master replica." Log "Removing soft deleted items older than $SOFT_DELETE_DAYS days on master replica."
find $MASTER_DELETE_DIR/ -ctime +$SOFT_DELETE_DAYS | xargs rm -rf find "$MASTER_DELETE_DIR/" -ctime +$SOFT_DELETE_DAYS | xargs rm -rf
fi fi
if [ "$REMOTE_SYNC" == "yes" ] if [ "$REMOTE_SYNC" == "yes" ]
then then
Log "Removing soft deleted items older than $SOFT_DELETE_DAYS days on remote slave replica." Log "Removing soft deleted items older than $SOFT_DELETE_DAYS days on remote slave replica."
eval "$SSH_CMD \"if [ -d $SLAVE_DELETE_DIR ]; then find $SLAVE_DELETE_DIR/ -ctime +$SOFT_DELETE_DAYS | xargs rm -rf; fi\"" eval "$SSH_CMD \"if [ -d \"$SLAVE_DELETE_DIR\" ]; then $COMMAND_SUDO find \"$SLAVE_DELETE_DIR/\" -ctime +$SOFT_DELETE_DAYS | xargs rm -rf; fi\""
else else
if [ -d $SLAVE_DELETE_DIR ] if [ -d "$SLAVE_DELETE_DIR" ]
then then
Log "Removing soft deleted items older than $SOFT_DELETE_DAYS days on slave replica." Log "Removing soft deleted items older than $SOFT_DELETE_DAYS days on slave replica."
find $SLAVE_DELETE_DIR/ -ctime +$SOFT_DELETE_DAYS | xargs rm -rf find "$SLAVE_DELETE_DIR/" -ctime +$SOFT_DELETE_DAYS | xargs rm -rf
fi fi
fi fi
fi fi
@ -1023,7 +1063,7 @@ function Init
trap TrapQuit EXIT trap TrapQuit EXIT
if [ "$DEBUG" == "yes" ] if [ "$DEBUG" == "yes" ]
then then
trap 'TrapError ${LINENO} $?' ERR trap 'TrapError ${LINENO} $? $BASH_COMMAND' ERR
fi fi
LOG_FILE=/var/log/osync_$OSYNC_VERSION-$SYNC_ID.log LOG_FILE=/var/log/osync_$OSYNC_VERSION-$SYNC_ID.log
@ -1198,6 +1238,7 @@ then
Log " $DRY_WARNING $DATE - Osync v$OSYNC_VERSION script begin." Log " $DRY_WARNING $DATE - Osync v$OSYNC_VERSION script begin."
Log "-------------------------------------------------------------" Log "-------------------------------------------------------------"
CheckMasterSlaveDirs CheckMasterSlaveDirs
CheckMinimumSpace
if [ $? == 0 ] if [ $? == 0 ]
then then
RunBeforeHook RunBeforeHook
@ -1207,7 +1248,6 @@ then
SoftDelete SoftDelete
fi fi
RunAfterHook RunAfterHook
CleanUp
fi fi
else else
LogError "Configuration file could not be loaded." LogError "Configuration file could not be loaded."

6
sync.conf
Unescape Escape View File

@ -2,7 +2,7 @@
###### Osync - Rsync based two way sync engine with fault tolerance ###### Osync - Rsync based two way sync engine with fault tolerance
###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr) ###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr)
#### Config file rev 2207201301 #### Config file rev 2207201302
## Sync job identification, any string you want, no spaces ## Sync job identification, any string you want, no spaces
SYNC_ID="sync_test" SYNC_ID="sync_test"
@ -20,7 +20,7 @@ RSYNC_EXCLUDE_PATTERN="nosyncdir;otherdir"
PATH_SEPARATOR_CHAR=";" PATH_SEPARATOR_CHAR=";"
## Generate an alert if master or slave have lass space than given value in KB. ## Generate an alert if master or slave have lass space than given value in KB.
MINIMUM_SPACE=1048576 MINIMUM_SPACE=10240
## If enabled, synchronization will be processed with sudo command. See documentation ## If enabled, synchronization will be processed with sudo command. See documentation
SUDO_EXEC=yes SUDO_EXEC=yes
@ -83,7 +83,7 @@ LOCAL_RUN_AFTER_CMD=""
REMOTE_RUN_BEFORE_CMD="du /var/log" REMOTE_RUN_BEFORE_CMD="du /var/log"
REMOTE_RUN_AFTER_CMD="du /tmp" REMOTE_RUN_AFTER_CMD="du /tmp"
## Maximum execution time for commands before sync task. Commands get killed if not finished after MAX_EXC_TIMEe. Set this to 0 to disable killing. ## Maximum execution time for commands before sync task. Commands get killed if not finished after MAX_EXC_TIME. Set this to 0 to disable killing.
MAX_EXEC_TIME_PER_CMD_BEFORE=0 MAX_EXEC_TIME_PER_CMD_BEFORE=0
## Maximum execution time for commands after sync task. Commands get killed if not finished after MAX_EXEC_TIME. Set this to 0 to disable killing command. ## Maximum execution time for commands after sync task. Commands get killed if not finished after MAX_EXEC_TIME. Set this to 0 to disable killing command.
MAX_EXEC_TIME_PER_CMD_AFTER=0 MAX_EXEC_TIME_PER_CMD_AFTER=0

Write Preview
Loading…
Cancel
Save