Archives
/
osync
mirror of https://github.com/deajan/osync
2
0
Fork 0
Code Issues Releases Wiki Activity

A whole bunch of functionnality added.

Browse Source
pull/2/head
deajan 11 years ago
parent fab0ae5a03
commit 4aca92e62c
2 changed files with 162 additions and 25 deletions
Show Stats Download Patch File Download Diff File

179
osync.sh
Unescape Escape View File

@ -5,15 +5,13 @@
## todo:
# add dryrun
# add dryrun, DOC: never run dry when already run real or you will lose your deleted file history
# add logging
# add resume on error
# add master/slave prevalance
# thread sync function
# remote functionnality
OSYNC_VERSION=0.4
OSYNC_BUILD=1607201302
OSYNC_BUILD=1607201305
DEBUG=no
SCRIPT_PID=$$
@ -32,10 +30,10 @@ KEEP_LOGGING=1801
function Log
{
echo "TIME: $SECONDS - $1" >> "$LOG_FILE"
echo -e "TIME: $SECONDS - $1" >> "$LOG_FILE"
if [ $silent -eq 0 ]
then
echo "TIME: $SECONDS - $1"
echo -e "TIME: $SECONDS - $1"
fi
}
@ -56,6 +54,23 @@ function TrapError
fi
}
function TrapStop
{
LogError " /!\ WARNING: Manual exit of osync is really not recommended. Sync will be in inconsistent state."
LogError " /!\ WARNING: If you are sure, please hit CTRL+C another time to quit."
if [ $soft_stop -eq 0 ]
then
soft_stop=1
fi
if [ "$DEBUG" == "no" ] && [ $soft_stop -eq 1 ]
then
CleanUp
exit 1
fi
}
function Spinner
{
if [ $silent -eq 1 ]
@ -96,6 +111,14 @@ function CleanUp
rm -f /dev/shm/osync_config_$SCRIPT_PID
rm -f /dev/shm/osync_run_local_$SCRIPT_PID
rm -f /dev/shm/osync_run_remote_$SCRIPT_PID
rm -f /dev/shm/osync_master-tree-current_$SCRIPT_PID
rm -f /dev/shm/osync_slave-tree-current_$SCRIPT_PID
rm -f /dev/shm/osync_master-tree-before_$SCRIPT_PID
rm -f /dev/shm/osync_slave-tree-before_$SCRIPT_PID
rm -f /dev/shm/osync_update_master_replica_$SCRIPT_PID
rm -f /dev/shm/osync_update_slave_replica_$SCRIPT_PID
rm -f /dev/shm/osync_deletition_on_master_$SCRIPT_PID
rm -f /dev/shm/osync_deletition_on_slave_$SCRIPT_PID
}
function SendAlert
@ -381,14 +404,44 @@ function LockSlave
function sync_update_master
{
Log "Updating slave replica."
rsync -rlptgodEui --backup --backup-dir "$MASTER_BACKUP_DIR" --exclude "$OSYNC_DIR" --exclude-from "$STATE_DIR/master-deleted-list" --exclude-from "$STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/
rsync $DRY_OPTION -rlptgodEui --backup --backup-dir "$SLAVE_BACKUP_DIR" --exclude "$OSYNC_DIR" --exclude-from "$STATE_DIR/master-deleted-list" --exclude-from "$STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ > /dev/shm/osync_update_slave_replica_$SCRIPT_PID 2>&1 &
child_pid=$!
WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME 0
retval=$?
if [ $retval != 0 ]
then
LogError "Updating slave replica failed. Stopping execution."
else
Log "Updating slave replica succeded."
fi
if [ "$VERBOSE_LOGS" == "yes" ]
then
Log "List:\n$(cat /dev/shm/osync_update_slave_replica_$SCRIPT_PID)"
fi
return $retval
}
# Subfunction of Sync
function sync_update_slave
{
Log "Updating master replica."
rsync -rlptgodEui --backup --backup-dir "$SLAVE_BACKUP_DIR" --exclude "$OSYNC_DIR" --exclude-from "$STATE_DIR/slave-deleted-list" --exclude-from "$STATE_DIR/master-deleted-list" $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/
rsync $DRY_OPTION -rlptgodEui --backup --backup-dir "$MASTER_BACKUP_DIR" --exclude "$OSYNC_DIR" --exclude-from "$STATE_DIR/slave-deleted-list" --exclude-from "$STATE_DIR/master-deleted-list" $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_update_master_replica_$SCRIPT_PID 2>&1 &
child_pid=$!
WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME 0
retval=$?
if [ $retval != 0 ]
then
LogError "Updating master replica failed. Stopping execution."
else
Log "Updating master replica succeded."
fi
if [ "$VERBOSE_LOGS" == "yes" ]
then
Log "List:\n$(cat /dev/shm/osync_update_master_replica_$SCRIPT_PID)"
fi
return $retval
}
function Sync
@ -398,14 +451,32 @@ function Sync
Log "Starting synchronization task."
## Create local file list
Log "Creating master replica file list."
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $MASTER_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > $STATE_DIR/master-tree-current
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $MASTER_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > /dev/shm/osync_master-tree-current_$SCRIPT_PID &
child_pid=$!
WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
retval=$?
if [ $retval == 0 ] && [ -f /dev/shm/osync_master-tree-current_$SCRIPT_PID ]
then
mv /dev/shm/osync_master-tree-current_$SCRIPT_PID $STATE_DIR/master-tree-current
else
LogError "Cannot create master file list."
return 1
fi
Log "Creating slave replica file list."
## Create distant file list
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > $STATE_DIR/slave-tree-current
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > /dev/shm/osync_slave-tree-current_$SCRIPT_PID &
child_pid=$!
WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
retval=$?
if [ $retval == 0 ] && [ -f /dev/shm/osync_slave-tree-current_$SCRIPT_PID ]
then
mv /dev/shm/osync_slave-tree-current_$SCRIPT_PID $STATE_DIR/slave-tree-current
else
LogError "Cannot create slave file list."
return 1
fi
## diff local file list and before file list except if before file list is empty >> deleted
Log "Creating master replica deleted file list."
if [ -f $STATE_DIR/master-tree-before ]
then
@ -414,7 +485,6 @@ function Sync
touch $STATE_DIR/master-deleted-list
fi
## diff local file list and before file list except if before file list is empty >> deleted
Log "Creating slave replica deleted file list."
if [ -f $STATE_DIR/slave-tree-before ]
then
@ -426,26 +496,83 @@ function Sync
if [ "$CONFLICT_PREVALANCE" != "master" ]
then
sync_update_slave
if [ $? != 0 ]
then
return 1
fi
sync_update_master
if [ $? != 0 ]
then
return 1
fi
else
sync_update_master
if [ $? != 0 ]
then
return 1
fi
sync_update_slave
if [ $? != 0 ]
then
return 1
fi
fi
Log "Propagating deletitions to slave replica."
rsync -rlptgodEui --backup --backup-dir "$MASTER_DELETE_DIR" --delete --exclude "$OSYNC_DIR" --include-from "$STATE_DIR/master-deleted-list" --exclude-from "$STATE_DIR/slave-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/
rsync $DRY_OPTION -rlptgodEui --backup --backup-dir "$SLAVE_DELETE_DIR" --delete --exclude "$OSYNC_DIR" --exclude-from "$STATE_DIR/slave-deleted-list" --include-from "$STATE_DIR/master-deleted-list" $MASTER_SYNC_DIR/ $SLAVE_SYNC_DIR/ > /dev/shm/osync_deletition_on_slave_$SCRIPT_PID 2>&1 &
child_pid=$!
WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME 0
if [ $? != 0 ]
then
LogError "Deletition on slave failed."
return 1
fi
if [ "$VERBOSE_LOGS" == "yes" ]
then
Log "List:\n$(cat /dev/shm/osync_deletition_on_slave_$SCRIPT_PID)"
fi
Log "Propagating deletitions to master replica."
rsync -rlptgodEui --backup --backup-dir "$SLAVE_DELETE_DIR" --delete --exclude "$OSYNC_DIR" --include-from "$STATE_DIR/slave-deleted-list" --exclude-from "$STATE_DIR/master-deleted-list" $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/
rsync $DRY_OPTION -rlptgodEui --backup --backup-dir "$MASTER_DELETE_DIR" --delete --exclude "$OSYNC_DIR" --exclude-from "$STATE_DIR/master-deleted-list" --include-from "$STATE_DIR/slave-deleted-list" $SLAVE_SYNC_DIR/ $MASTER_SYNC_DIR/ > /dev/shm/osync_deletition_on_master_$SCRIPT_PID 2>&1 &
child_pid=$!
WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME 0
if [ $? != 0 ]
then
LogError "Deletition on master failed."
return 1
fi
if [ "$VERBOSE_LOGS" == "yes" ]
then
Log "List:\n$(cat /dev/shm/osync_deletition_on_master_$SCRIPT_PID)"
fi
## Create local file list
Log "Creating new master replica file list."
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $MASTER_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > $STATE_DIR/master-tree-before
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $MASTER_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > /dev/shm/osync_master-tree-before_$SCRIPT_PID &
child_pid=$!
WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME 0
retval=$?
if [ $retval == 0 ] && [ -f /dev/shm/osync_master-tree-before_$SCRIPT_PID ]
then
mv /dev/shm/osync_master-tree-before_$SCRIPT_PID $STATE_DIR/master-tree-before
else
LogError "Cannot create slave file list."
return 1
fi
Log "Creating new slave replica file list."
## Create distant file list
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > $STATE_DIR/slave-tree-before
rsync -rlptgodE --exclude "$OSYNC_DIR" --list-only $SLAVE_SYNC_DIR/ | grep "^-\|^d" | awk '{print $5}' | grep -v "^\.$" > /dev/shm/osync_slave-tree-before_$SCRIPT_PID &
child_pid=$!
WaitForTaskCompletition $child_pid $SOFT_MAX_EXEC_TIME 0
retval=$?
if [ $retval == 0 ] && [ -f /dev/shm/osync_slave-tree-before_$SCRIPT_PID ]
then
mv /dev/shm/osync_slave-tree-before_$SCRIPT_PID $STATE_DIR/slave-tree-before
else
LogError "Cannot create slave file list."
return 1
fi
Log "Finished synchronization task."
}
function SoftDelete
@ -511,6 +638,13 @@ function Init
else
RUNNER=""
fi
## Dryrun option
if [ $dryrun -eq 1 ]
then
DRY_OPTION=--dry-run
DRY_WARNING="/!\ DRY RUN"
fi
}
function Main
@ -536,6 +670,7 @@ silent=0
# Alert flags
soft_alert_total=0
error_alert=0
soft_stop=0
if [ $# -eq 0 ]
then
@ -569,7 +704,7 @@ then
Init
DATE=$(date)
Log "-----------------------------------------------------------"
Log "$DATE - Osync v$OSYNC_VERSION script begin."
Log " $DRY_WARNING $DATE - Osync v$OSYNC_VERSION script begin."
Log "-----------------------------------------------------------"
CheckMasterSlaveDirs
if [ $? == 0 ]

8
sync.conf
Unescape Escape View File

@ -2,14 +2,14 @@
###### Osync file synchronizer tool based on rsync
###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr)
#### Config file rev 1507201301
#### Config file rev 1607201301
## Sync job identification, any string you want
SYNC_ID="sync_test"
## Directories to synchronize
MASTER_SYNC_DIR="/home/git/osync/test/dir1"
SLAVE_SYNC_DIR="/home/git/osync/test/dir2"
MASTER_SYNC_DIR="/root/backups"
SLAVE_SYNC_DIR="/srv/DONNEES/sl"
## Generate an alert if master or slave have lass space than given value in KB.
MINIMUM_SPACE=1048576
@ -45,6 +45,8 @@ CONFLICT_BACKUP_DAYS=30
SOFT_DELETE=yes
SOFT_DELETE_DAYS=30
VERBOSE_LOGS=yes
DESTINATION_MAILS="ozy@badministrateur.com"

Write Preview
Loading…
Cancel
Save