Archives
/
osync
mirror of https://github.com/deajan/osync
2
0
Fork 0
Code Issues Releases Wiki Activity

Refactored soft deletion code

Browse Source
pull/17/head
deajan 9 years ago
parent 41781aa0da
commit f45086dad3
1 changed files with 104 additions and 175 deletions
Show Stats Download Patch File Download Diff File

279
osync.sh
Unescape Escape View File

@ -4,7 +4,7 @@ PROGRAM="Osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(L) 2013-2015 by Orsiris \"Ozy\" de Jong" AUTHOR="(L) 2013-2015 by Orsiris \"Ozy\" de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.00pre PROGRAM_VERSION=1.00pre
PROGRAM_BUILD=3003201502 PROGRAM_BUILD=3103201501
## type doesn't work on platforms other than linux (bash). If if doesn't work, always assume output is not a zero exitcode ## type doesn't work on platforms other than linux (bash). If if doesn't work, always assume output is not a zero exitcode
if ! type -p "$BASH" > /dev/null if ! type -p "$BASH" > /dev/null
@ -1491,198 +1491,129 @@ function SoftDelete
{ {
if [ "$CONFLICT_BACKUP" != "no" ] && [ $CONFLICT_BACKUP_DAYS -ne 0 ] if [ "$CONFLICT_BACKUP" != "no" ] && [ $CONFLICT_BACKUP_DAYS -ne 0 ]
then then
if [ -d "$MASTER_SYNC_DIR$MASTER_BACKUP_DIR" ] Log "Running conflict backup cleanup."
then _SoftDelete $CONFLICT_BACKUP_DAYS "$MASTER_SYNC_DIR$MASTER_BACKUP_DIR/" "$SLAVE_SYNC_DIR$SLAVE_BACKUP_DIR/"
if [ $dryrun -eq 1 ] fi
then
Log "Listing backups older than $CONFLICT_BACKUP_DAYS days on master replica. Won't remove anything."
else
Log "Removing backups older than $CONFLICT_BACKUP_DAYS days on master replica."
fi
if [ $verbose -eq 1 ] if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]
then then
# Cannot launch log function from xargs, ugly hack Log "Running soft deletion cleanup."
$FIND_CMD "$MASTER_SYNC_DIR$MASTER_BACKUP_DIR/" -type f -ctime +$CONFLICT_BACKUP_DAYS -print0 | xargs -0 -I {} echo "Will delete file {}" > $RUN_DIR/osync_conflict_backup_master_$SCRIPT_PID _SoftDelete $SOFT_DELETE_DAYS "$MASTER_SYNC_DIR$MASTER_DELETE_DIR/" "$SLAVE_SYNC_DIR$SLAVE_DELETE_DIR/"
Log "Command output:\n$(cat $RUN_DIR/osync_conflict_backup_master_$SCRIPT_PID)" fi
$FIND_CMD "$MASTER_SYNC_DIR$MASTER_BACKUP_DIR/" -type d -empty -ctime +$CONFLICT_BACKUP_DAYS -print0 | xargs -0 -I {} echo "Will delete directory {}" > $RUN_DIR/osync_conflict_backup_master_$SCRIPT_PID }
Log "Command output:\n$(cat $RUN_DIR/osync_conflict_backup_master_$SCRIPT_PID)"
fi
if [ $dryrun -ne 1 ]
then
$FIND_CMD "$MASTER_SYNC_DIR$MASTER_BACKUP_DIR/" -type f -ctime +$CONFLICT_BACKUP_DAYS -print0 | xargs -0 -I {} rm -f "{}" && $FIND_CMD "$MASTER_SYNC_DIR$MASTER_BACKUP_DIR/" -type d -empty -ctime +$CONFLICT_BACKUP_DAYS -print0 | xargs -0 -I {} rm -rf "{}" > $RUN_DIR/osync_conflict_backup_master_$SCRIPT_PID &
else
Dummy &
fi
child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
retval=$?
if [ $retval -ne 0 ]
then
LogError "Error while executing conflict backup cleanup on master replica."
Log "Command output:\n$(cat $RUN_DIR/osync_conflict_backup_master_$SCRIPT_PID)"
else
Log "Conflict backup cleanup complete on master replica."
fi
elif [ -d "$MASTER_SYNC_DIR$MASTER_BACKUP_DIR" ] && ! [ -w "$MASTER_SYNC_DIR$MASTER_BACKUP_DIR" ] # Takes 3 arguments
# $1 = ctime (CONFLICT_BACKUP_DAYS or SOFT_DELETE_DAYS), $2 = MASTER_(BACKUP/DELETED)_DIR, $3 = SLAVE_(BACKUP/DELETED)_DIR
function _SoftDelete
{
if [ -d "$2" ]
then
if [ $dryrun -eq 1 ]
then then
LogError "Warning: Master replica conflict backup dir [$MASTER_SYNC_DIR$MASTER_BACKUP_DIR] isn't writable. Cannot clean old files." Log "Listing files older than $1 days on master replica. Won't remove anything."
else
Log "Removing files older than $1 days on master replica."
fi fi
if [ "$REMOTE_SYNC" == "yes" ]
then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
if [ $dryrun -eq 1 ]
then
Log "Listing backups older than $CONFLICT_BACKUP_DAYS days on slave replica. Won't remove anything."
else
Log "Removing backups older than $CONFLICT_BACKUP_DAYS days on slave replica."
fi
if [ $verbose -eq 1 ] if [ $verbose -eq 1 ]
then then
# Cannot launch log function from xargs, ugly hack # Cannot launch log function from xargs, ugly hack
eval "$SSH_CMD \"if [ -w \\\"$SLAVE_SYNC_DIR$SLAVE_BACKUP_DIR\\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\"$SLAVE_SYNC_DIR$SLAVE_BACKUP_DIR/\\\" -type f -ctime +$CONFLICT_BACKUP_DAYS -print0 | xargs -0 -I {} echo Will delete file {} && $REMOTE_FIND_CMD \\\"$SLAVE_SYNC_DIR$SLAVE_BACKUP_DIR/\\\" -type d -empty -ctime $CONFLICT_BACKUP_DAYS -print0 | xargs -0 -I {} echo Will delete directory {}; fi\"" > $RUN_DIR/osync_conflict_backup_slave_$SCRIPT_PID $FIND_CMD "$2/" -type f -ctime +$1 -print0 | xargs -0 -I {} echo "Will delete file {}" > $RUN_DIR/osync_soft_delete_master_$SCRIPT_PID
Log "Command output:\n$(cat $RUN_DIR/osync_conflict_backup_slave_$SCRIPT_PID)" Log "Command output:\n$(cat $RUN_DIR/osync_soft_delete_master_$SCRIPT_PID)"
fi $FIND_CMD "$2/" -type d -empty -ctime +$1 -print0 | xargs -0 -I {} echo "Will delete directory {}" > $RUN_DIR/osync_soft_delete_master_$SCRIPT_PID
Log "Command output:\n$(cat $RUN_DIR/osync_soft_delete_master_$SCRIPT_PID)"
fi
if [ $dryrun -ne 1 ] if [ $dryrun -ne 1 ]
then then
eval "$SSH_CMD \"if [ -w \\\"$SLAVE_SYNC_DIR$SLAVE_BACKUP_DIR\\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\"$SLAVE_SYNC_DIR$SLAVE_BACKUP_DIR/\\\" -type f -ctime +$CONFLICT_BACKUP_DAYS -print0 | xargs -0 -I {} rm -f \\\"{}\\\" && $REMOTE_FIND_CMD \\\"$SLAVE_SYNC_DIR$SLAVE_BACKUP_DIR/\\\" -type d -empty -ctime $CONFLICT_BACKUP_DAYS -print0 | xargs -0 -I {} rm -rf \\\"{}\\\"; fi\"" > $RUN_DIR/osync_conflict_backup_slave_$SCRIPT_PID & $FIND_CMD "$2/" -type f -ctime +$1 -print0 | xargs -0 -I {} rm -f "{}" && $FIND_CMD "$2/" -type d -empty -ctime +$1 -print0 | xargs -0 -I {} rm -rf "{}" > $RUN_DIR/osync_soft_delete_master_$SCRIPT_PID &
else
Dummy &
fi
child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
retval=$?
if [ $retval -ne 0 ]
then
LogError "Error while executing conflict backup cleanup on slave replica."
Log "Command output:\n$(cat $RUN_DIR/osync_conflict_backup_slave_$SCRIPT_PID)"
else
Log "Conflict backup cleanup complete on slave replica."
fi
else else
if [ -w "$SLAVE_SYNC_DIR$SLAVE_BACKUP_DIR" ] Dummy &
then fi
if [ $dryrun -eq 1 ] child_pid=$!
then WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
Log "Listing backups older than $CONFLICT_BACKUP_DAYS days on slave replica. Won't remove anything." retval=$?
else if [ $retval -ne 0 ]
Log "Removing backups older than $CONFLICT_BACKUP_DAYS days on slave replica." then
fi LogError "Error while executing cleanup on master replica."
Log "Command output:\n$(cat $RUN_DIR/osync_soft_delete_master_$SCRIPT_PID)"
if [ $verbose -eq 1 ] else
then Log "Cleanup complete on master replica."
# Cannot launch log function from xargs, ugly hack
$FIND_CMD "$SLAVE_SYNC_DIR$SLAVE_BACKUP_DIR/" -type f -ctime +$CONFLICT_BACKUP_DAYS -print0 | xargs -0 -I {} echo "Will delete file {}" > $RUN_DIR/osync_conflict_backup_slave_$SCRIPT_PID
Log "Command output:\n$(cat $RUN_DIR/osync_conflict_backup_slave_$SCRIPT_PID)"
$FIND_CMD "$SLAVE_SYNC_DIR$SLAVE_BACKUP_DIR/" -type d -empty -ctime +$CONFLICT_BACKUP_DAYS -print0 | xargs -0 -I {} echo "Will delete directory {}" > $RUN_DIR/osync_conflict_backup_slave_$SCRIPT_PID
Log "Command output:\n$(cat $RUN_DIR/osync_conflict_backup_slave_$SCRIPT_PID)"
Dummy &
fi
if [ $dryrun -ne 1 ]
then
$FIND_CMD "$SLAVE_SYNC_DIR$SLAVE_BACKUP_DIR/" -type f -ctime +$CONFLICT_BACKUP_DAYS -print0 | xargs -0 -I {} rm -f "{}" && $FIND_CMD "$SLAVE_SYNC_DIR$SLAVE_BACKUP_DIR/" -type d -empty -ctime +$CONFLICT_BACKUP_DAYS -print0 | xargs -0 -I {} rm -rf "{}" > $RUN_DIR/osync_conflict_backup_slave_$SCRIPT_PID &
fi
child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
retval=$?
if [ $retval -ne 0 ]
then
LogError "Error while executing conflict backup cleanup on slave replica."
Log "Command output:\n$(cat $RUN_DIR/osync_conflict_backup_slave_$SCRIPT_PID)"
else
Log "Conflict backup cleanup complete on slave replica."
fi
elif [ -d "$SLAVE_SYNC_DIR$SLAVE_BACKUP_DIR" ] && ! [ -w "$SLAVE_SYNC_DIR$SLAVE_BACKUP_DIR" ]
then
LogError "Warning: Slave replica conflict backup dir [$SLAVE_SYNC_DIR$SLAVE_BACKUP_DIR] isn't writable. Cannot clean old files."
fi
fi fi
elif [ -d "$2" ] && ! [ -w "$2" ]
then
LogError "Warning: Master replica dir [$2] isn't writable. Cannot clean old files."
fi fi
if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ] if [ "$REMOTE_SYNC" == "yes" ]
then then
if [ -d "$MASTER_SYNC_DIR$MASTER_DELETE_DIR" ] CheckConnectivity3rdPartyHosts
then CheckConnectivityRemoteHost
if [ $dryrun -eq 1 ] if [ $dryrun -eq 1 ]
then
Log "Listing soft deleted items older than $SOFT_DELETE_DAYS days on master replica. Won't remove anything."
$FIND_CMD "$MASTER_SYNC_DIR$MASTER_DELETE_DIR/" -ctime +$SOFT_DELETE_DAYS &
else
Log "Removing soft deleted items older than $SOFT_DELETE_DAYS days on master replica."
$FIND_CMD "$MASTER_SYNC_DIR$MASTER_DELETE_DIR/" -ctime +$SOFT_DELETE_DAYS -exec rm -rf '{}' \; &
fi
child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
retval=$?
if [ $retval -ne 0 ]
then
LogError "Error while executing soft delete cleanup on master replica."
else
Log "Soft delete cleanup complete on master replica."
fi
elif [ -d "$MASTER_SYNC_DIR$MASTER_DELETE_DIR" ] && ! [ -w "$MASTER_SYNC_DIR$MASTER_DELETE_DIR" ]
then then
LogError "Warning: Master replica deletion backup dir [$MASTER_SYNC_DIR$MASTER_DELETE_DIR] isn't writable. Cannot clean old files." Log "Listing files older than $1 days on slave replica. Won't remove anything."
else
Log "Removing files older than $1 days on slave replica."
fi fi
if [ $verbose -eq 1 ]
if [ "$REMOTE_SYNC" == "yes" ]
then then
CheckConnectivity3rdPartyHosts # Cannot launch log function from xargs, ugly hack
CheckConnectivityRemoteHost eval "$SSH_CMD \"if [ -w \\\"$3\\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\"$3/\\\" -type f -ctime +$1 -print0 | xargs -0 -I {} echo Will delete file {} && $REMOTE_FIND_CMD \\\"$3/\\\" -type d -empty -ctime $1 -print0 | xargs -0 -I {} echo Will delete directory {}; fi\"" > $RUN_DIR/osync_soft_delete_slave_$SCRIPT_PID
if [ $dryrun -eq 1 ] Log "Command output:\n$(cat $RUN_DIR/osync_soft_delete_slave_$SCRIPT_PID)"
then fi
Log "Listing soft deleted items older than $SOFT_DELETE_DAYS days on slave replica. Won't remove anything." if [ $dryrun -ne 1 ]
eval "$SSH_CMD \"if [ -w \\\"$SLAVE_SYNC_DIR$SLAVE_DELETE_DIR\\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\"$SLAVE_SYNC_DIR$SLAVE_DELETE_DIR/\\\" -ctime +$SOFT_DELETE_DAYS; fi\"" & then
else eval "$SSH_CMD \"if [ -w \\\"$3\\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\"$3/\\\" -type f -ctime +$1 -print0 | xargs -0 -I {} rm -f \\\"{}\\\" && $REMOTE_FIND_CMD \\\"$3/\\\" -type d -empty -ctime $1 -print0 | xargs -0 -I {} rm -rf \\\"{}\\\"; fi\"" > $RUN_DIR/osync_soft_delete_slave_$SCRIPT_PID &
Log "Removing soft deleted items older than $SOFT_DELETE_DAYS days on remote slave replica." else
eval "$SSH_CMD \"if [ -w \\\"$SLAVE_SYNC_DIR$SLAVE_DELETE_DIR\\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\"$SLAVE_SYNC_DIR$SLAVE_DELETE_DIR/\\\" -ctime +$SOFT_DELETE_DAYS -exec rm -rf '{}' \;; fi\"" & Dummy &
fi fi
child_pid=$! child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
retval=$? retval=$?
if [ $retval -ne 0 ] if [ $retval -ne 0 ]
then then
LogError "Error while executing soft delete cleanup on slave replica." LogError "Error while executing cleanup on slave replica."
Log "Command output:\n$(cat $RUN_DIR/osync_soft_delete_slave_$SCRIPT_PID)"
else else
Log "Soft delete cleanup complete on slave replica." Log "Cleanup complete on slave replica."
fi fi
else
if [ -w "$3" ]
then
if [ $dryrun -eq 1 ]
then
Log "Listing files older than $1 days on slave replica. Won't remove anything."
else else
if [ -w "$SLAVE_SYNC_DIR$SLAVE_DELETE_DIR" ] Log "Removing files older than $1 days on slave replica."
then fi
if [ $dryrun -eq 1 ] if [ $verbose -eq 1 ]
then then
Log "Listing soft deleted items older than $SOFT_DELETE_DAYS days on slave replica. Won't remove anything." # Cannot launch log function from xargs, ugly hack
$FIND_CMD "$SLAVE_SYNC_DIR$SLAVE_DELETE_DIR/" -ctime +$SOFT_DELETE_DAYS & $FIND_CMD "$3/" -type f -ctime +$1 -print0 | xargs -0 -I {} echo "Will delete file {}" > $RUN_DIR/osync_soft_delete_slave_$SCRIPT_PID
else Log "Command output:\n$(cat $RUN_DIR/osync_soft_delete_slave_$SCRIPT_PID)"
Log "Removing soft deleted items older than $SOFT_DELETE_DAYS days on slave replica." $FIND_CMD "$3/" -type d -empty -ctime +$1 -print0 | xargs -0 -I {} echo "Will delete directory {}" > $RUN_DIR/osync_soft_delete_slave_$SCRIPT_PID
$FIND_CMD "$SLAVE_SYNC_DIR$SLAVE_DELETE_DIR/" -ctime +$SOFT_DELETE_DAYS -exec rm -rf '{}' \; & Log "Command output:\n$(cat $RUN_DIR/osync_conflict_backup_slave_$SCRIPT_PID)"
fi Dummy &
child_pid=$! fi
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME if [ $dryrun -ne 1 ]
retval=$? then
if [ $retval -ne 0 ] $FIND_CMD "$3/" -type f -ctime +$1 -print0 | xargs -0 -I {} rm -f "{}" && $FIND_CMD "$3/" -type d -empty -ctime +$1 -print0 | xargs -0 -I {} rm -rf "{}" > $RUN_DIR/osync_soft_delete_slave_$SCRIPT_PID &
then fi
LogError "Error while executing soft delete cleanup on slave replica." child_pid=$!
else WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
Log "Soft delete cleanup complete on slave replica." retval=$?
fi if [ $retval -ne 0 ]
elif [ -d "$SLAVE_SYNC_DIR$SLAVE_DELETE_DIR" ] && ! [ -w "$SLAVE_SYNC_DIR$SLAVE_DELETE_DIR" ] then
then LogError "Error while executing cleanup on slave replica."
LogError "Warning: Slave replica deletion backup dir [$SLAVE_SYNC_DIR$SLAVE_DELETE_DIR] isn't writable. Cannot clean old files." Log "Command output:\n$(cat $RUN_DIR/osync_soft_delete_slave_$SCRIPT_PID)"
fi else
Log "Cleanup complete on slave replica."
fi
elif [ -d "$3" ] && ! [ -w "$3" ]
then
LogError "Warning: Slave replica dir [$3] isn't writable. Cannot clean old files."
fi fi
fi fi
} }
function Init function Init
@ -2211,8 +2142,6 @@ then
Init Init
GetRemoteOS GetRemoteOS
InitRemoteOSSettings InitRemoteOSSettings
SoftDelete
exit $?
if [ $sync_on_changes -eq 1 ] if [ $sync_on_changes -eq 1 ]
then then
SyncOnChanges SyncOnChanges

Write Preview
Loading…
Cancel
Save