Archives
/
osync
mirror of https://github.com/deajan/osync
2
0
Fork 0
Code Issues Releases Wiki Activity

Finishing refactor

Browse Source
pull/36/head
deajan 9 years ago
parent fd98af1391
commit c44ebd094a
2 changed files with 70 additions and 99 deletions
Show Stats Download Patch File Download Diff File

7
CHANGELOG.md
Unescape Escape View File

@ -6,8 +6,11 @@ KNOWN ISSUES
RECENT CHANGES RECENT CHANGES
-------------- --------------
! Refactor [local master, local slave, remote slave] code to [local, remote][initiator, target]code ! Check existing dirs before checking for state dirs
- Added some automatic typo checks in code, for DEBUG mode (and PARANOIA_DEBUG now) ! Add Logger DEBUG to all eval statements
! Replace child_pid by $? directly
- Refactor [local master, local slave, remote slave] code to [local, remote][initiator, target]code
- Added some automatic typo checks in code, for _DEBUG mode (and _PARANOIA_DEBUG now)
- Improved Logging - Improved Logging
- Updated osync to be fully compliant with coding style - Updated osync to be fully compliant with coding style
- Uploaded coding style manifest - Uploaded coding style manifest

162
osync.sh
Unescape Escape View File

@ -4,7 +4,7 @@ PROGRAM="Osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(L) 2013-2015 by Orsiris \"Ozy\" de Jong" AUTHOR="(L) 2013-2015 by Orsiris \"Ozy\" de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.1-unstable PROGRAM_VERSION=1.1-unstable
PROGRAM_BUILD=2015091103 PROGRAM_BUILD=2015091201
## type doesn't work on platforms other than linux (bash). If if doesn't work, always assume output is not a zero exitcode ## type doesn't work on platforms other than linux (bash). If if doesn't work, always assume output is not a zero exitcode
if ! type -p "$BASH" > /dev/null; then if ! type -p "$BASH" > /dev/null; then
@ -1646,132 +1646,100 @@ function Sync {
echo "0" > "$INITIATOR_RESUME_COUNT" echo "0" > "$INITIATOR_RESUME_COUNT"
} }
#WIP: Need to refactor here (split local and remote code)
function _SoftDeleteLocal { function _SoftDeleteLocal {
local change_time="{1}" local replica_type="${1}" # replica type (initiator, target)
local replica_deletion_path="{2}" # Contains the full path to softdelete / backup directory local replica_deletion_path="${2}" # Contains the full path to softdelete / backup directory without ending slash
__CheckArguments 2 $# $FUNCNAME "$*" local change_time="${3}"
} __CheckArguments 3 $# $FUNCNAME "$*"
function _SoftDeleteRemote {
local change_time"${1}"
local replica_deletion_path="{2}" # Contains the full path to softdelete / backup directory
__CheckArguments 2 $# $FUNCNAME "$*"
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
}
# Takes 3 arguments
# $1 = ctime (CONFLICT_BACKUP_DAYS or SOFT_DELETE_DAYS), $2 = INITIATOR_(BACKUP/DELETED)_DIR, $3 = TARGET_(BACKUP/DELETED)_DIR
function _SoftDelete {
local change_time="${1}" # Contains the number of days a file needs to be old to be processed here (conflict or soft deletion)
local initiator_directory="${2}" # Master backup / deleted directory to search in
local target_directory="${3}" # Slave backup / deleted directory to search in
__CheckArguments 3 "$#" "$FUNCNAME" "$*"
if [ -d "$initiator_directory" ]; then if [ -d "$replica_deletion_path" ]; then
if [ $_DRYRUN -eq 1 ]; then if [ $_DRYRUN -eq 1 ]; then
Logger "Listing files older than $change_time days on initiator replica. Won't remove anything." "NOTICE" Logger "Listing files older than $change_time days on [$replica_type] replica. Won't remove anything." "NOTICE"
else else
Logger "Removing files older than $change_time days on initiator replica." "NOTICE" Logger "Removing files older than $change_time days on [$replica_type] replica." "NOTICE"
fi fi
if [ $_VERBOSE -eq 1 ]; then if [ $_VERBOSE -eq 1 ]; then
# Cannot launch log function from xargs, ugly hack # Cannot launch log function from xargs, ugly hack
$FIND_CMD "$initiator_directory/" -type f -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete file {}" > $RUN_DIR/osync_soft_delete_initiator_$SCRIPT_PID $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete file {}" > $RUN_DIR/osync_$FUNCNAME_$SCRIPT_PID
Logger "Command output:\n$(cat $RUN_DIR/osync_soft_delete_initiator_$SCRIPT_PID)" "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/osync_$FUNCNAME_$SCRIPT_PID)" "NOTICE"
$FIND_CMD "$initiator_directory/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete directory {}" > $RUN_DIR/osync_soft_delete_initiator_$SCRIPT_PID $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete directory {}" > $RUN_DIR/osync_$FUNCNAME_$SCRIPT_PID
Logger "Command output:\n$(cat $RUN_DIR/osync_soft_delete_initiator_$SCRIPT_PID)" "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/osync_$FUNCNAME_$SCRIPT_PID)" "NOTICE"
fi fi
if [ $_DRYRUN -ne 1 ]; then if [ $_DRYRUN -ne 1 ]; then
$FIND_CMD "$initiator_directory/" -type f -ctime +$change_time -print0 | xargs -0 -I {} rm -f "{}" && $FIND_CMD "$initiator_directory/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} rm -rf "{}" > $RUN_DIR/osync_soft_delete_initiator_$SCRIPT_PID 2>&1 & $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} rm -f "{}" && $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} rm -rf "{}" > $RUN_DIR/osync_$FUNCNAME_$SCRIPT_PID 2>&1 &
else else
Dummy & Dummy &
fi fi
child_pid=$! WaitForCompletion $? $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
retval=$? retval=$?
if [ $retval -ne 0 ]; then if [ $retval -ne 0 ]; then
Logger "Error while executing cleanup on initiator replica." "ERROR" Logger "Error while executing cleanup on [$replica_type] replica." "ERROR"
Logger "Command output:\n$(cat $RUN_DIR/osync_soft_delete_initiator_$SCRIPT_PID)" "NOTICE" Logger "Command output:\n$(cat $RUN_DIR/osync_$FUNCNAME_$SCRIPT_PID)" "NOTICE"
else else
Logger "Cleanup complete on initiator replica." "NOTICE" Logger "Cleanup complete on [$replica_type] replica." "NOTICE"
fi fi
elif [ -d "$initiator_directory" ] && ! [ -w "$initiator_directory" ]; then elif [ -d "$replica_deletion_path" ] && ! [ -w "$replica_deletion_path" ]; then
Logger "Warning: Master replica dir [$initiator_directory] isn't writable. Cannot clean old files." "ERROR" Logger "Warning: [$replica_type] replica dir [$replica_deletion_path] isn't writable. Cannot clean old files." "ERROR"
fi fi
}
if [ "$REMOTE_SYNC" == "yes" ]; then function _SoftDeleteRemote {
CheckConnectivity3rdPartyHosts local replica_type="${1}"
CheckConnectivityRemoteHost local replica_deletion_path="${2}" # Contains the full path to softdelete / backup directory without ending slash
if [ $_DRYRUN -eq 1 ]; then local change_time"${3}"
Logger "Listing files older than $change_time days on target replica. Won't remove anything." "NOTICE" __CheckArguments 3 $# $FUNCNAME "$*"
else
Logger "Removing files older than $change_time days on target replica." "NOTICE" CheckConnectivity3rdPartyHosts
fi CheckConnectivityRemoteHost
if [ $_VERBOSE -eq 1 ]; then
# Cannot launch log function from xargs, ugly hack if [ $_DRYRUN -eq 1 ]; then
eval "$SSH_CMD \"if [ -w \\\"$target_directory\\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\"$target_directory/\\\" -type f -ctime +$change_time -print0 | xargs -0 -I {} echo Will delete file {} && $REMOTE_FIND_CMD \\\"$target_directory/\\\" -type d -empty -ctime $change_time -print0 | xargs -0 -I {} echo Will delete directory {}; fi\"" > $RUN_DIR/osync_soft_delete_target_$SCRIPT_PID Logger "Listing files older than $change_time days on target replica. Won't remove anything." "NOTICE"
Logger "Command output:\n$(cat $RUN_DIR/osync_soft_delete_target_$SCRIPT_PID)" "NOTICE"
fi
if [ $_DRYRUN -ne 1 ]; then
eval "$SSH_CMD \"if [ -w \\\"$target_directory\\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\"$target_directory/\\\" -type f -ctime +$change_time -print0 | xargs -0 -I {} rm -f \\\"{}\\\" && $REMOTE_FIND_CMD \\\"$target_directory/\\\" -type d -empty -ctime $change_time -print0 | xargs -0 -I {} rm -rf \\\"{}\\\"; fi 2>&1\"" > $RUN_DIR/osync_soft_delete_target_$SCRIPT_PID &
else
Dummy &
fi
child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
retval=$?
if [ $retval -ne 0 ]; then
Logger "Error while executing cleanup on target replica." "ERROR"
Logger "Command output:\n$(cat $RUN_DIR/osync_soft_delete_target_$SCRIPT_PID)" "NOTICE"
else
Logger "Cleanup complete on target replica." "NOTICE"
fi
else else
if [ -w "$target_directory" ]; then Logger "Removing files older than $change_time days on target replica." "NOTICE"
if [ $_DRYRUN -eq 1 ]; then
Logger "Listing files older than $change_time days on target replica. Won't remove anything." "NOTICE"
else
Logger "Removing files older than $change_time days on target replica." "NOTICE"
fi
if [ $_VERBOSE -eq 1 ]; then
# Cannot launch log function from xargs, ugly hack
$FIND_CMD "$target_directory/" -type f -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete file {}" > $RUN_DIR/osync_soft_delete_target_$SCRIPT_PID
Logger "Command output:\n$(cat $RUN_DIR/osync_soft_delete_target_$SCRIPT_PID)" "NOTICE"
$FIND_CMD "$target_directory/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete directory {}" > $RUN_DIR/osync_soft_delete_target_$SCRIPT_PID
Logger "Command output:\n$(cat $RUN_DIR/osync_soft_delete_target_$SCRIPT_PID)" "NOTICE"
Dummy &
fi
if [ $_DRYRUN -ne 1 ]; then
$FIND_CMD "$target_directory/" -type f -ctime +$change_time -print0 | xargs -0 -I {} rm -f "{}" && $FIND_CMD "$target_directory/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} rm -rf "{}" > $RUN_DIR/osync_soft_delete_target_$SCRIPT_PID &
fi
child_pid=$!
WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
retval=$?
if [ $retval -ne 0 ]; then
Logger "Error while executing cleanup on target replica." "ERROR"
Logger "Command output:\n$(cat $RUN_DIR/osync_soft_delete_target_$SCRIPT_PID)" "NOTICE"
else
Logger "Cleanup complete on target replica." "NOTICE"
fi
elif [ -d "$target_directory" ] && ! [ -w "$target_directory" ]; then
Logger "Warning: Slave replica dir [$target_directory] isn't writable. Cannot clean old files." "ERROR"
fi
fi fi
if [ $_VERBOSE -eq 1 ]; then
# Cannot launch log function from xargs, ugly hack
eval "$SSH_CMD \"if [ -w \\\"$replica_deletion_path\\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\"$replica_deletion_path/\\\" -type f -ctime +$change_time -print0 | xargs -0 -I {} echo Will delete file {} && $REMOTE_FIND_CMD \\\"$replica_deletion_path/\\\" -type d -empty -ctime $change_time -print0 | xargs -0 -I {} echo Will delete directory {}; fi\"" > $RUN_DIR/osync_$FUNCNAME_$SCRIPT_PID
Logger "Command output:\n$(cat $RUN_DIR/osync_$FUNCNAME_$SCRIPT_PID)" "NOTICE"
fi
if [ $_DRYRUN -ne 1 ]; then
eval "$SSH_CMD \"if [ -w \\\"$replica_deletion_path\\\" ]; then $COMMAND_SUDO $REMOTE_FIND_CMD \\\"$replica_deletion_path/\\\" -type f -ctime +$change_time -print0 | xargs -0 -I {} rm -f \\\"{}\\\" && $REMOTE_FIND_CMD \\\"$replica_deletion_path/\\\" -type d -empty -ctime $change_time -print0 | xargs -0 -I {} rm -rf \\\"{}\\\"; fi 2>&1\"" > $RUN_DIR/osync_$FUNCNAME_$SCRIPT_PID &
else
Dummy &
fi
WaitForCompletion $? $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME
retval=$?
if [ $retval -ne 0 ]; then
Logger "Error while executing cleanup on remote target replica." "ERROR"
Logger "Command output:\n$(cat $RUN_DIR/osync_$FUNCNAME_$SCRIPT_PID)" "NOTICE"
else
Logger "Cleanup complete on target replica." "NOTICE"
fi
} }
function SoftDelete { function SoftDelete {
if [ "$CONFLICT_BACKUP" != "no" ] && [ $CONFLICT_BACKUP_DAYS -ne 0 ]; then if [ "$CONFLICT_BACKUP" != "no" ] && [ $CONFLICT_BACKUP_DAYS -ne 0 ]; then
Logger "Running conflict backup cleanup." "NOTICE" Logger "Running conflict backup cleanup." "NOTICE"
_SoftDelete $CONFLICT_BACKUP_DAYS "$INITIATOR_SYNC_DIR$INITIATOR_BACKUP_DIR" "$TARGET_SYNC_DIR$TARGET_BACKUP_DIR"
_SoftDeleteLocal "intiator" "$INITIATOR_SYNC_DIR$INITIATOR_BACKUP_DIR" $CONFLICT_BACKUP_DAYS
if [ "$REMOTE_SYNC" != "yes" ]; then
_SoftDeleteLocal "target" "$TARGET_SYNC_DIR$TARGET_BACKUP_DIR" $CONFLICT_BACKUP_DAYS
else
_SoftDeleteRemote "target" "$TARGET_SYNC_DIR$TARGET_BACKUP_DIR" $CONFLICT_BACKUP_DAYS
fi
fi fi
if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]; then if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]; then
Logger "Running soft deletion cleanup." "NOTICE" Logger "Running soft deletion cleanup." "NOTICE"
_SoftDelete $SOFT_DELETE_DAYS "$INITIATOR_SYNC_DIR$INITIATOR_DELETE_DIR" "$TARGET_SYNC_DIR$TARGET_DELETE_DIR"
_SoftDeleteLocal "initiator" "$INITIATOR_SYNC_DIR$INITIATOR_DELETE_DIR" $SOFT_DELETE_DAYS
if [ "$REMOTE_SYNC" != "yes" ]; then
_SoftDeleteLocal "target" "$TARGET_SYNC_DIR$TARGET_DELETE_DIR" $SOFT_DELETE_DAYS
else
_SoftDeleteRemote "target" "$TARGET_SYNC_DIR$TARGET_DELETE_DIR" $SOFT_DELETE_DAYS
fi
fi fi
} }

Write Preview
Loading…
Cancel
Save