Archives
/
osync
mirror of https://github.com/deajan/osync
2
0
Fork 0
Code Issues Releases Wiki Activity

Rebuilt targets

Browse Source
pull/84/head
deajan 8 years ago
parent 63c64c5d5e
commit 9c9f5f01a2
2 changed files with 232 additions and 128 deletions
Show Stats Download Patch File Download Diff File

181
dev/debug_osync.sh
Unescape Escape View File

@ -4,7 +4,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.2-beta2
PROGRAM_BUILD=2016111401
PROGRAM_BUILD=2016111403
IS_STABLE=no
# Execution order #__WITH_PARANOIA_DEBUG
@ -45,15 +45,16 @@ IS_STABLE=no
#### MINIMAL-FUNCTION-SET BEGIN ####
## FUNC_BUILD=2016111401
## FUNC_BUILD=2016111402
## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## To use in a program, define the following variables:
## PROGRAM=program-name
## INSTANCE_ID=program-instance-name
## _DEBUG=yes/no
## _LOGGER_STDERR=True/False
## _LOGGER_ERR_ONLY=True/False
## _LOGGER_STDERR=true/false
## _LOGGER_ERR_ONLY=true/false
## _LOGGER_PREFIX="date"/"time"/""
#TODO: Windows checks, check sendmail & mailsend
@ -189,7 +190,7 @@ function Logger {
WARN_ALERT=true
return
elif [ "$level" == "NOTICE" ]; then
if [ "$_LOGGER_ERR_ONLY" != True ]; then
if [ "$_LOGGER_ERR_ONLY" != true ]; then
_Logger "$prefix$value"
fi
return
@ -198,6 +199,11 @@ function Logger {
_Logger "$prefix$value"
fi
return
elif [ "$level" == "ALWAYS" ]; then
if [ $_SILENT != true ]; then
_Logger "$prefix$value" "$prefix$level:$value" "$level:$value"
fi
return
elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then
_Logger "$prefix$value"
@ -321,9 +327,9 @@ function SendAlert {
eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE"
if [ $? != 0 ]; then
Logger "Cannot create [$ALERT_LOG_FILE]" "WARN"
attachment=False
attachment=false
else
attachment=True
attachment=true
fi
body="$MAIL_ALERT_MSG"$'\n\n'"$CURRENT_LOG"
@ -341,14 +347,14 @@ function SendAlert {
subject="Fnished run - $subject"
fi
if [ "$attachment" == True ]; then
if [ "$attachment" == true ]; then
attachmentFile="$ALERT_LOG_FILE"
fi
SendEmail "$subject" "$body" "$DESTINATION_MAILS" "$attachmentFile" "$SENDER_MAIL" "$SMTP_SERVER" "$SMTP_PORT" "$ENCRYPTION" "SMTP_USER" "$SMTP_PASSWORD"
# Delete tmp log file
if [ "$attachment" == True ]; then
if [ "$attachment" == true ]; then
if [ -f "$ALERT_LOG_FILE" ]; then
rm -f "$ALERT_LOG_FILE"
fi
@ -540,7 +546,7 @@ function LoadConfigFile {
}
function Spinner {
if [ $_SILENT == true ]; then
if [ $_SILENT == true ] || [ $_LOGGER_ERR_ONLY == true ]; then
return 0
fi
@ -2562,6 +2568,7 @@ function _deleteLocal {
Logger "Moving deleted file [$replicaDir$files] to [$replicaDir$deletionDir]." "VERBOSE"
mv -f "$replicaDir$files" "$replicaDir$deletionDir"
fi
echo "$replicaDir$files" >> "$RUN_DIR/$PROGRAM.delete.$replicaType.$SCRIPT_PID"
if [ $? != 0 ]; then
Logger "Cannot move [$replicaDir$files] to deletion directory." "ERROR"
echo "$files" >> "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__failedDeletedListFile]}"
@ -3239,6 +3246,43 @@ function SoftDelete {
fi
}
function _SummaryFromFile {
local summaryFile="${1}"
local direction="${2}"
__CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG
if [ -f "$summaryFile" ]; then
while read -r file; do
Logger "$direction $(echo $file | awk '{for (i=2; i<NF; i++) printf $i " "; print $NF}')" "ALWAYS"
done < "$summaryFile"
fi
}
function Summary {
__CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG
local prefix
prefix="$_LOGGER_PREFIX"
_LOGGER_PREFIX=""
Logger "File attribute updates: INITIATOR << >> TARGET" "ALWAYS"
_SummaryFromFile "$RUN_DIR/$PROGRAM.attr-update.target.$SCRIPT_PID" ">>"
_SummaryFromFile "$RUN_DIR/$PROGRAM.attr-update.initiator.$SCRIPT_PID" "<<"
Logger "File transfers: INITIATOR << >> TARGET" "ALWAYS"
_SummaryFromFile "$RUN_DIR/$PROGRAM.update.target.$SCRIPT_PID" ">>"
_SummaryFromFile "$RUN_DIR/$PROGRAM.update.initiator.$SCRIPT_PID" "<<"
Logger "File deletions: INITIATOR << >> TARGET" "ALWAYS"
_SummaryFromFile "$RUN_DIR/$PROGRAM.attr-update.target.$SCRIPT_PID" ">>"
_SummaryFromFile "$RUN_DIR/$PROGRAM.attr-update.initiator.$SCRIPT_PID" "<<"
_LOGGER_PREFIX="$prefix"
}
function Init {
__CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG
@ -3374,7 +3418,7 @@ function Init {
## Set sync only function arguments for rsync
SYNC_OPTS="-u"
if [ $_VERBOSE == true ]; then
if [ $_VERBOSE == true ] || [ $_SUMMARY == true ]; then
SYNC_OPTS=$SYNC_OPTS" -i"
fi
@ -3435,8 +3479,10 @@ function Usage {
echo ""
echo "[OPTIONS]"
echo "--dry Will run osync without actually doing anything; just testing"
echo "--no-prefix Will suppress time / date suffix from output"
echo "--silent Will run osync without any output to stdout, used for cron jobs"
echo "--errors-only Output only errors (can be combined with silent or verbose)"
echo "--summary Outputs a list of transferred / deleted files at the end of the run"
echo "--verbose Increases output"
echo "--stats Adds rsync transfer statistics to verbose output"
echo "--partial Allows rsync to keep partial downloads that can be resumed later (experimental)"
@ -3510,7 +3556,6 @@ fi
INITIATOR_LOCK_FILE_EXISTS=false
TARGET_LOCK_FILE_EXISTS=false
FORCE_UNLOCK=false
no_maxtime=false
opts=""
@ -3523,6 +3568,7 @@ _QUICK_SYNC=0
sync_on_changes=false
_NOLOCKS=false
osync_cmd=$0
_SUMMARY=false
if [ $# -eq 0 ]
then
@ -3601,9 +3647,16 @@ for i in "$@"; do
--no-locks)
_NOLOCKS=true
;;
--only-errors)
_LOGGER_STDERR=True
_LOGGER_ERR_ONLY=True
--errors-only)
#TODO: let err_only only output to stderr
#_LOGGER_STDERR=true
_LOGGER_ERR_ONLY=true
;;
--summary)
_SUMMARY=true
;;
--no-prefix)
_LOGGER_PREFIX=""
;;
*)
if [ $first == "0" ]; then
@ -3657,59 +3710,59 @@ opts="${opts# *}"
LoadConfigFile "$ConfigFile"
fi
if [ "$LOGFILE" == "" ]; then
if [ -w /var/log ]; then
LOG_FILE="/var/log/$PROGRAM.$INSTANCE_ID.log"
elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
LOG_FILE="$HOME/$PROGRAM.$INSTANCE_ID.log"
else
LOG_FILE="./$PROGRAM.$INSTANCE_ID.log"
fi
if [ "$LOGFILE" == "" ]; then
if [ -w /var/log ]; then
LOG_FILE="/var/log/$PROGRAM.$INSTANCE_ID.log"
elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
LOG_FILE="$HOME/$PROGRAM.$INSTANCE_ID.log"
else
LOG_FILE="$LOGFILE"
fi
if [ ! -w "$(dirname $LOG_FILE)" ]; then
echo "Cannot write to log [$(dirname $LOG_FILE)]."
else
Logger "Script begin, logging to [$LOG_FILE]." "DEBUG"
LOG_FILE="./$PROGRAM.$INSTANCE_ID.log"
fi
else
LOG_FILE="$LOGFILE"
fi
if [ ! -w "$(dirname $LOG_FILE)" ]; then
echo "Cannot write to log [$(dirname $LOG_FILE)]."
else
Logger "Script begin, logging to [$LOG_FILE]." "DEBUG"
fi
if [ "$IS_STABLE" != "yes" ]; then
Logger "This is an unstable dev build [$PROGRAM_BUILD]. Please use with caution." "WARN"
if [ "$IS_STABLE" != "yes" ]; then
Logger "This is an unstable dev build [$PROGRAM_BUILD]. Please use with caution." "WARN"
fi
GetLocalOS
InitLocalOSSettings
PreInit
Init
CheckEnvironment
PostInit
if [ $_QUICK_SYNC -lt 2 ]; then
CheckCurrentConfig
GetLocalOS
InitLocalOSSettings
PreInit
Init
CheckEnvironment
PostInit
if [ $_QUICK_SYNC -lt 2 ]; then
CheckCurrentConfig
fi
CheckCurrentConfigAll
DATE=$(date)
Logger "-------------------------------------------------------------" "NOTICE"
Logger "$DRY_WARNING $DATE - $PROGRAM $PROGRAM_VERSION script begin." "ALWAYS"
Logger "-------------------------------------------------------------" "NOTICE"
Logger "Sync task [$INSTANCE_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" "NOTICE"
if [ $sync_on_changes == true ]; then
SyncOnChanges
else
GetRemoteOS
InitRemoteOSSettings
if [ $no_maxtime == true ]; then
SOFT_MAX_EXEC_TIME=0
HARD_MAX_EXEC_TIME=0
fi
CheckReplicaPaths
CheckDiskSpace
RunBeforeHook
Main
if [ $? == 0 ]; then
SoftDelete
fi
CheckCurrentConfigAll
DATE=$(date)
Logger "-------------------------------------------------------------" "NOTICE"
Logger "$DRY_WARNING $DATE - $PROGRAM $PROGRAM_VERSION script begin." "NOTICE"
Logger "-------------------------------------------------------------" "NOTICE"
Logger "Sync task [$INSTANCE_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" "NOTICE"
if [ $sync_on_changes == true ]; then
SyncOnChanges
else
GetRemoteOS
InitRemoteOSSettings
if [ $no_maxtime == true ]; then
SOFT_MAX_EXEC_TIME=0
HARD_MAX_EXEC_TIME=0
fi
CheckReplicaPaths
CheckDiskSpace
RunBeforeHook
Main
if [ $? == 0 ]; then
SoftDelete
fi
if [ $_SUMMARY == true ]; then
Summary
fi
fi

179
osync.sh
Unescape Escape View File

@ -4,22 +4,23 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.2-beta2
PROGRAM_BUILD=2016111401
PROGRAM_BUILD=2016111403
IS_STABLE=no
#### MINIMAL-FUNCTION-SET BEGIN ####
## FUNC_BUILD=2016111401
## FUNC_BUILD=2016111402
## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## To use in a program, define the following variables:
## PROGRAM=program-name
## INSTANCE_ID=program-instance-name
## _DEBUG=yes/no
## _LOGGER_STDERR=True/False
## _LOGGER_ERR_ONLY=True/False
## _LOGGER_STDERR=true/false
## _LOGGER_ERR_ONLY=true/false
## _LOGGER_PREFIX="date"/"time"/""
#TODO: Windows checks, check sendmail & mailsend
@ -150,7 +151,7 @@ function Logger {
WARN_ALERT=true
return
elif [ "$level" == "NOTICE" ]; then
if [ "$_LOGGER_ERR_ONLY" != True ]; then
if [ "$_LOGGER_ERR_ONLY" != true ]; then
_Logger "$prefix$value"
fi
return
@ -159,6 +160,11 @@ function Logger {
_Logger "$prefix$value"
fi
return
elif [ "$level" == "ALWAYS" ]; then
if [ $_SILENT != true ]; then
_Logger "$prefix$value" "$prefix$level:$value" "$level:$value"
fi
return
elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then
_Logger "$prefix$value"
@ -272,9 +278,9 @@ function SendAlert {
eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE"
if [ $? != 0 ]; then
Logger "Cannot create [$ALERT_LOG_FILE]" "WARN"
attachment=False
attachment=false
else
attachment=True
attachment=true
fi
body="$MAIL_ALERT_MSG"$'\n\n'"$CURRENT_LOG"
@ -292,14 +298,14 @@ function SendAlert {
subject="Fnished run - $subject"
fi
if [ "$attachment" == True ]; then
if [ "$attachment" == true ]; then
attachmentFile="$ALERT_LOG_FILE"
fi
SendEmail "$subject" "$body" "$DESTINATION_MAILS" "$attachmentFile" "$SENDER_MAIL" "$SMTP_SERVER" "$SMTP_PORT" "$ENCRYPTION" "SMTP_USER" "$SMTP_PASSWORD"
# Delete tmp log file
if [ "$attachment" == True ]; then
if [ "$attachment" == true ]; then
if [ -f "$ALERT_LOG_FILE" ]; then
rm -f "$ALERT_LOG_FILE"
fi
@ -489,7 +495,7 @@ function LoadConfigFile {
}
function Spinner {
if [ $_SILENT == true ]; then
if [ $_SILENT == true ] || [ $_LOGGER_ERR_ONLY == true ]; then
return 0
fi
@ -2404,6 +2410,7 @@ function _deleteLocal {
Logger "Moving deleted file [$replicaDir$files] to [$replicaDir$deletionDir]." "VERBOSE"
mv -f "$replicaDir$files" "$replicaDir$deletionDir"
fi
echo "$replicaDir$files" >> "$RUN_DIR/$PROGRAM.delete.$replicaType.$SCRIPT_PID"
if [ $? != 0 ]; then
Logger "Cannot move [$replicaDir$files] to deletion directory." "ERROR"
echo "$files" >> "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__failedDeletedListFile]}"
@ -3075,6 +3082,41 @@ function SoftDelete {
fi
}
function _SummaryFromFile {
local summaryFile="${1}"
local direction="${2}"
if [ -f "$summaryFile" ]; then
while read -r file; do
Logger "$direction $(echo $file | awk '{for (i=2; i<NF; i++) printf $i " "; print $NF}')" "ALWAYS"
done < "$summaryFile"
fi
}
function Summary {
local prefix
prefix="$_LOGGER_PREFIX"
_LOGGER_PREFIX=""
Logger "File attribute updates: INITIATOR << >> TARGET" "ALWAYS"
_SummaryFromFile "$RUN_DIR/$PROGRAM.attr-update.target.$SCRIPT_PID" ">>"
_SummaryFromFile "$RUN_DIR/$PROGRAM.attr-update.initiator.$SCRIPT_PID" "<<"
Logger "File transfers: INITIATOR << >> TARGET" "ALWAYS"
_SummaryFromFile "$RUN_DIR/$PROGRAM.update.target.$SCRIPT_PID" ">>"
_SummaryFromFile "$RUN_DIR/$PROGRAM.update.initiator.$SCRIPT_PID" "<<"
Logger "File deletions: INITIATOR << >> TARGET" "ALWAYS"
_SummaryFromFile "$RUN_DIR/$PROGRAM.attr-update.target.$SCRIPT_PID" ">>"
_SummaryFromFile "$RUN_DIR/$PROGRAM.attr-update.initiator.$SCRIPT_PID" "<<"
_LOGGER_PREFIX="$prefix"
}
function Init {
# Set error exit code if a piped command fails
@ -3209,7 +3251,7 @@ function Init {
## Set sync only function arguments for rsync
SYNC_OPTS="-u"
if [ $_VERBOSE == true ]; then
if [ $_VERBOSE == true ] || [ $_SUMMARY == true ]; then
SYNC_OPTS=$SYNC_OPTS" -i"
fi
@ -3268,8 +3310,10 @@ function Usage {
echo ""
echo "[OPTIONS]"
echo "--dry Will run osync without actually doing anything; just testing"
echo "--no-prefix Will suppress time / date suffix from output"
echo "--silent Will run osync without any output to stdout, used for cron jobs"
echo "--errors-only Output only errors (can be combined with silent or verbose)"
echo "--summary Outputs a list of transferred / deleted files at the end of the run"
echo "--verbose Increases output"
echo "--stats Adds rsync transfer statistics to verbose output"
echo "--partial Allows rsync to keep partial downloads that can be resumed later (experimental)"
@ -3342,7 +3386,6 @@ fi
INITIATOR_LOCK_FILE_EXISTS=false
TARGET_LOCK_FILE_EXISTS=false
FORCE_UNLOCK=false
no_maxtime=false
opts=""
@ -3355,6 +3398,7 @@ _QUICK_SYNC=0
sync_on_changes=false
_NOLOCKS=false
osync_cmd=$0
_SUMMARY=false
if [ $# -eq 0 ]
then
@ -3433,9 +3477,16 @@ for i in "$@"; do
--no-locks)
_NOLOCKS=true
;;
--only-errors)
_LOGGER_STDERR=True
_LOGGER_ERR_ONLY=True
--errors-only)
#TODO: let err_only only output to stderr
#_LOGGER_STDERR=true
_LOGGER_ERR_ONLY=true
;;
--summary)
_SUMMARY=true
;;
--no-prefix)
_LOGGER_PREFIX=""
;;
*)
if [ $first == "0" ]; then
@ -3489,59 +3540,59 @@ opts="${opts# *}"
LoadConfigFile "$ConfigFile"
fi
if [ "$LOGFILE" == "" ]; then
if [ -w /var/log ]; then
LOG_FILE="/var/log/$PROGRAM.$INSTANCE_ID.log"
elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
LOG_FILE="$HOME/$PROGRAM.$INSTANCE_ID.log"
else
LOG_FILE="./$PROGRAM.$INSTANCE_ID.log"
fi
else
LOG_FILE="$LOGFILE"
fi
if [ ! -w "$(dirname $LOG_FILE)" ]; then
echo "Cannot write to log [$(dirname $LOG_FILE)]."
if [ "$LOGFILE" == "" ]; then
if [ -w /var/log ]; then
LOG_FILE="/var/log/$PROGRAM.$INSTANCE_ID.log"
elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
LOG_FILE="$HOME/$PROGRAM.$INSTANCE_ID.log"
else
Logger "Script begin, logging to [$LOG_FILE]." "DEBUG"
LOG_FILE="./$PROGRAM.$INSTANCE_ID.log"
fi
else
LOG_FILE="$LOGFILE"
fi
if [ ! -w "$(dirname $LOG_FILE)" ]; then
echo "Cannot write to log [$(dirname $LOG_FILE)]."
else
Logger "Script begin, logging to [$LOG_FILE]." "DEBUG"
fi
if [ "$IS_STABLE" != "yes" ]; then
Logger "This is an unstable dev build [$PROGRAM_BUILD]. Please use with caution." "WARN"
if [ "$IS_STABLE" != "yes" ]; then
Logger "This is an unstable dev build [$PROGRAM_BUILD]. Please use with caution." "WARN"
fi
GetLocalOS
InitLocalOSSettings
PreInit
Init
CheckEnvironment
PostInit
if [ $_QUICK_SYNC -lt 2 ]; then
CheckCurrentConfig
GetLocalOS
InitLocalOSSettings
PreInit
Init
CheckEnvironment
PostInit
if [ $_QUICK_SYNC -lt 2 ]; then
CheckCurrentConfig
fi
CheckCurrentConfigAll
DATE=$(date)
Logger "-------------------------------------------------------------" "NOTICE"
Logger "$DRY_WARNING $DATE - $PROGRAM $PROGRAM_VERSION script begin." "ALWAYS"
Logger "-------------------------------------------------------------" "NOTICE"
Logger "Sync task [$INSTANCE_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" "NOTICE"
if [ $sync_on_changes == true ]; then
SyncOnChanges
else
GetRemoteOS
InitRemoteOSSettings
if [ $no_maxtime == true ]; then
SOFT_MAX_EXEC_TIME=0
HARD_MAX_EXEC_TIME=0
fi
CheckReplicaPaths
CheckDiskSpace
RunBeforeHook
Main
if [ $? == 0 ]; then
SoftDelete
fi
CheckCurrentConfigAll
DATE=$(date)
Logger "-------------------------------------------------------------" "NOTICE"
Logger "$DRY_WARNING $DATE - $PROGRAM $PROGRAM_VERSION script begin." "NOTICE"
Logger "-------------------------------------------------------------" "NOTICE"
Logger "Sync task [$INSTANCE_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" "NOTICE"
if [ $sync_on_changes == true ]; then
SyncOnChanges
else
GetRemoteOS
InitRemoteOSSettings
if [ $no_maxtime == true ]; then
SOFT_MAX_EXEC_TIME=0
HARD_MAX_EXEC_TIME=0
fi
CheckReplicaPaths
CheckDiskSpace
RunBeforeHook
Main
if [ $? == 0 ]; then
SoftDelete
fi
if [ $_SUMMARY == true ]; then
Summary
fi
fi

Write Preview
Loading…
Cancel
Save