dupbackup.sh

#!/bin/bash
#@(#) dupbackup.sh  Ver 0.6.1  16/11/22 Chris Ray
###  This script works with duplicity to keep a backup of the local system.
###
##
## Version 1  April 2020	First attempt
# 12/4/20	0.1.0	Modified to use a pre-established NFS mount rather than rsync. As the Netgear rsync performs badly under duplicity.
#			Also modified to run daily. Looks like duplicity will pick up from where it left off in an aborted backup
#			So we can run this daily, and if the previous run did not finish cleanly, we can run duplicity again to finish it.
# 15/4/20	    1	Bug in day recognition
# 8/5/20	    2   Failing to create current month file, moved location and added check..
# 7/6/20	    3 	Modified ACTION variable so that duplicity actually takes a FULL backup monthly
# 7/1/21	0.2.0	Need to include a cleanup script as the Jan Backup filled the NAS but continued trying until manually stopped.
#			Replaced explicit full or incr commands in ACTION var with --full-if-older-than option and set how many weeks to run increments.
#			with a full backup every month we only have enough space for 8 months history. So don't need to know the current month either.
# 10/1/21	    1	As the lockfile is now used to restart an aborted duplicity run, we need another file to test file access to the archive.
# 14/1/21	    2	Didn't turn on the desktop yesterday, so no backup was taken. Need a mechanism to take a backup the next day the PC is turned on, 
#			 if a wednesday is missed.
# 15/1/21	    3	General tidy up and bug fix.
# 3/2/21	    4	More tidy up and testing.
# 7/1/21	0.3.0	Added logic for stopping and starting the NFS mount. 1, It is more secure. 2. prevents duplicity trying to access the bare mount point
#			plus various tests to make sure the NFS mount is actually there
# 18/2/21	    1	Have now had two RESTART runs each claiming to finish a partial full backup. Both reported a clean Duplicity run, but both failed to
#			actually complete the backup, and left the incomplete set. So will now set the Cleanup script to remove any incomplete sets whether or not it
#			needs the space.	
# 19/2/21	    2	error in clean detection logic as awk writes a blank into the variable when it does NOT detect a match
# 20/2/21	    3	Modified cleanup logs so we get a collection-status dump before and after every cleanup.
# 12/6/21	0.4.0	Problem if a wednesday is missed FOLLOWING a missed Wednesday. The three day "window" is calculated from the actual day the eventual backup was taken
#			not the Wednesday it SHOULD have been taken, which compounds the error. Solution: When a backup finishes, stop saving the date of the last backup, and 
#			instead calculate and save the date of the NEXT backup day. This makes calculating the three day over-run window a lot easier.
# 13/8/21	    1	Added further text explanation of NFS mount checking.
# 28/8/21	    2	removed lines that stopped Access-test logging its output during a check-status run. Shows no diagnostics if the NAS is offline.
# 19/12/21	0.5.0	Problem if the system is down for more than one week. If the "back on-line" day is not a backup day the code used to wait until next backup day.
#			Have included code to calculate when the LAST backup day SHOULD have been and calculate the over-run window from there.
# 29/10/22	0.6.0	Modified the code that chaks the UID and GID of the monuted NAS drive. Has to be numeric in case the imported ID matches a local values and gets a text value.
# 16/11/22	0.6.1	Removed duplicity command from under if statements as the final tee command is masking the exit code and failing the if statement.
#

###  Architectural Description
###
## The Bazaar Systems Linux Desktop uses two drives for storing User data, /mnt/DATA2 and /mnt/MEDIA2. As well as various other mounts.
## A link in /etc/cron.daily point to a small script called runbackup that checks for the presence of this script and the logfiles and then 
## triggers this script using the systemd-inhibit command that should prevent accidental reboots during the backup. This script also includes a cleanup function
## that checks there is enough space on the NAS drive to fit at least one full backup.

# For debug testing
TEST="0"

## First set the external drive mappings
DIR=/opt/Backup
TDIR=${DIR}/tmp
LOGF=Dupbackup.log
LOGT=DupbackupTEST.log
if [[ $TEST -eq "0" ]]
	then LOG=${DIR}/${LOGF}
	else LOG=${DIR}/${LOGT}
fi
CLOG=${DIR}/Dupcleanup.log
STATUS=$DIR/chain-status
NEXT_BACKUP_DATE=${TDIR}/next_backup_date
ACCESSF=Access-test
ACCESS=${DIR}/${ACCESSF}
LOCKF=ScriptRunning.lck
LOCK=${DIR}/${LOCKF}

# An NFS mount is very much faster because of the way that duplicity uses rsync
MNT="/mnt/BackupDesktop"
MNTUID="98"
MNTGID="401"
TARDIR="$MNT/Ubuntu"
TARGET="file://$TARDIR"

## Backups only run ONCE per week, despite being triggered daily, on THIS day
DAY="Wed"  		# day of week to run backups, must be in date +%a format

##
### Set up the Duplicity options here
##
# Exclusions file. Used if backing up root to exclude the usual unwanted directories
EXCLUS=$DIR/exclusions
# The FILES var currently contains just an exclude file list but could include others.
FILES=" --exclude-filelist $EXCLUS"
# Duplicity will automatically take an incremental backup if a full backup exists, otherwise it takes a full,  and will do so forever
# This setting causes a full backup every "OLDER_THAN" 
OLDER_THAN="12W"
# Normal options used during the backup
#OPTIONS=" --no-encryption  --asynchronous-upload                           --dry-run  --max-blocksize 65536  --verbosity 4 --volsize 800 "
OPTIONS=" --no-encryption  --asynchronous-upload            			       --max-blocksize 65536  --verbosity 4 --volsize 800 "
#OPTIONS=" --no-encryption  --asynchronous-upload  --allow-source-mismatch             --max-blocksize 65536  --verbosity 4 --volsize 800 "


# Email settings
SYSID=$(uname -n|cut -f1 -d.)
SYDSMATES="Backup-support.$SYSID@bazaarsystems.co.uk"
SYDSOURCE="Duplicity_Backup@$SYSID.bazaarsystems.co.uk"


## Next the sub-routines

function displaytime {
  local T=$1
  local D=$((T/60/60/24))
  local H=$((T/60/60%24))
  local M=$((T/60%60))
  local S=$((T%60))
  [[ $D > 0 ]] && printf '%d days ' $D
  [[ $H > 0 ]] && printf '%d hours ' $H
  [[ $M > 0 ]] && printf '%d minutes ' $M
  [[ $D > 0 || $H > 0 || $M > 0 ]] && printf 'and '
  printf '%d seconds\n' $S
}

function mail-admin {
	echo sending mail with Subject: $subject  |tee -a $LOG
	echo "To: $SYDSMATES" > $TDIR/dupmail
	echo "From: $SYDSOURCE" >> $TDIR/dupmail
	echo "Subject: $subject" >> $TDIR/dupmail
	sendmail  $SYDSMATES < $TDIR/dupmail
}

function cleanup-log-start {
	echo -e "\r\n\r\n=====================================================" |tee -a $CLOG
	echo Duplicity Cleanup Process Started at $(date) | tee -a $CLOG
	# First send the collection status to the log file ONLY
	echo Current Collection Status >> $CLOG
	echo "*********************************************************************" >> $CLOG
	cat $STATUS >> $CLOG
	echo "*********************************************************************" >> $CLOG
}

function cleanup-log-stop {
	echo  Updating for Post-cleanup Collection Status >> $CLOG
	echo "*********************************************************************" >> $CLOG
	duplicity collection-status $TARGET 2>&1 > $TDIR/dup_out
	RETCODE=$?
	#echo $RETCODE
	if [[ $RETCODE -eq "0" ]]
		cat $TDIR/dup_out >> $CLOG
		then echo duplicity collection-status ran cleanly  >> $CLOG
		else echo duplicity collection-status run failure  >> $CLOG
	       	subject="Duplicity Backup failure: Duplicity post-cleanup collection-status run failed"
        	mail-admin
	fi
	echo "*********************************************************************" >> $CLOG
}

function check-space {
	echo Getting Collection-status from NAS.... |tee -a $LOG
	duplicity collection-status $TARGET 2>&1 > $STATUS
	RETCODE=$?
	#echo $RETCODE
	if [[ $RETCODE -eq "0" ]]
		then echo duplicity collection-status ran cleanly |tee -a $LOG
		else echo duplicity collection-status run failure |tee -a $LOG
        	subject="Duplicity Backup failure: Duplicity collection-status run failed"
        	mail-admin
	fi
	# unless awk matches one or more lines it returns a blank, but a 1 for each matched line
	CLEAN=$(cat $STATUS |awk '$0~/^No orphaned or incomplete backup sets found.$/ {print"1"}')
	## echo clean=$CLEAN
	# so if the $clean variable is blank, i.e. has no length execute cleanup
	if [[ -z $CLEAN ]]
		then
		## echo CHAINLESSSETS=$(cat $STATUS |awk '$0~/^Also found/&&$0~/backup sets not part of any chain,$/ {print$3}')
		CHAINLESSSETS=$(cat $STATUS |awk '$0~/^Also found/&&$0~/backup sets not part of any chain,$/ {print$3}')
		## echo INCOMPLETESETS=$(cat $STATUS |awk '$0~/^and/&&$0~/incomplete backup set/ {print$2}')
		INCOMPLETESETS=$(cat $STATUS |awk '$0~/^and/&&$0~/incomplete backup set/ {print$2}')
		if [[ $CHAINLESSSETS != 0 || $INCOMPLETESETS != 0 ]]
			then
			if [[ $CHAINLESSSETS != 0 ]]
				then echo $CHAINLESSSETS backup sets found that are not part of a chain. Removing...| tee -a $LOG
 
			elif [[ $INCOMPLETESETS != 0 ]]
				then echo $INCOMPLETESETS incomplete backup sets found. Removing...| tee -a $LOG
			fi
			echo cleanup needed....running duplicity with cleanup command...|tee -a $LOG
			cleanup-log-start
			# --no=encryption needed or it asks for a GNU passphrase.
			duplicity cleanup --no-encryption --force $TARGET 2>&1 > $TDIR/dup_out
			RETCODE=$?
			#echo $RETCODE
			if [[ $RETCODE -eq "0" ]]
				cat $TDIR/dup_out |tee -a $LOG
				then echo duplicity cleanup ran cleanly |tee -a $LOG
				else echo duplicity cleanup run failure |tee -a $LOG
        			subject="Duplicity Backup failure: Duplicity cleanup run failed"
        			mail-admin
			fi
			cleanup-log-stop
		fi
		else echo backup sets are clean, so no incomplete or chainless sets to remove. | tee -a $LOG
	fi
  # Now check the size of the last backup
	echo Checking space...  |tee -a $LOG
	PBVOLS=$(cat $STATUS |awk '$0~/Found primary backup chain/{primary=1} primary==1&&$0~/Total number of contained volumes/ {print$6}')
	PBSIZE=$(($PBVOLS * 800000))
	echo $PBVOLS volumes, so size of $PBSIZE KB
	echo Space used by current Primary Backup:  $PBSIZE KB  |tee -a $LOG
  # then check the space on the NAS drive
	echo Now checking space on the NAS drive...  |tee -a $LOG
	##df -h $MNT
	##df -k $MNT|awk -v tg="$MNT" '$0~tg {print $4}'
	AVAIL=$( df -k $MNT|awk -v tg="$MNT" '$0~tg {print $4}') 
	echo Space available on Archive :"         "  $AVAIL KB |tee -a $LOG
	echo space needed $(( $AVAIL - $PBSIZE ))
	if (( $AVAIL >= $PBSIZE ))
		then echo plenty of space for the next full backup | tee -a $LOG
		else echo not enough room for next full backup, Cleaning up.....  | tee -a $LOG
		cleanup-log-start
		echo From the current status: | tee -a $LOG
		echo Space needed"    ":  $PBSIZE KB | tee -a $LOG
		echo Space available :  $AVAIL KB | tee -a $LOG
		echo Space difference:  $(( $PBSIZE - $AVAIL )) KB | tee -a $LOG
		echo Volumes needed:  $(( ($PBSIZE - $AVAIL) /800000 )) | tee -a $LOG
		echo Need to remove oldest backup set..... | tee -a $LOG
		# If we use remove-older-than option then we need the time, in datetime format, for the start of the SECOND backup set.
		#	duplicity should then remove the whole of the FIRST backup set as ALL its sets will be older.
		echo -n "date of second oldest set is: "   | tee -a $LOG
		CHAIN2=$(cat $STATUS |awk '$0~/Secondary chain 2 of/{second=1} second==1&&$0~/Chain start time:/ {second=0; print$4" "$5" "$6" "$7" "$8}')
		echo -n  $CHAIN2 | tee -a $LOG
		#  I can use "date -Iseconds --date="Wed Nov 11 00:07:06 2020"" to convert the timestamps into ISO format that is acceptable to duplicity.
		SECONDOLDEST=$(date -Iseconds --date="$CHAIN2")
		echo " OR  $SECONDOLDEST  IN ISO format" | tee -a $LOG
		echo Using duplicity to remove the single full chain older than  $SECONDOLDEST  ....  |tee -a $LOG
		duplicity remove-older-than $SECONDOLDEST --force $TARGET 2>&1 > $TDIR/dup_out
		RETCODE=$?
		#echo $RETCODE
		if [[ $RETCODE -eq "0" ]]
			cat $TDIR/dup_out |tee -a $LOG
			then echo duplicity remove-older-than ran cleanly  |tee -a $LOG
			else echo duplicity remove-older-than run failure  |tee -a $LOG
        		subject="Duplicity Backup failure: Duplicity remove-older-than run failed"
        		mail-admin
		fi
		# And finally send an updated collection status to the log file
		cleanup-log-stop
	fi
}

function access-test {
## Now start up the NFS mount point and check that it is present and operating correctly......    

MOUNTPOINT="mnt-BackupDesktop.mount"
echo Starting mount point $MNT with systemctl $MOUNTPOINT... |tee -a $LOG
systemctl start $MOUNTPOINT

echo Testing presence of NFS mount | tee -a $LOG
# First check the NFS drive is mounted
if ((systemctl show $MOUNTPOINT |grep ActiveState=active) && (systemctl show $MOUNTPOINT |grep SubState=mounted))
	then echo NFS mount started by systemctl |tee -a $LOG
	else echo systemctl failed to mount the NFS drive |tee -a $LOG
       	subject="Duplicity Backup failure: Failed to mount $MOUNTPOINT"
       	mail-admin
	echo Desktop Backup Process Finished at $(date) | tee -a $LOG
	echo ------------------------------- | tee -a $LOG
	exit
fi
# and check the actual mount point is now covered.
# ls -aln ${MNT%/*}
declare -a MOUNTDATA
#ls -aln ${MNT%/*} |awk -v m=${MNT##*/} '$0~m{print $2,"\n",$3,"\n",$4,"\n",$5}'
mapfile -t MOUNTDATA < <(ls -aln ${MNT%/*} |awk -v m=${MNT##*/} '$0~m{print $2,"\n",$3,"\n",$4,"\n",$5}')
# echo ${MOUNTDATA[*]}
echo Check Mount Data: hardlinks=${MOUNTDATA[0]} should be 1,  UID=${MOUNTDATA[1]} should be $MNTUID,  GID=${MOUNTDATA[2]} should be $MNTGID,  size=${MOUNTDATA[3]} should be less than 100 | tee -a $LOG
if [[ ${MOUNTDATA[0]} -ne 1 || ${MOUNTDATA[1]} -ne $MNTUID || ${MOUNTDATA[2]} -ne $MNTGID || ${MOUNTDATA[3]} -gt 100 ]]
	then echo mount point  $MOUNTPOINT is NOT covered by an NFS mount | tee -a $LOG
       	subject="Duplicity Backup failure: Mount point $MOUNTPOINT not covered by NFS drive"
       	mail-admin
	echo Desktop Backup Process Finished at $(date) | tee -a $LOG
	echo ------------------------------- | tee -a $LOG
	exit
fi

# second, create an access-test file
echo creating Access-test file | tee -a $LOG
echo $(date +%s) Access-test created $(date) > $ACCESS

# third, test if the mounted Directory is present and writable
if [[ ! -d $TARDIR ]]
	then echo $TARDIR not present | tee -a $LOG
	elif [[ ! -w $TARDIR ]]
		then echo $TARDIR is not writable  | tee -a $LOG
		elif ! cp -p $ACCESS $TARDIR
			then echo Failed to Copy Access-test file to NFS mount  | tee -a $LOG
			elif ! cp -p $TARDIR/$ACCESSF $TDIR	
				then echo Failed to copy back the Access-test file  | tee -a $LOG
fi
 echo Sent:  $ACCESS  :  $(<$ACCESS)  | tee -a $LOG
 echo Returned:  $TDIR/$ACCESSF  :  $(<$TDIR/$ACCESSF)  | tee -a $LOG

# finally compare the original file written with the returned file...
if [[ ! -r ${TDIR}/${ACCESSF} ]]
        then
        echo Unable to access the returned file.  Permissions failure. Aborting script.... | tee -a $LOG
        subject="Duplicity Backup script failure: Unable to read returned check file"
        mail-admin
        finish
elif    diff $ACCESS ${TDIR}/${ACCESSF}
        then
        echo Successfully Wrote and Read back from $TARDIR NAS share | tee -a $LOG
        rm ${TDIR}/${ACCESSF}
        else
        echo Returned file different to original, aborting script.... | tee -a $LOG
        subject="Duplicity Backup script failure: Returned check file different from that sent"
        mail-admin
        finish
fi
}

function finish {
echo -e "\r\nStopping NFS mount for security" |tee -a $LOG
# Stop the mount point to make it safe from encryption attacks
systemctl stop $MOUNTPOINT
# Now check the NFS drive is no longer mounted
if ! systemctl status $MOUNTPOINT > $TDIR/sysctl.txt
	then echo NFS mount now offline |tee -a $LOG
	else echo system failed to unmount the NFS drive |tee -a $LOG
	cat $TDIR/sysctl.txt | tee -a $LOG
fi
echo Duplicity Backup Process Finished at $(date) | tee -a $LOG
echo -e "-------------------------------\r\n\r\n" | tee -a $LOG
# Sleep before exiting to allow the DOS screen to be visible after task completion.
sleep 10s
exit
}

# initialise stuff
echo -e "\r\n\r\n=====================================================" |tee -a $LOG
grep "@(#)" $0|grep -v grep |tee -a $LOG
echo $0 $* $(date) | tee -a $LOG
echo -n Running as username:  | tee -a $LOG
whoami | tee -a $LOG
echo Running as UID: $UID | tee -a $LOG
##echo test 1 $0 # expands to name of invoking file
##echo test 2 $* # expands to position parameters if any
##echo test 3 $$ # expands to process ID of shell
##echo test 4 $BASH # expands to full filename of this instance of bash

if [[ -e $LOCK ]]
	then echo "Lock file exists, is a session actually running, or did a previous run abort?" | tee -a $LOG
	ps -ef| grep ${BASH}.*$0 |grep -v grep | tee -a $LOG
##	ps -ef| grep ${BASH}.*$0 |grep -v grep |grep -v $$|wc -l
	echo This instance is process ID: $$ | tee -a $LOG
	if [[ $( ps -ef| grep ${BASH}.*$0 |grep -v grep |grep -v $$|wc -l) != 0 ]]
		then echo Another instance of this script is running... | tee -a $LOG
		WAS=$(cat $LOCK|awk '{print $1}')
		echo It started: $(date -d @$WAS)  | tee -a $LOG
		NOW=$(date +%s)
		DELAY=$(( $NOW - $WAS))
##		echo $DELAY
		displaytime $DELAY | tee -a $LOG
		# Now check if the other script has a copy of duplicity running under it...
		ps -ef| grep ${BASH}.*$0 |grep -v grep |grep -v $$  | tee -a $LOG
		PSID=$(ps -ef| grep ${BASH}.*$0 |grep -v grep |grep -v $$|awk '{print $2}')
		echo $PSID  | tee -a $LOG
		if ps -ef|grep duplicity |grep -v grep | tee -a $LOG
			then echo A copy of duplicty is running... | tee -a $LOG
			if ps -ef|grep duplicity |grep $PSID | tee -a $LOG
				then echo and it is owned by the shell... | tee -a $LOG
				else echo and is NOT owned by the other shell, sending warning email | tee -a $LOG
				subject="A copy of duplicity is running NOT owned by a copy of this script!!"
				mail-admin
				finish
			fi
			else echo NO copy of duplicity is running, but another copy of this script is, sending warning email  | tee -a $LOG
			subject="A copy of this script is running, but no copy of duplicity!!"  | tee -a $LOG
			mail-admin
			finish
		fi
		# so now we assume the other copy is a backup taking a long time, and just exit...
		finish
		else echo Lock file exists, but no other instances of this script  | tee -a $LOG
		if ! ps -ef|grep duplicity |grep -v grep
			then echo and no copy of duplicity itself | tee -a $LOG
			echo So assume the lockfile is from an aborted run of duplicity and run it again to complete the partial run. | tee -a $LOG
			subject="A lock file for the dupbackup script exists, assuming an aborted previous attempt and running again....."
			mail-admin
		fi
	fi
	else
	# just a final check that duplicity is NOT running
	if ps -ef|grep duplicity |grep -v grep
		then echo A copy of duplicty is running, but there is no script lock file???? | tee -a $LOG
		subject="A copy of duplicity, but there is no script lock file!!" | tee -a $LOG
		mail-admin
		finish
# # Backup or Cleanup stuff starts here......
		else echo No lock file, no duplicity running, is it Backup Day?  | tee -a $LOG
		# Check if it's backup day, or if we missed backup day by up to 3 days....
		NOW=$(date +%s)
##		# NOW=$(date +%s --date="2021-01-25")
		echo Today is $( date --date=@$NOW )  $NOW |tee -a $LOG
		if [[ $(date +%a --date=@$NOW) != $DAY ]]
			then echo Not the day for a backup, did we miss one by 3 days or less?  | tee -a $LOG
			# In this release we log the date of the next scheduled backup day at 03:00 in the morning when we take a backup, so the backup won't run if I happen to be 
			# working past midnight on backup day.  This makes preventing early execution easy.
			# This doesn't work: Anacron triggers at midnight, if it's a backup day a backup is taken.
			NEXT=$(<$NEXT_BACKUP_DATE)
			echo Next backup date is/was $( date --date=@$NEXT )  $NEXT | tee -a $LOG
			DELAY=$(( $NOW - $NEXT ))  
			echo elapsed seconds are: $DELAY
			if  [[ $DELAY -lt "0" ]]
				then echo Have not yet reached the scheduled backup day, checking space....  | tee -a $LOG
				access-test
				check-space
				finish
				else
				# using date +%j --date="@12345"  leads to some interesting results. Any number from 0 to 82799 counts as 1
				# 82800 to 169200 counts as 2, and then every 86400 seconds per day. But the change from day 1 to day 2 
				# occurs at 82800 which is only 23 hours.  There must be a reason for this.i
				# A week ticks over at 601200 NOT 604800 and counts 8 not 7.
				DELDAY=$(date +%j --date="@$DELAY")
				# Any number in an arithmetic expression starting with a zero is interpreted as OCTAL.
				#  so the output of "date +%j" normally  001 - 031 is interpreted as octal unless preceeded with a base definition i.e. 10#
				echo delayed days: $DELDAY
				#  So adding zero forces the variable to be interpreted as base 10 and supresses leading zeros
	 			DELDAY=$(( 10#$DELDAY + 0 ))
				echo $DELDAY days since the last scheduled backup, where backup day is day 1  | tee -a $LOG
				if  [[ $DELDAY -gt "8" ]]
					then echo more than one week overdue, so determine last backup day.  | tee -a $LOG
					LAST=$(date +%s --date="last $DAY + 3 hours")	
					echo Last backup date was $( date --date=@$LAST )  $LAST | tee -a $LOG
					DELAY=$(( $NOW - $LAST ))  
					DELDAY=$(date +%j --date="@$DELAY")
					echo days passed last $DAY : $DELDAY | tee -a $LOG
	 				DELDAY=$(( 10#$DELDAY + 0 ))
				fi
				if  [[ $DELDAY -gt "4" ]]
					then echo and 4 or more days overdue, so wait until next backup day. For now check space...  | tee -a $LOG
					access-test
					check-space
					finish
					else echo but less than 4 days overdue, so take a backup now..... | tee -a $LOG
				fi
			fi
		fi
	fi
fi

echo Taking Backup..... | tee -a $LOG
echo creating Lock file | tee -a $LOG
echo $(date +%s) Lock created $(date) > $LOCK

# Test the archive is available
access-test

###
#### So now run the actual backup
###
echo Running Duplicity to the NFS mount | tee -a $LOG
date | tee -a $LOG
# prepare the duplicity command line....
FILES=" --exclude-filelist $EXCLUS"
echo --full-if-older-than $OLDER_THAN $OPTIONS $FILES "/" $TARGET |tee -a $LOG
if [[ $TEST -eq "0" ]]
	then 
	#if false  --full-if-older-than $OLDER_THAN $OPTIONS $FILES $TARGET  |tee -a $LOG
	duplicity  --full-if-older-than $OLDER_THAN $OPTIONS $FILES / $TARGET 2>&1 > $TDIR/dup_out
	RETCODE=$?
	echo $RETCODE  
	cat $TDIR/dup_out |tee -a $LOG
	#if duplicity  --full-if-older-than $OLDER_THAN $OPTIONS $FILES / $TARGET 2>&1  |tee -a $LOG
	if [[ $RETCODE -eq "0" ]]
		then echo duplicity ran cleanly, calculating date of next backup.... |tee -a $LOG
		# Save date of next scheduled backup day at 03:00 in epoch format
		date +%s --date="next $DAY + 3 hours" > $NEXT_BACKUP_DATE
		date --date="@$(<$NEXT_BACKUP_DATE)"  |tee -a $LOG
		else echo duplicity run failure |tee -a $LOG
        	subject="Duplicity Backup failure: Duplicity run failed"
        	mail-admin
	fi
	else echo "TEST mode set... NOT taking backup." |tee -a $LOG
fi

echo -e "\r\nStopping NFS mount for security" |tee -a $LOG
# Stop the mount point to make it safe from encryption attacks
systemctl stop $MOUNTPOINT
# Now check the NFS drive is no longer mounted
if ! systemctl status $MOUNTPOINT > $TDIR/sysctl.txt
	then echo NFS mount now offline |tee -a $LOG
	else echo system failed to unmount the NFS drive |tee -a $LOG
	cat $TDIR/sysctl.txt | tee -a $LOG
fi

echo removing Lock...  | tee -a $LOG
rm $LOCK
echo Duplicity Backup Process Finished at $(date) | tee -a $LOG
echo ------------------------------- | tee -a $LOG
## Sleep before exiting to allow the DOS screen to be visible after task completion.
sleep 10s
exit