#!/bin/bash

#
# Backup a set of servers.
# This program version can handle 0 or more rotations of backups.
#
# If servers are not specified on command line, they are read from
# the ../config/SERVERS_TO_BACKUP file.
#
# Unless the input is from the terminal, the program records its
# output in automatically created report files, one for each day of
# the week, and reports on disk usage.
#
# Requirements:
#	apt-get install rsync
#

PROGNAME=$( basename $0 )
USAGE="Usage: [ BACKUP_ROTATIONS=N ] $PROGNAME [ server_name ... ]"

DEFAULT_BACKUP_ROTATIONS=2	# default number of rotations before overwriting prevous backups
BACKUP_ROTATIONS=${BACKUP_ROTATIONS:-$DEFAULT_BACKUP_ROTATIONS}	# look in environment variable first

MYDIR=$( dirname $0 )
PARENTDIR=$( cd "$MYDIR"/..; pwd )
CONFIGDIR="$PARENTDIR/config"
REPORTDIR="$PARENTDIR/reports"
REPORT="$REPORTDIR/BACKUP_REPORT"
DULOG="$REPORTDIR/du.log"
DOW=$( date '+%a' )		# Mon, Tue, ...
TIMEFORMAT='Time: %0lR'		# display only elapsed real time

BACKUP="/*"			# files to backup on servers
RSYNC="/usr/bin/rsync"		# rsync program used to copy files
WRAP="fmt -w 96"		# program used to wrap report output

# Rotate backups between one or more rotations.
if [ "$BACKUP_ROTATIONS" -gt 1 ]
then
	# Store backups in directories named rotation-1, rotation-2, ...
	# Compute day from epoch to avoid Jan 1 backups from overwriting Dec 31 backups.
	CURRENT_ROTATION="rotation-$( date +'%s' | awk -v divisor=$BACKUP_ROTATIONS '{print ((int($1 / (24*60*60))) % divisor) + 1}')"
	BACKUPDIR="$PARENTDIR/servers/$CURRENT_ROTATION"
else
	# No rotation of backup files. Only one backup is kept in the "servers" directory.
	CURRENT_ROTATION=
	BACKUPDIR="$PARENTDIR/servers"
fi

# Get standard options from config (overwrite above definitions).
if [ -f "$CONFIGDIR/STANDARD_OPTIONS" ]
then
	STANDARD_OPTIONS="$( sed -e 's/\s*#.*//' "$CONFIGDIR/STANDARD_OPTIONS" )"
	EXCLUDES=''	# now assumed to be included in STANDARD_OPTIONS from config dir
else
	STANDARD_OPTIONS="-Sa -e ssh --delete --numeric-ids --hard-links --compress --human-readable"
	EXCLUDES="
--exclude=/dev/
--exclude=/lost+found/
--exclude=/media/
--exclude=/mnt/
--exclude=/proc/
--exclude=/run/
--exclude=/swapfile
--exclude=/sys/
--exclude=/tmp/
--exclude=/var/cache/
--exclude=/var/lock/
--exclude=/var/run/
--exclude=/var/tmp/
"
fi

case "$1" in
"")	SERVERS_TO_BACKUP=$( sed -e 's/\s*#.*//' $CONFIGDIR/SERVERS_TO_BACKUP ) ;;
*)	SERVERS_TO_BACKUP="$@" ;;
esac

MANUAL_BACKUP=
[ -t 0 ] && MANUAL_BACKUP=yes

if [ ! "$MANUAL_BACKUP" ]
then
	# Send all output to report file.
	[ -d "$REPORTDIR" ] || mkdir -p "$REPORTDIR"

	# Begin new report
	exec &> $REPORT.$DOW

	# Link "current" report to today's report
	[ -f $REPORT ] && rm $REPORT 
	ln -s $REPORT.$DOW $REPORT

	# Start report with banner
	echo "Backup Server: $(hostname -f)"
	echo
fi

echo "Backup started:  $( date )" 
[ "$CURRENT_ROTATION" ] && echo "Rotation: $CURRENT_ROTATION of $BACKUP_ROTATIONS"

echo
echo "Servers to backup: $(echo $SERVERS_TO_BACKUP | wc -w)" 
echo $SERVERS_TO_BACKUP | $WRAP

echo
echo "Standard options:"
echo $RSYNC $STANDARD_OPTIONS $EXCLUDES | $WRAP

function do_backup {
	# Function to allow timing of each backup.
	# Uncomment the following if you want only the specific server options to be included in report.
	echo "Backing up: $BACKUP  To: $BACKUPDIR/$server"
	echo ${SERVER_OPTIONS:-"Using only standard options."} | $WRAP

	# Uncomment the following if you want the full rsync command to be included in report.
	#echo $RSYNC $STANDARD_OPTIONS $EXCLUDES $SERVER_OPTIONS $server:\"$BACKUP\" \"$BACKUPDIR/$server/\"

	$RSYNC $STANDARD_OPTIONS $EXCLUDES $SERVER_OPTIONS $server:"$BACKUP" "$BACKUPDIR/$server/"
	status=$?

	# Change top directory file modification time to time of last backup.
	[ $status -eq 0 ] && touch "$BACKUPDIR/$server"
	return $status
}

nservers=0 bservers=0
for server in $SERVERS_TO_BACKUP
do
	[ -d $BACKUPDIR/$server ] || mkdir --mode=755 -p $BACKUPDIR/$server

	SERVER_OPTIONS=$( [ -f "$CONFIGDIR/$server" ] && sed -e 's/\s*#.*//' "$CONFIGDIR/$server" )

	# Display line from config file for the server with its comments.
	echo
	grep "^$server[ 	]" "$CONFIGDIR/SERVERS_TO_BACKUP" || echo "$server:"
	echo "Start: $( date +%r )"

	time do_backup && nservers=$(($nservers + 1)) || bservers=$(($bservers + 1)) | grep -v vanished
done

if [ "$CURRENT_ROTATION" ]
then
	# Change rotation directory's file modification time to time of last backup.
	touch "$BACKUPDIR"

	# Link current-rotation to this rotation's directory.
	(
		cd "$BACKUPDIR"/../ && {
			rm current-rotation &>/dev/null
			ln -s "$CURRENT_ROTATION" current-rotation
			touch "$CURRENT_ROTATION"
		}
	)
fi

echo
echo "Backup finished: $( date )"
echo "$nservers servers backed up."
[ $bservers -gt 0 ] && echo "$bservers servers failed."

[ "$MANUAL_BACKUP" ] && exit

echo 
echo "Backup Sizes of Last 7 Backups"
echo $(now=`date +%s`; for i in `seq 6 -1 0`;do date --date=@$(( $now - (86400*$i) )) +"%a"; done) "Diff" | sed -e 's/ /	/g' # day column headers

# Compute and keep log of server disk usage
hostname=$( hostname )
for server in $SERVERS_TO_BACKUP
do
	[ "$server" = "$hostname" ] && continue		# skip this server's log, done below
	size=$( du -sBG $BACKUPDIR/$server | awk '{print $1;exit}' )
	timestamp=$( date --rfc-3339=seconds )
	echo -e "$timestamp\t$server\t$size" >> $DULOG
done

# Include disk usage of this backup server in log.
size=$( df -h "$BACKUPDIR" | perl -ane 'END{print "$F[2]\n";}' )
timestamp=$( date --rfc-3339=seconds )
echo -e "$timestamp\t$hostname\t$size" >> $DULOG

# Set servers to report. Remove this host name from list first as it is added to the end.
SERVERS_TO_REPORT="${SERVERS_TO_BACKUP/$hostname/} $hostname"

# Show server disk usage of last week
cat $DULOG | perl -ane '
{
        # Make collection of arrays named after servers names containing list of disk usage numbers.
        unshift @{$F[2]},$F[3];         # save most recent size first
}
END {
        foreach my $server (qw('"$SERVERS_TO_REPORT"')) {
                # Show sizes of last 7 days for a server on one line.
                print join "\t", reverse @{$server}[0..6];
                print "\t";
                printf("%4.1f",${$server}[0]-${$server}[6]);	# print difference between begin and end sizes
                print "\t";
		print `grep "^$server\\s" '"$CONFIGDIR/SERVERS_TO_BACKUP"' || echo $server`;	# print config comments too
        }
}
'

echo
echo "Free Space"
# List free space: friendly units, list filesystem type, exclude tmpfs and udev filesystems, total
df -hT -x tmpfs -x udev -x devtmpfs --total

echo
echo "Done: $( date )"