rsnapshot

Secure incremental backups over the network

First configure a backup user and set up ssh keys for remote servers with sudo for the rsync command, see this article.

rsnapshot.conf

config_version  1.2
snapshot_root   /storage/rsnapshot/
cmd_cp          /bin/cp
cmd_rm          /bin/rm
cmd_rsync       /usr/bin/rsync
cmd_ssh         /usr/bin/ssh
cmd_logger      /usr/bin/logger
retain  hourly  2
retain  daily   7
retain  weekly  4
retain  monthly 3
verbose         4
loglevel        3
logfile         /var/log/rsnapshot.log
lockfile        /var/run/rsnapshot.pid
rsync_long_args --delete --numeric-ids --relative --delete-excluded --stats --rsync-path="sudo rsync"
ssh_args        -i /root/.ssh/id_rsa_backup
link_dest       1
exclude	        .git
exclude	        sess_
exclude	        proc
exclude	        sys
exclude	        dev
backup	        backup-user@srv1:/opt  server1/
backup	        backup-user@srv1:/etc  server1/

cron.d/rsnapshot

0 */12        * * *           root    /usr/bin/rsnapshot hourly
30 3          * * *           root    /usr/bin/rsnapshot daily 2>&1 | /usr/local/bin/rsnapreport.pl | /usr/bin/mail -s"rsnapshot daily" root
0  3          * * 1           root    /usr/bin/rsnapshot weekly
30 2          1 * *           root    /usr/bin/rsnapshot monthly

rsnapreport.pl

#!/usr/bin/env perl
# this script prints a pretty report from rsnapshot output
# in the rsnapshot.conf you must set
# verbose >= 4
# and add --stats to rsync_long_args
# then setup crontab 'rsnapshot daily 2>&1 | rsnapreport.pl | mail -s"SUBJECT" backupadm@adm.com
# don't forget the 2>&1 or your errors will be lost to stderr
################################
## Copyright 2006 William Bear
## This program is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 2 of the License, or
## (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this program; if not, write to the Free Software
## Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
################################
use strict;
use warnings;
use English '-no_match_vars';

my $bufsz = 2;
my %bkdata=();
my @errors=();

sub pretty_print(){
	my $ofh = select(STDOUT);
	$FORMAT_NAME="BREPORTBODY";
	$FORMAT_TOP_NAME="BREPORTHEAD";
	select($ofh);

	foreach my $source (sort keys %bkdata){
		if($bkdata{$source} =~ /error/i) { print "ERROR $source $bkdata{$source}"; next; }
		my $files = $bkdata{$source}{'files'};
		my $filest = $bkdata{$source}{'files_tran'};
		my $filelistgentime = $bkdata{$source}{'file_list_gen_time'};
		my $filelistxfertime = $bkdata{$source}{'file_list_trans_time'};
		my $bytes = $bkdata{$source}{'file_size'}/1000000; # convert to MB
		my $bytest = $bkdata{$source}{'file_tran_size'}/1000000; # convert to MB
		$source =~ s/^[^\@]+\@//; # remove username
		format BREPORTHEAD =
SOURCE                          TOTAL FILES   FILES TRANS      TOTAL MB     MB TRANS   LIST GEN TIME  FILE XFER TIME
--------------------------------------------------------------------------------------------------------------------
.
		format BREPORTBODY =
@<<<<<<<<<<<<<<<<<<<<<<<<<<<<<	@>>>>>>>>>>   @>>>>>>>>>> @#########.## @########.##   @>>>>>>>>>>>>  @>>>>>>>>>>>>>
$source,                        $files,       $filest,    $bytes,       $bytest,       $filelistgentime, $filelistxfertime
.
		write STDOUT;
	}
}

sub nextLine($){
	my($lines) = @_;
	my $line = <>;
	push(@$lines,$line);
	return shift @$lines;
}


my @rsnapout = ();

# load readahead buffer
for(my $i=0; $i < $bufsz; $i++){
	$rsnapout[$i] = <>;
}
while (my $line = nextLine(\@rsnapout)){
	if($line =~ /^[\/\w]+\/rsync/) { # find start rsync command line
		my @rsynccmd=();
		while($line =~ /\s+\\$/){ # combine wrapped lines
			$line =~ s/\\$//g;
			$line .= nextLine(\@rsnapout);
		}
		push(@rsynccmd,split(/\s+/,$line)); # split into command components
		my $source = $rsynccmd[-2]; # count backwards: source always second to last
		#print $source;
		while($line = nextLine(\@rsnapout)){
  			# this means we are missing stats info
			if($line =~ /^[\/\w]+\/rsync/){
				unshift(@rsnapout,$line);
				push(@errors,"$source NO STATS DATA");
				last;
			}
			# stat record
			if($line =~ /^total size is\s+\d+/){ last; } # this ends the rsync stats record
			# Number of files: 1,325 (reg: 387, dir: 139, link: 799)
			elsif($line =~ /Number of files:\s+([\d,]+)/){
				$bkdata{$source}{'files'}=$1;
				$bkdata{$source}{'files'}=~ s/,//g;
			}
			# Number of regular files transferred: 1
			elsif($line =~ /Number of (regular )?files transferred:\s+([\d,]+)/){
				$bkdata{$source}{'files_tran'}=$2;
			}
			# Total file size: 1,865,857 bytes
			elsif($line =~ /Total file size:\s+([\d,]+)/){
				$bkdata{$source}{'file_size'}=$1;
				$bkdata{$source}{'file_size'}=~ s/,//g;
			}
			elsif($line =~ /Total transferred file size:\s+([\d,]+)/){
				$bkdata{$source}{'file_tran_size'}=$1;
				$bkdata{$source}{'file_tran_size'}=~ s/,//g;
			}
			elsif($line =~ /File list generation time:\s+(.+)/){
				$bkdata{$source}{'file_list_gen_time'}=$1;
			}
			elsif($line =~ /File list transfer time:\s+(.+)/){
				$bkdata{$source}{'file_list_trans_time'}=$1;
			}
			elsif($line =~ /^(rsync error|ERROR): /){ push(@errors,"$source $line"); } # we encountered an rsync error
		}
	}
	elsif($line =~ /^(rsync error|ERROR): /){ push(@errors,$line); } # we encountered an rsync error
}

pretty_print();
if(scalar @errors > 0){
	print "\nERRORS\n";
	print join("\n",@errors);
	print "\n";
}