rsnapshot

Secure incremental backups over the network

First configure a backup user and set up ssh keys for remote servers with sudo for the rsync command, see this article.

rsnapshot.conf

config_version 1.2
snapshot_root /storage/rsnapshot/
cmd_cp /bin/cp
cmd_rm /bin/rm
cmd_rsync /usr/bin/rsync
cmd_ssh /usr/bin/ssh
cmd_logger /usr/bin/logger
retain hourly 2
retain daily 7
retain weekly 4
retain monthly 3
verbose 4
loglevel 3
logfile /var/log/rsnapshot.log
lockfile /var/run/rsnapshot.pid
rsync_long_args --delete --numeric-ids --relative --delete-excluded --stats --rsync-path="sudo rsync"
ssh_args -i /root/.ssh/id_rsa_backup
link_dest 1
exclude .git
exclude sess_
exclude proc
exclude sys
exclude dev
backup backup-user@srv1:/opt server1/
backup backup-user@srv1:/etc server1/
config_version 1.2 snapshot_root /storage/rsnapshot/ cmd_cp /bin/cp cmd_rm /bin/rm cmd_rsync /usr/bin/rsync cmd_ssh /usr/bin/ssh cmd_logger /usr/bin/logger retain hourly 2 retain daily 7 retain weekly 4 retain monthly 3 verbose 4 loglevel 3 logfile /var/log/rsnapshot.log lockfile /var/run/rsnapshot.pid rsync_long_args --delete --numeric-ids --relative --delete-excluded --stats --rsync-path="sudo rsync" ssh_args -i /root/.ssh/id_rsa_backup link_dest 1 exclude .git exclude sess_ exclude proc exclude sys exclude dev backup backup-user@srv1:/opt server1/ backup backup-user@srv1:/etc server1/
config_version  1.2
snapshot_root   /storage/rsnapshot/
cmd_cp          /bin/cp
cmd_rm          /bin/rm
cmd_rsync       /usr/bin/rsync
cmd_ssh         /usr/bin/ssh
cmd_logger      /usr/bin/logger
retain  hourly  2
retain  daily   7
retain  weekly  4
retain  monthly 3
verbose         4
loglevel        3
logfile         /var/log/rsnapshot.log
lockfile        /var/run/rsnapshot.pid
rsync_long_args --delete --numeric-ids --relative --delete-excluded --stats --rsync-path="sudo rsync"
ssh_args        -i /root/.ssh/id_rsa_backup
link_dest       1
exclude	        .git
exclude	        sess_
exclude	        proc
exclude	        sys
exclude	        dev
backup	        backup-user@srv1:/opt  server1/
backup	        backup-user@srv1:/etc  server1/

cron.d/rsnapshot

0 */12 * * * root /usr/bin/rsnapshot hourly
30 3 * * * root /usr/bin/rsnapshot daily 2>&1 | /usr/local/bin/rsnapreport.pl | /usr/bin/mail -s"rsnapshot daily" root
0 3 * * 1 root /usr/bin/rsnapshot weekly
30 2 1 * * root /usr/bin/rsnapshot monthly
0 */12 * * * root /usr/bin/rsnapshot hourly 30 3 * * * root /usr/bin/rsnapshot daily 2>&1 | /usr/local/bin/rsnapreport.pl | /usr/bin/mail -s"rsnapshot daily" root 0 3 * * 1 root /usr/bin/rsnapshot weekly 30 2 1 * * root /usr/bin/rsnapshot monthly
0 */12        * * *           root    /usr/bin/rsnapshot hourly
30 3          * * *           root    /usr/bin/rsnapshot daily 2>&1 | /usr/local/bin/rsnapreport.pl | /usr/bin/mail -s"rsnapshot daily" root
0  3          * * 1           root    /usr/bin/rsnapshot weekly
30 2          1 * *           root    /usr/bin/rsnapshot monthly

rsnapreport.pl

#!/usr/bin/env perl
# this script prints a pretty report from rsnapshot output
# in the rsnapshot.conf you must set
# verbose >= 4
# and add --stats to rsync_long_args
# then setup crontab 'rsnapshot daily 2>&1 | rsnapreport.pl | mail -s"SUBJECT" backupadm@adm.com
# don't forget the 2>&1 or your errors will be lost to stderr
################################
## Copyright 2006 William Bear
## This program is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 2 of the License, or
## (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this program; if not, write to the Free Software
## Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
################################
use strict;
use warnings;
use English '-no_match_vars';
my $bufsz = 2;
my %bkdata=();
my @errors=();
sub pretty_print(){
my $ofh = select(STDOUT);
$FORMAT_NAME="BREPORTBODY";
$FORMAT_TOP_NAME="BREPORTHEAD";
select($ofh);
foreach my $source (sort keys %bkdata){
if($bkdata{$source} =~ /error/i) { print "ERROR $source $bkdata{$source}"; next; }
my $files = $bkdata{$source}{'files'};
my $filest = $bkdata{$source}{'files_tran'};
my $filelistgentime = $bkdata{$source}{'file_list_gen_time'};
my $filelistxfertime = $bkdata{$source}{'file_list_trans_time'};
my $bytes = $bkdata{$source}{'file_size'}/1000000; # convert to MB
my $bytest = $bkdata{$source}{'file_tran_size'}/1000000; # convert to MB
$source =~ s/^[^\@]+\@//; # remove username
format BREPORTHEAD =
SOURCE TOTAL FILES FILES TRANS TOTAL MB MB TRANS LIST GEN TIME FILE XFER TIME
--------------------------------------------------------------------------------------------------------------------
.
format BREPORTBODY =
@<<<<<<<<<<<<<<<<<<<<<<<<<<<<< @>>>>>>>>>> @>>>>>>>>>> @#########.## @########.## @>>>>>>>>>>>> @>>>>>>>>>>>>>
$source, $files, $filest, $bytes, $bytest, $filelistgentime, $filelistxfertime
.
write STDOUT;
}
}
sub nextLine($){
my($lines) = @_;
my $line = <>;
push(@$lines,$line);
return shift @$lines;
}
my @rsnapout = ();
# load readahead buffer
for(my $i=0; $i < $bufsz; $i++){
$rsnapout[$i] = <>;
}
while (my $line = nextLine(\@rsnapout)){
if($line =~ /^[\/\w]+\/rsync/) { # find start rsync command line
my @rsynccmd=();
while($line =~ /\s+\\$/){ # combine wrapped lines
$line =~ s/\\$//g;
$line .= nextLine(\@rsnapout);
}
push(@rsynccmd,split(/\s+/,$line)); # split into command components
my $source = $rsynccmd[-2]; # count backwards: source always second to last
#print $source;
while($line = nextLine(\@rsnapout)){
# this means we are missing stats info
if($line =~ /^[\/\w]+\/rsync/){
unshift(@rsnapout,$line);
push(@errors,"$source NO STATS DATA");
last;
}
# stat record
if($line =~ /^total size is\s+\d+/){ last; } # this ends the rsync stats record
# Number of files: 1,325 (reg: 387, dir: 139, link: 799)
elsif($line =~ /Number of files:\s+([\d,]+)/){
$bkdata{$source}{'files'}=$1;
$bkdata{$source}{'files'}=~ s/,//g;
}
# Number of regular files transferred: 1
elsif($line =~ /Number of (regular )?files transferred:\s+([\d,]+)/){
$bkdata{$source}{'files_tran'}=$2;
}
# Total file size: 1,865,857 bytes
elsif($line =~ /Total file size:\s+([\d,]+)/){
$bkdata{$source}{'file_size'}=$1;
$bkdata{$source}{'file_size'}=~ s/,//g;
}
elsif($line =~ /Total transferred file size:\s+([\d,]+)/){
$bkdata{$source}{'file_tran_size'}=$1;
$bkdata{$source}{'file_tran_size'}=~ s/,//g;
}
elsif($line =~ /File list generation time:\s+(.+)/){
$bkdata{$source}{'file_list_gen_time'}=$1;
}
elsif($line =~ /File list transfer time:\s+(.+)/){
$bkdata{$source}{'file_list_trans_time'}=$1;
}
elsif($line =~ /^(rsync error|ERROR): /){ push(@errors,"$source $line"); } # we encountered an rsync error
}
}
elsif($line =~ /^(rsync error|ERROR): /){ push(@errors,$line); } # we encountered an rsync error
}
pretty_print();
if(scalar @errors > 0){
print "\nERRORS\n";
print join("\n",@errors);
print "\n";
}
#!/usr/bin/env perl # this script prints a pretty report from rsnapshot output # in the rsnapshot.conf you must set # verbose >= 4 # and add --stats to rsync_long_args # then setup crontab 'rsnapshot daily 2>&1 | rsnapreport.pl | mail -s"SUBJECT" backupadm@adm.com # don't forget the 2>&1 or your errors will be lost to stderr ################################ ## Copyright 2006 William Bear ## This program is free software; you can redistribute it and/or modify ## it under the terms of the GNU General Public License as published by ## the Free Software Foundation; either version 2 of the License, or ## (at your option) any later version. ## ## This program is distributed in the hope that it will be useful, ## but WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## GNU General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with this program; if not, write to the Free Software ## Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA ################################ use strict; use warnings; use English '-no_match_vars'; my $bufsz = 2; my %bkdata=(); my @errors=(); sub pretty_print(){ my $ofh = select(STDOUT); $FORMAT_NAME="BREPORTBODY"; $FORMAT_TOP_NAME="BREPORTHEAD"; select($ofh); foreach my $source (sort keys %bkdata){ if($bkdata{$source} =~ /error/i) { print "ERROR $source $bkdata{$source}"; next; } my $files = $bkdata{$source}{'files'}; my $filest = $bkdata{$source}{'files_tran'}; my $filelistgentime = $bkdata{$source}{'file_list_gen_time'}; my $filelistxfertime = $bkdata{$source}{'file_list_trans_time'}; my $bytes = $bkdata{$source}{'file_size'}/1000000; # convert to MB my $bytest = $bkdata{$source}{'file_tran_size'}/1000000; # convert to MB $source =~ s/^[^\@]+\@//; # remove username format BREPORTHEAD = SOURCE TOTAL FILES FILES TRANS TOTAL MB MB TRANS LIST GEN TIME FILE XFER TIME -------------------------------------------------------------------------------------------------------------------- . format BREPORTBODY = @<<<<<<<<<<<<<<<<<<<<<<<<<<<<< @>>>>>>>>>> @>>>>>>>>>> @#########.## @########.## @>>>>>>>>>>>> @>>>>>>>>>>>>> $source, $files, $filest, $bytes, $bytest, $filelistgentime, $filelistxfertime . write STDOUT; } } sub nextLine($){ my($lines) = @_; my $line = <>; push(@$lines,$line); return shift @$lines; } my @rsnapout = (); # load readahead buffer for(my $i=0; $i < $bufsz; $i++){ $rsnapout[$i] = <>; } while (my $line = nextLine(\@rsnapout)){ if($line =~ /^[\/\w]+\/rsync/) { # find start rsync command line my @rsynccmd=(); while($line =~ /\s+\\$/){ # combine wrapped lines $line =~ s/\\$//g; $line .= nextLine(\@rsnapout); } push(@rsynccmd,split(/\s+/,$line)); # split into command components my $source = $rsynccmd[-2]; # count backwards: source always second to last #print $source; while($line = nextLine(\@rsnapout)){ # this means we are missing stats info if($line =~ /^[\/\w]+\/rsync/){ unshift(@rsnapout,$line); push(@errors,"$source NO STATS DATA"); last; } # stat record if($line =~ /^total size is\s+\d+/){ last; } # this ends the rsync stats record # Number of files: 1,325 (reg: 387, dir: 139, link: 799) elsif($line =~ /Number of files:\s+([\d,]+)/){ $bkdata{$source}{'files'}=$1; $bkdata{$source}{'files'}=~ s/,//g; } # Number of regular files transferred: 1 elsif($line =~ /Number of (regular )?files transferred:\s+([\d,]+)/){ $bkdata{$source}{'files_tran'}=$2; } # Total file size: 1,865,857 bytes elsif($line =~ /Total file size:\s+([\d,]+)/){ $bkdata{$source}{'file_size'}=$1; $bkdata{$source}{'file_size'}=~ s/,//g; } elsif($line =~ /Total transferred file size:\s+([\d,]+)/){ $bkdata{$source}{'file_tran_size'}=$1; $bkdata{$source}{'file_tran_size'}=~ s/,//g; } elsif($line =~ /File list generation time:\s+(.+)/){ $bkdata{$source}{'file_list_gen_time'}=$1; } elsif($line =~ /File list transfer time:\s+(.+)/){ $bkdata{$source}{'file_list_trans_time'}=$1; } elsif($line =~ /^(rsync error|ERROR): /){ push(@errors,"$source $line"); } # we encountered an rsync error } } elsif($line =~ /^(rsync error|ERROR): /){ push(@errors,$line); } # we encountered an rsync error } pretty_print(); if(scalar @errors > 0){ print "\nERRORS\n"; print join("\n",@errors); print "\n"; }
#!/usr/bin/env perl
# this script prints a pretty report from rsnapshot output
# in the rsnapshot.conf you must set
# verbose >= 4
# and add --stats to rsync_long_args
# then setup crontab 'rsnapshot daily 2>&1 | rsnapreport.pl | mail -s"SUBJECT" backupadm@adm.com
# don't forget the 2>&1 or your errors will be lost to stderr
################################
## Copyright 2006 William Bear
## This program is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 2 of the License, or
## (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this program; if not, write to the Free Software
## Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
################################
use strict;
use warnings;
use English '-no_match_vars';

my $bufsz = 2;
my %bkdata=();
my @errors=();

sub pretty_print(){
	my $ofh = select(STDOUT);
	$FORMAT_NAME="BREPORTBODY";
	$FORMAT_TOP_NAME="BREPORTHEAD";
	select($ofh);

	foreach my $source (sort keys %bkdata){
		if($bkdata{$source} =~ /error/i) { print "ERROR $source $bkdata{$source}"; next; }
		my $files = $bkdata{$source}{'files'};
		my $filest = $bkdata{$source}{'files_tran'};
		my $filelistgentime = $bkdata{$source}{'file_list_gen_time'};
		my $filelistxfertime = $bkdata{$source}{'file_list_trans_time'};
		my $bytes = $bkdata{$source}{'file_size'}/1000000; # convert to MB
		my $bytest = $bkdata{$source}{'file_tran_size'}/1000000; # convert to MB
		$source =~ s/^[^\@]+\@//; # remove username
		format BREPORTHEAD =
SOURCE                          TOTAL FILES   FILES TRANS      TOTAL MB     MB TRANS   LIST GEN TIME  FILE XFER TIME
--------------------------------------------------------------------------------------------------------------------
.
		format BREPORTBODY =
@<<<<<<<<<<<<<<<<<<<<<<<<<<<<<	@>>>>>>>>>>   @>>>>>>>>>> @#########.## @########.##   @>>>>>>>>>>>>  @>>>>>>>>>>>>>
$source,                        $files,       $filest,    $bytes,       $bytest,       $filelistgentime, $filelistxfertime
.
		write STDOUT;
	}
}

sub nextLine($){
	my($lines) = @_;
	my $line = <>;
	push(@$lines,$line);
	return shift @$lines;
}


my @rsnapout = ();

# load readahead buffer
for(my $i=0; $i < $bufsz; $i++){
	$rsnapout[$i] = <>;
}
while (my $line = nextLine(\@rsnapout)){
	if($line =~ /^[\/\w]+\/rsync/) { # find start rsync command line
		my @rsynccmd=();
		while($line =~ /\s+\\$/){ # combine wrapped lines
			$line =~ s/\\$//g;
			$line .= nextLine(\@rsnapout);
		}
		push(@rsynccmd,split(/\s+/,$line)); # split into command components
		my $source = $rsynccmd[-2]; # count backwards: source always second to last
		#print $source;
		while($line = nextLine(\@rsnapout)){
  			# this means we are missing stats info
			if($line =~ /^[\/\w]+\/rsync/){
				unshift(@rsnapout,$line);
				push(@errors,"$source NO STATS DATA");
				last;
			}
			# stat record
			if($line =~ /^total size is\s+\d+/){ last; } # this ends the rsync stats record
			# Number of files: 1,325 (reg: 387, dir: 139, link: 799)
			elsif($line =~ /Number of files:\s+([\d,]+)/){
				$bkdata{$source}{'files'}=$1;
				$bkdata{$source}{'files'}=~ s/,//g;
			}
			# Number of regular files transferred: 1
			elsif($line =~ /Number of (regular )?files transferred:\s+([\d,]+)/){
				$bkdata{$source}{'files_tran'}=$2;
			}
			# Total file size: 1,865,857 bytes
			elsif($line =~ /Total file size:\s+([\d,]+)/){
				$bkdata{$source}{'file_size'}=$1;
				$bkdata{$source}{'file_size'}=~ s/,//g;
			}
			elsif($line =~ /Total transferred file size:\s+([\d,]+)/){
				$bkdata{$source}{'file_tran_size'}=$1;
				$bkdata{$source}{'file_tran_size'}=~ s/,//g;
			}
			elsif($line =~ /File list generation time:\s+(.+)/){
				$bkdata{$source}{'file_list_gen_time'}=$1;
			}
			elsif($line =~ /File list transfer time:\s+(.+)/){
				$bkdata{$source}{'file_list_trans_time'}=$1;
			}
			elsif($line =~ /^(rsync error|ERROR): /){ push(@errors,"$source $line"); } # we encountered an rsync error
		}
	}
	elsif($line =~ /^(rsync error|ERROR): /){ push(@errors,$line); } # we encountered an rsync error
}

pretty_print();
if(scalar @errors > 0){
	print "\nERRORS\n";
	print join("\n",@errors);
	print "\n";
}