From 8aaaa8fa7cb527937859419580b6988fffe1c344 Mon Sep 17 00:00:00 2001 From: Tobias Trabelsi Date: Tue, 1 Jan 2019 19:38:24 +0100 Subject: [PATCH] mirror of https://github.com/erikw/restic-systemd-automatic-backup without B2 --- .backup_exclude | 17 +++ LICENSE | 15 ++ Makefile | 48 +++++++ README.md | 129 ++++++++++++++++++ etc/cron.d/restic | 8 ++ etc/restic/pw.txt | 1 + etc/systemd/system/restic-backup.service | 10 ++ etc/systemd/system/restic-backup.timer | 9 ++ etc/systemd/system/restic-check.service | 9 ++ etc/systemd/system/restic-check.timer | 9 ++ etc/systemd/system/status-email-user@.service | 11 ++ usr/local/sbin/restic_backup.sh | 79 +++++++++++ usr/local/sbin/restic_check.sh | 34 +++++ 13 files changed, 379 insertions(+) create mode 100644 .backup_exclude create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 etc/cron.d/restic create mode 100644 etc/restic/pw.txt create mode 100644 etc/systemd/system/restic-backup.service create mode 100644 etc/systemd/system/restic-backup.timer create mode 100644 etc/systemd/system/restic-check.service create mode 100644 etc/systemd/system/restic-check.timer create mode 100644 etc/systemd/system/status-email-user@.service create mode 100644 usr/local/sbin/restic_backup.sh create mode 100644 usr/local/sbin/restic_check.sh diff --git a/.backup_exclude b/.backup_exclude new file mode 100644 index 0000000..495a1e1 --- /dev/null +++ b/.backup_exclude @@ -0,0 +1,17 @@ +an2linux +amps-standalone-jira-7.12.3 +Android/Sdk +autostart +cantata +cef_user_data +Cerebro +chromium +Downloads +Electron +leanote +Leanote +octobox-nativefier-87a70c +ownCloud +sublime-text-3 +Unity-* +.cache \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..07e15f3 --- /dev/null +++ b/LICENSE @@ -0,0 +1,15 @@ +restic-systemd-automatic-backup - My restic backup solution using Backblaze B2 storage, systemd timers (or cron) and email notifications on failure. + + +Copyright (c) 2018, see commit log for auhtors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * Neither the name of the nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +http://opensource.org/licenses/BSD-3-Clause diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ea8ef64 --- /dev/null +++ b/Makefile @@ -0,0 +1,48 @@ +# Not file targets. +.PHONY: help install install-scripts install-conf install-exclude install-systemd + +### Macros ### +SRCS_SCRIPTS = $(filter-out %cron_mail, $(wildcard usr/local/sbin/*)) +SRCS_CONF = $(wildcard etc/restic/*) +SRCS_EXCLUDE = .backup_exclude +SRCS_SYSTEMD = $(wildcard etc/systemd/system/*) + +# Just set PREFIX in envionment, like +# $ PREFIX=/tmp/test make +#PREFIX=/ +DEST_SCRIPTS = $(PREFIX)/usr/local/sbin +DEST_CONF = $(PREFIX)/etc/restic +DEST_EXCLUDE = $(PREFIX)/ +DEST_SYSTEMD = $(PREFIX)/etc/systemd/system + + +### Targets ### +# target: all - Default target. +all: install + +# target: help - Display all targets. +help: + @egrep "#\starget:" [Mm]akefile | sed 's/\s-\s/\t\t\t/' | cut -d " " -f3- | sort -d + +# target: install - Install all files +install: install-scripts install-exclude install-systemd + + +# target: install-scripts - Install executables. +install-scripts: + install -d $(DEST_SCRIPTS) + install -m 744 $(SRCS_SCRIPTS) $(DEST_SCRIPTS) + +# target: install-conf - Install restic configuration files. +install-conf: + install -d $(DEST_CONF) -m 700 + install $(SRCS_CONF) $(DEST_CONF) + +# target: install-exclude - Install backup exclude file. +install-exclude: + install $(SRCS_EXCLUDE) $(DEST_EXCLUDE) + +# target: install-systemd - Install systemd timer and service files +install-systemd: + install -d $(DEST_SYSTEMD) + install -m 0644 $(SRCS_SYSTEMD) $(DEST_SYSTEMD) diff --git a/README.md b/README.md new file mode 100644 index 0000000..b2574e7 --- /dev/null +++ b/README.md @@ -0,0 +1,129 @@ +# Automatic restic backups using systemd services and timers + +## Restic + +[restic](https://restic.net/) is a command-line tool for making backups, the right way. Check the official website for a feature explanation. As a storage backend, I recommend [Backblaze B2](https://www.backblaze.com/b2/cloud-storage.html) as restic works well with it, and it is (at the time of writing) very affordable for the hobbyist hacker! + +Unfortunately restic does not come pre-configured with a way to run automated backups, say every day. However it's possible to set this up yourself using systemd/cron and some wrappers. This example also features email notifications when a backup fails to complete. + +Here follows a step-by step tutorial on how to set it up, with my sample script and configurations that you can modify to suit your needs. + + +Note, you can use any of the supported [storage backends](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html). The setup should be similar but you will have to use other configuration variables to match your backend of choice. + + +## Set up + +Tip: The steps in this section will instruct you to copy files from this repo to system directories. If you don't want to do this manually, you can use the Makefile: + +```bash +$ git clone https://github.com/erikw/restic-systemd-automatic-backup.git +$ cd restic-systemd-automatic-backup +$ sudo make install +```` + +### 1. Create Backblaze B2 account + +First, see this official Backblaze [tutorial](https://help.backblaze.com/hc/en-us/articles/115002880514-How-to-configure-Backblaze-B2-with-Restic-on-Linux) on restic, and follow the instructions ("Create Backblaze account with B2 enabled") there on how to create a new B2 bucket. + +Take note of the your account ID, application key and password for the next steps. + + + +### 2. Configure your B2 account locally +Put these files in `/etc/restic/`: +* `b2_env.sh`: Fill this file out with your B2 bucket settings etc. The reason for putting these in a separate file is that it can be used also for you to simply source, when you want to issue some restic commands. For example: +```bash +$ source /etc/restic/b2_env.sh +$ restic snapshots # You don't have to supply all parameters like --repo, as they are now in your environment! +```` +* `b2_pw.txt`: Put your B2 password in this file. + +### 3. Initialize remote repo +Now we must initialize the repository on the remote end: +```bash +source /etc/restic/b2_env.sh +restic init +``` + +### 4. Script for doing the backup +Put this file in `/usr/local/sbin`: +* `restic_backup.sh`: A script that defines how to run the backup. Edit this file to respect your needs in terms of backup which paths to backup, retention (number of backups to save), etc. + +Put this file in `/`: +* `.backup_exclude`: A list of file pattern paths to exclude from you backups, files that just occupy storage space, backup-time, network and money. + + +### 5. Make first backup & verify +Now see if the backup itself works, by running + +```bash +$ /usr/local/sbin/restic_backup.sh +$ restic snapshots +```` + +### 6. Backup automatically; systemd service + timer +Now we can do the modern version of a cron-job, a systemd service + timer, to run the backup every day! + + +Put these files in `/etc/systemd/system/`: +* `restic-backup.service`: A service that calls the backup script. +* `restic-backup.timer`: A timer that starts the backup every day. + + +Now simply enable the timer with: +```bash +$ systemctl start restic-backup.timer +$ systemctl enable restic-backup.timer +```` + +You can see when your next backup is scheduled to run with +```bash +$ systemctl list-timers | grep restic +``` + +and see the status of a currently running backup with + +```bash +$ systemctl status restic-backup +``` + +or start a backup manually + +```bash +$ systemctl start restic-backup +``` + +You can follow the backup stdout output live as backup is running with: + +```bash +$ journalctl -f -u restic-backup.service +```` + +(skip `-f` to see all backups that has run) + + + +### 7. Email notification on failure +We want to be aware when the automatic backup fails, so we can fix it. Since my laptop does not run a mail server, I went for a solution to set up my laptop to be able to send emails with [postfix via my Gmail](https://easyengine.io/tutorials/linux/ubuntu-postfix-gmail-smtp/). Follow the instructions over there. + +Put this file in `/usr/local/sbin`: +* `systemd-email`: Sends email using sendmail(1). This script also features time-out for not spamming Gmail servers and getting my account blocked. + +Put this files in `/etc/systemd/system/`: +* `status-email-user@.service`: A service that can notify you via email when a systemd service fails. Edit the target email address in this file. + +As you maybe noticed already before, `restic-backup.service` is configured to start `status-email-user.service` on failure. + + +### 8. Optional: automated backup checks +Once in a while it can be good to do a health check of the remote repository, to make sure it's not getting corrupt. This can be done with `$ restic check`. + +There are some `*-check*`-files in this git repo. Install these in the same way you installed the `*-backup*`-files. + + +## Cron? +If you want to run an all-classic cron job instead, do like this: + +* `etc/cron.d/restic`: Depending on your system's cron, put this in `/etc/cron.d/` or similar, or copy the contents to $(sudo crontab -e). The format of this file is tested under FreeBSD, and might need adaptions depending on your cron. +* `usr/local/sbin/cron_mail`: A wrapper for running cron jobs, that sends output of the job as an email using the mail(1) command. diff --git a/etc/cron.d/restic b/etc/cron.d/restic new file mode 100644 index 0000000..b50698f --- /dev/null +++ b/etc/cron.d/restic @@ -0,0 +1,8 @@ +SHELL=/bin/sh +PATH=/etc:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin/:/usr/local/sbin/ +# Order of crontab fields +# minute hour mday month wday command +# Reference: https://www.freebsd.org/doc/handbook/configtuning-cron.html +# Reference: crontab(5). +@weekly lerentis cron_mail restic_backup.sh +@monthly lerentis cron_mail restic_check.sh diff --git a/etc/restic/pw.txt b/etc/restic/pw.txt new file mode 100644 index 0000000..ca0cb03 --- /dev/null +++ b/etc/restic/pw.txt @@ -0,0 +1 @@ +PASSWORD GOES HERES \ No newline at end of file diff --git a/etc/systemd/system/restic-backup.service b/etc/systemd/system/restic-backup.service new file mode 100644 index 0000000..8743058 --- /dev/null +++ b/etc/systemd/system/restic-backup.service @@ -0,0 +1,10 @@ +[Unit] +Description=Backup with restic to Backblaze B2 +OnFailure=status-email-user@%n.service + +[Service] +Type=simple +Nice=10 +ExecStart=/usr/local/sbin/restic_backup.sh +# $HOME or $XDG_CACHE_HOME must be set for restic to find /root/.cache/restic/ +Environment="HOME=/home/lerentis/" \ No newline at end of file diff --git a/etc/systemd/system/restic-backup.timer b/etc/systemd/system/restic-backup.timer new file mode 100644 index 0000000..2d7401f --- /dev/null +++ b/etc/systemd/system/restic-backup.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Backup with restic on schedule + +[Timer] +OnCalendar=weekly +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/etc/systemd/system/restic-check.service b/etc/systemd/system/restic-check.service new file mode 100644 index 0000000..7f90e78 --- /dev/null +++ b/etc/systemd/system/restic-check.service @@ -0,0 +1,9 @@ +[Unit] +Description=Check restic backup Backblaze B2 for errors +OnFailure=status-email-user@%n.service +Conflicts=restic.service + +[Service] +Type=simple +Nice=10 +ExecStart=/usr/local/sbin/restic_check.sh diff --git a/etc/systemd/system/restic-check.timer b/etc/systemd/system/restic-check.timer new file mode 100644 index 0000000..294183f --- /dev/null +++ b/etc/systemd/system/restic-check.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Check restic backup for errors on a schedule + +[Timer] +OnCalendar=monthly +Persistent=true + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/etc/systemd/system/status-email-user@.service b/etc/systemd/system/status-email-user@.service new file mode 100644 index 0000000..da45d7b --- /dev/null +++ b/etc/systemd/system/status-email-user@.service @@ -0,0 +1,11 @@ +# Source: https://serverfault.com/questions/876233/how-to-send-an-email-if-a-systemd-service-is-restarted +# Source: https://wiki.archlinux.org/index.php/Systemd/Timers#MAILTO + +[Unit] +Description=Send status email for %i to user + +[Service] +Type=oneshot +ExecStart=/usr/bin/notify-send "Backup Status:" %i +User=lerentis +Group=systemd-journal diff --git a/usr/local/sbin/restic_backup.sh b/usr/local/sbin/restic_backup.sh new file mode 100644 index 0000000..992c808 --- /dev/null +++ b/usr/local/sbin/restic_backup.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Make backup my system with restic to Backblaze B2. +# This script is typically run by: /etc/systemd/system/restic-backup.{service,timer} + +# Exit on failure, pipe failure +set -e -o pipefail + +export RESTIC_PASSWORD_FILE="/etc/restic/pw.txt" +export RESTIC_REPOSITORY="sftp:lerentis@asgard.lan:/srv/dev-disk-by-id-md-name-asgard-Data/Backup/Laptop/Huginn" + +# Clean up lock if we are killed. +# If killed by systemd, like $(systemctl stop restic), then it kills the whole cgroup and all it's subprocesses. +# However if we kill this script ourselves, we need this trap that kills all subprocesses manually. +exit_hook() { + echo "In exit_hook(), being killed" >&2 + jobs -p | xargs kill + restic unlock +} +trap exit_hook INT TERM + +# How many backups to keep. +RETENTION_DAYS=14 +RETENTION_WEEKS=16 +RETENTION_MONTHS=18 +RETENTION_YEARS=3 + +# What to backup, and what to not +BACKUP_PATHS="/home/lerentis/" +BACKUP_EXCLUDES="--exclude-file /home/lerentis/.backup_exclude" +BACKUP_TAG=systemd.timer + + +# Set all environment variables like +# B2_ACCOUNT_ID, B2_ACCOUNT_KEY, RESTIC_REPOSITORY etc. +#source /etc/restic/b2_env.sh + +# How many network connections to set up to B2. Default is 5. +B2_CONNECTIONS=50 + +# NOTE start all commands in background and wait for them to finish. +# Reason: bash ignores any signals while child process is executing and thus my trap exit hook is not triggered. +# However if put in subprocesses, wait(1) waits until the process finishes OR signal is received. +# Reference: https://unix.stackexchange.com/questions/146756/forward-sigterm-to-child-in-bash + +# Remove locks from other stale processes to keep the automated backup running. +restic unlock & +wait $! + +# Do the backup! +# See restic-backup(1) or http://restic.readthedocs.io/en/latest/040_backup.html +# --one-file-system makes sure we only backup exactly those mounted file systems specified in $BACKUP_PATHS, and thus not directories like /dev, /sys etc. +# --tag lets us reference these backups later when doing restic-forget. +restic backup --verbose --tag $BACKUP_TAG $BACKUP_EXCLUDES $BACKUP_PATHS & +wait $! + +# Dereference old backups. +# See restic-forget(1) or http://restic.readthedocs.io/en/latest/060_forget.html +# --group-by only the tag and path, and not by hostname. This is because I create a B2 Bucket per host, and if this hostname accidentially change some time, there would now be multiple backup sets. +restic forget \ + --verbose \ + --tag $BACKUP_TAG \ + --group-by "paths,tags" \ + --keep-daily $RETENTION_DAYS \ + --keep-weekly $RETENTION_WEEKS \ + --keep-monthly $RETENTION_MONTHS \ + --keep-yearly $RETENTION_YEARS & +wait $! + +# Remove old data not linked anymore. +# See restic-prune(1) or http://restic.readthedocs.io/en/latest/060_forget.html +restic prune --verbose & +wait $! + +# Check repository for errors. +# NOTE this takes much time (and data transfer from remote repo?), do this in a separate systemd.timer which is run less often. +#restic check & +#wait $! + +echo "Backup & cleaning is done." diff --git a/usr/local/sbin/restic_check.sh b/usr/local/sbin/restic_check.sh new file mode 100644 index 0000000..2a7941a --- /dev/null +++ b/usr/local/sbin/restic_check.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Check my backup with restic to Backblaze B2 for errors. +# This script is typically run by: /etc/systemd/system/restic-check.{service,timer} + +# Exit on failure, pipe failure +set -e -o pipefail + +export RESTIC_PASSWORD_FILE="/etc/restic/pw.txt" +export RESTIC_REPOSITORY="sftp:lerentis@asgard.lan:/srv/dev-disk-by-id-md-name-asgard-Data/Backup/Laptop/Huginn" + +# Clean up lock if we are killed. +# If killed by systemd, like $(systemctl stop restic), then it kills the whole cgroup and all it's subprocesses. +# However if we kill this script ourselves, we need this trap that kills all subprocesses manually. +exit_hook() { + echo "In exit_hook(), being killed" >&2 + jobs -p | xargs kill + restic unlock +} +trap exit_hook INT TERM + + +#source /etc/restic/b2_env.sh + +# How many network connections to set up to B2. Default is 5. +B2_CONNECTIONS=50 + +# Remove locks from other stale processes to keep the automated backup running. +# NOTE nope, dont' unlock liek restic_backup.sh. restic_backup.sh should take preceedance over this script. +#restic unlock & +#wait $! + +# Check repository for errors. +restic check --verbose & +wait $!