mirror of https://github.com/erikw/restic-systemd-automatic-backup without B2

.backup_exclude

@@ -0,0 +1,17 @@
+an2linux
+amps-standalone-jira-7.12.3
+Android/Sdk
+autostart
+cantata
+cef_user_data
+Cerebro
+chromium
+Downloads
+Electron
+leanote
+Leanote
+octobox-nativefier-87a70c
+ownCloud
+sublime-text-3
+Unity-*
+.cache

LICENSE

@@ -0,0 +1,15 @@
+restic-systemd-automatic-backup -  My restic backup solution using Backblaze B2 storage, systemd timers (or cron) and email notifications on failure.
+
+
+Copyright (c) 2018, see commit log for auhtors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+ * Neither the name of the <ORGANIZATION> nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+http://opensource.org/licenses/BSD-3-Clause

Makefile

@@ -0,0 +1,48 @@
+# Not file targets.
+.PHONY: help install install-scripts install-conf install-exclude install-systemd
+
+### Macros ###
+SRCS_SCRIPTS	= $(filter-out %cron_mail, $(wildcard usr/local/sbin/*))
+SRCS_CONF	= $(wildcard etc/restic/*)
+SRCS_EXCLUDE	= .backup_exclude
+SRCS_SYSTEMD	= $(wildcard etc/systemd/system/*)
+
+# Just set PREFIX in envionment, like
+# $ PREFIX=/tmp/test make
+#PREFIX=/
+DEST_SCRIPTS	= $(PREFIX)/usr/local/sbin
+DEST_CONF	= $(PREFIX)/etc/restic
+DEST_EXCLUDE	= $(PREFIX)/
+DEST_SYSTEMD	= $(PREFIX)/etc/systemd/system
+
+
+### Targets ###
+# target: all - Default target.
+all: install
+
+# target: help - Display all targets.
+help:
+	@egrep "#\starget:" [Mm]akefile  | sed 's/\s-\s/\t\t\t/' | cut -d " " -f3- | sort -d
+
+# target: install - Install all files
+install: install-scripts install-exclude install-systemd
+
+
+# target: install-scripts - Install executables.
+install-scripts:
+	install -d $(DEST_SCRIPTS)
+	install -m 744 $(SRCS_SCRIPTS) $(DEST_SCRIPTS)
+
+# target: install-conf - Install restic configuration files.
+install-conf:
+	install -d $(DEST_CONF) -m 700
+	install $(SRCS_CONF) $(DEST_CONF)
+
+# target: install-exclude - Install backup exclude file.
+install-exclude:
+	install $(SRCS_EXCLUDE) $(DEST_EXCLUDE)
+
+# target: install-systemd - Install systemd timer and service files
+install-systemd:
+	install -d $(DEST_SYSTEMD)
+	install -m 0644 $(SRCS_SYSTEMD) $(DEST_SYSTEMD)

README.md

@@ -0,0 +1,129 @@
+# Automatic restic backups using systemd services and timers
+
+## Restic
+
+[restic](https://restic.net/) is a command-line tool for making backups, the right way. Check the official website for a feature explanation. As a storage backend, I recommend [Backblaze B2](https://www.backblaze.com/b2/cloud-storage.html) as restic works well with it, and it is (at the time of writing) very affordable for the hobbyist hacker!
+
+Unfortunately restic does not come pre-configured with a way to run automated backups, say every day. However it's possible to set this up yourself using systemd/cron and some wrappers. This example also features email notifications when a backup fails to complete.
+
+Here follows a step-by step tutorial on how to set it up, with my sample script and configurations that you can modify to suit your needs.
+
+
+Note, you can use any of the supported [storage backends](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html). The setup should be similar but you will have to use other configuration variables to match your backend of choice.
+
+
+## Set up
+
+Tip: The steps in this section will instruct you to copy files from this repo to system directories. If you don't want to do this manually, you can use the Makefile:
+
+```bash
+$ git clone https://github.com/erikw/restic-systemd-automatic-backup.git
+$ cd restic-systemd-automatic-backup
+$ sudo make install
+````
+
+### 1. Create Backblaze B2 account
+
+First, see this official Backblaze [tutorial](https://help.backblaze.com/hc/en-us/articles/115002880514-How-to-configure-Backblaze-B2-with-Restic-on-Linux) on restic, and follow the instructions ("Create Backblaze account with B2 enabled") there on how to create a new B2 bucket.
+
+Take note of the your account ID, application key and password for the next steps.
+
+
+
+### 2. Configure your B2 account locally
+Put these files in `/etc/restic/`:
+* `b2_env.sh`: Fill this file out with your B2 bucket settings etc. The reason for putting these in a separate file is that it can be used also for you to simply source, when you want to issue some restic commands. For example:
+```bash
+$ source /etc/restic/b2_env.sh
+$ restic snapshots    # You don't have to supply all parameters like --repo, as they are now in your environment!
+````
+* `b2_pw.txt`: Put your B2 password in this file.
+
+### 3. Initialize remote repo
+Now we must initialize the repository on the remote end:
+```bash
+source /etc/restic/b2_env.sh
+restic init
+```
+
+### 4. Script for doing the backup
+Put this file in `/usr/local/sbin`:
+* `restic_backup.sh`: A script that defines how to run the backup. Edit this file to respect your needs in terms of backup which paths to backup, retention (number of backups to save), etc.
+
+Put this file in `/`:
+* `.backup_exclude`: A list of file pattern paths to exclude from you backups, files that just occupy storage space, backup-time, network and money.
+
+
+### 5. Make first backup & verify
+Now see if the backup itself works, by running
+
+```bash
+$ /usr/local/sbin/restic_backup.sh
+$ restic snapshots
+````
+
+### 6. Backup automatically; systemd service + timer
+Now we can do the modern version of a cron-job, a systemd service + timer, to run the backup every day!
+
+
+Put these files in `/etc/systemd/system/`:
+* `restic-backup.service`: A service that calls the backup script.
+* `restic-backup.timer`: A timer that starts the backup every day.
+
+
+Now simply enable the timer with:
+```bash
+$ systemctl start restic-backup.timer
+$ systemctl enable restic-backup.timer
+````
+
+You can see when your next backup is scheduled to run with
+```bash
+$ systemctl list-timers | grep restic
+```
+
+and see the status of a currently running backup with
+
+```bash
+$ systemctl status restic-backup
+```
+
+or start a backup manually
+
+```bash
+$ systemctl start restic-backup
+```
+
+You can follow the backup stdout output live as backup is running with:
+
+```bash
+$ journalctl -f -u restic-backup.service
+````
+
+(skip `-f` to see all backups that has run)
+
+
+
+### 7. Email notification on failure
+We want to be aware when the automatic backup fails, so we can fix it. Since my laptop does not run a mail server, I went for a solution to set up my laptop to be able to send emails with [postfix via my Gmail](https://easyengine.io/tutorials/linux/ubuntu-postfix-gmail-smtp/). Follow the instructions over there.
+
+Put this file in `/usr/local/sbin`:
+* `systemd-email`: Sends email using sendmail(1). This script also features time-out for not spamming Gmail servers and getting my account blocked.
+
+Put this files in `/etc/systemd/system/`:
+* `status-email-user@.service`: A service that can notify you via email when a systemd service fails. Edit the target email address in this file.
+
+As you maybe noticed already before, `restic-backup.service` is configured to start `status-email-user.service` on failure.
+
+
+### 8. Optional: automated backup checks
+Once in a while it can be good to do a health check of the remote repository, to make sure it's not getting corrupt. This can be done with `$ restic check`.
+
+There are some `*-check*`-files in this git repo. Install these in the same way you installed the `*-backup*`-files.
+
+
+## Cron?
+If you want to run an all-classic cron job instead, do like this:
+
+* `etc/cron.d/restic`: Depending on your system's cron, put this in `/etc/cron.d/` or similar, or copy the contents to $(sudo crontab -e). The format of this file is tested under FreeBSD, and might need adaptions depending on your cron.
+* `usr/local/sbin/cron_mail`: A wrapper for running cron jobs, that sends output of the job as an email using the mail(1) command.

etc/cron.d/restic

@@ -0,0 +1,8 @@
+SHELL=/bin/sh
+PATH=/etc:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin/:/usr/local/sbin/
+# Order of crontab fields
+# minute        hour    mday    month   wday    command
+# Reference: https://www.freebsd.org/doc/handbook/configtuning-cron.html
+# Reference: crontab(5).
+@weekly 	lerentis	cron_mail restic_backup.sh
+@monthly	lerentis	cron_mail restic_check.sh

etc/restic/pw.txt

@@ -0,0 +1 @@
+PASSWORD GOES HERES

etc/systemd/system/restic-backup.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Backup with restic to Backblaze B2
+OnFailure=status-email-user@%n.service
+
+[Service]
+Type=simple
+Nice=10
+ExecStart=/usr/local/sbin/restic_backup.sh
+# $HOME or $XDG_CACHE_HOME must be set for restic to find /root/.cache/restic/
+Environment="HOME=/home/lerentis/"

etc/systemd/system/restic-backup.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Backup with restic on schedule
+
+[Timer]
+OnCalendar=weekly
+Persistent=true
+
+[Install]
+WantedBy=timers.target

etc/systemd/system/restic-check.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=Check restic backup Backblaze B2 for errors
+OnFailure=status-email-user@%n.service
+Conflicts=restic.service
+
+[Service]
+Type=simple
+Nice=10
+ExecStart=/usr/local/sbin/restic_check.sh

etc/systemd/system/restic-check.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Check restic backup for errors on a schedule
+
+[Timer]
+OnCalendar=monthly
+Persistent=true
+
+[Install]
+WantedBy=timers.target

etc/systemd/system/status-email-user@.service

@@ -0,0 +1,11 @@
+# Source: https://serverfault.com/questions/876233/how-to-send-an-email-if-a-systemd-service-is-restarted
+# Source: https://wiki.archlinux.org/index.php/Systemd/Timers#MAILTO
+
+[Unit]
+Description=Send status email for %i to user
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/notify-send "Backup Status:" %i
+User=lerentis
+Group=systemd-journal

usr/local/sbin/restic_backup.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+# Make backup my system with restic to Backblaze B2.
+# This script is typically run by: /etc/systemd/system/restic-backup.{service,timer}
+
+# Exit on failure, pipe failure
+set -e -o pipefail
+
+export RESTIC_PASSWORD_FILE="/etc/restic/pw.txt"
+export RESTIC_REPOSITORY="sftp:lerentis@asgard.lan:/srv/dev-disk-by-id-md-name-asgard-Data/Backup/Laptop/Huginn"
+
+# Clean up lock if we are killed.
+# If killed by systemd, like $(systemctl stop restic), then it kills the whole cgroup and all it's subprocesses.
+# However if we kill this script ourselves, we need this trap that kills all subprocesses manually.
+exit_hook() {
+	echo "In exit_hook(), being killed" >&2
+	jobs -p | xargs kill
+	restic unlock
+}
+trap exit_hook INT TERM
+
+# How many backups to keep.
+RETENTION_DAYS=14
+RETENTION_WEEKS=16
+RETENTION_MONTHS=18
+RETENTION_YEARS=3
+
+# What to backup, and what to not
+BACKUP_PATHS="/home/lerentis/"
+BACKUP_EXCLUDES="--exclude-file /home/lerentis/.backup_exclude"
+BACKUP_TAG=systemd.timer
+
+
+# Set all environment variables like
+# B2_ACCOUNT_ID, B2_ACCOUNT_KEY, RESTIC_REPOSITORY etc.
+#source /etc/restic/b2_env.sh
+
+# How many network connections to set up to B2. Default is 5.
+B2_CONNECTIONS=50
+
+# NOTE start all commands in background and wait for them to finish.
+# Reason: bash ignores any signals while child process is executing and thus my trap exit hook is not triggered.
+# However if put in subprocesses, wait(1) waits until the process finishes OR signal is received.
+# Reference: https://unix.stackexchange.com/questions/146756/forward-sigterm-to-child-in-bash
+
+# Remove locks from other stale processes to keep the automated backup running.
+restic unlock &
+wait $!
+
+# Do the backup!
+# See restic-backup(1) or http://restic.readthedocs.io/en/latest/040_backup.html
+# --one-file-system makes sure we only backup exactly those mounted file systems specified in $BACKUP_PATHS, and thus not directories like /dev, /sys etc.
+# --tag lets us reference these backups later when doing restic-forget.
+restic backup --verbose --tag $BACKUP_TAG $BACKUP_EXCLUDES $BACKUP_PATHS &
+wait $!
+
+# Dereference old backups.
+# See restic-forget(1) or http://restic.readthedocs.io/en/latest/060_forget.html
+# --group-by only the tag and path, and not by hostname. This is because I create a B2 Bucket per host, and if this hostname accidentially change some time, there would now be multiple backup sets.
+restic forget \
+	--verbose \
+	--tag $BACKUP_TAG \
+	--group-by "paths,tags" \
+	--keep-daily $RETENTION_DAYS \
+	--keep-weekly $RETENTION_WEEKS \
+	--keep-monthly $RETENTION_MONTHS \
+	--keep-yearly $RETENTION_YEARS &
+wait $!
+
+# Remove old data not linked anymore.
+# See restic-prune(1) or http://restic.readthedocs.io/en/latest/060_forget.html
+restic prune --verbose &
+wait $!
+
+# Check repository for errors.
+# NOTE this takes much time (and data transfer from remote repo?), do this in a separate systemd.timer which is run less often.
+#restic check &
+#wait $!
+
+echo "Backup & cleaning is done."

usr/local/sbin/restic_check.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Check my backup with  restic to Backblaze B2 for errors.
+# This script is typically run by: /etc/systemd/system/restic-check.{service,timer}
+
+# Exit on failure, pipe failure
+set -e -o pipefail
+
+export RESTIC_PASSWORD_FILE="/etc/restic/pw.txt"
+export RESTIC_REPOSITORY="sftp:lerentis@asgard.lan:/srv/dev-disk-by-id-md-name-asgard-Data/Backup/Laptop/Huginn"
+
+# Clean up lock if we are killed.
+# If killed by systemd, like $(systemctl stop restic), then it kills the whole cgroup and all it's subprocesses.
+# However if we kill this script ourselves, we need this trap that kills all subprocesses manually.
+exit_hook() {
+	echo "In exit_hook(), being killed" >&2
+	jobs -p | xargs kill
+	restic unlock
+}
+trap exit_hook INT TERM
+
+
+#source /etc/restic/b2_env.sh
+
+# How many network connections to set up to B2. Default is 5.
+B2_CONNECTIONS=50
+
+# Remove locks from other stale processes to keep the automated backup running.
+# NOTE nope, dont' unlock liek restic_backup.sh. restic_backup.sh should take preceedance over this script.
+#restic unlock &
+#wait $!
+
+# Check repository for errors.
+restic check --verbose &
+wait $!