add borg backup (#1727)
* add borg backup * lint fix * add exlclude patterns * missed in the #1726 fix for honoroit * feedback * Fix indentation * feedback * feedback * feedback Co-authored-by: Slavi Pantaleev <slavi@devture.com>development
Aine
3 years ago
committed by
GitHub
parent
b9d6f8b90f
commit
a5a3769ca9
@ -0,0 +1,56 @@
|
||||
# Setting up borg backup (optional)
|
||||
|
||||
The playbook can install and configure [borgbackup](https://www.borgbackup.org/) with [borgmatic](https://torsion.org/borgmatic/) for you.
|
||||
BorgBackup is a deduplicating backup program with optional compression and encryption.
|
||||
That means your daily incremental backups can be stored in a fraction of the space and is safe whether you store it at home or on a cloud service.
|
||||
|
||||
The backup will run based on `matrix_backup_borg_schedule` var (systemd timer calendar), default: 4am every day
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. Create ssh key on any machine:
|
||||
|
||||
```bash
|
||||
ssh-keygen -t ed25519 -N '' -f matrix-borg-backup -C matrix
|
||||
```
|
||||
|
||||
2. Add public part of that ssh key to your borg provider / server:
|
||||
|
||||
```bash
|
||||
# example to append the new PUBKEY contents, where:
|
||||
# PUBKEY is path to the public key,
|
||||
# USER is a ssh user on a provider / server
|
||||
# HOST is a ssh host of a provider / server
|
||||
cat PUBKEY | ssh USER@HOST 'dd of=.ssh/authorized_keys oflag=append conv=notrunc'
|
||||
```
|
||||
|
||||
## Adjusting the playbook configuration
|
||||
|
||||
Minimal working configuration (`inventory/host_vars/matrix.DOMAIN/vars.yml`) to enable borg backup:
|
||||
|
||||
```yaml
|
||||
matrix_backup_borg_enabled: true
|
||||
matrix_backup_borg_location_repositories:
|
||||
- USER@HOST:REPO
|
||||
matrix_backup_borg_storage_encryption_passphrase: "PASSPHRASE"
|
||||
matrix_backup_borg_ssh_key_private: |
|
||||
PRIVATE KEY
|
||||
```
|
||||
|
||||
where:
|
||||
|
||||
* USER - ssh user of a provider / server
|
||||
* HOST - ssh host of a provider / server
|
||||
* REPO - borg repository name, it will be initialized on backup start, eg: `matrix`
|
||||
* PASSPHRASE - super-secret borg passphrase, you may generate it with `pwgen -s 64 1` or use any password manager
|
||||
* PRIVATE KEY - the content of the public part of the ssh key you created before
|
||||
|
||||
Check the `roles/matrix-backup-borg/defaults/main.yml` for the full list of available options
|
||||
|
||||
## Installing
|
||||
|
||||
After configuring the playbook, run the [installation](installing.md) command again:
|
||||
|
||||
```
|
||||
ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start
|
||||
```
|
||||
@ -0,0 +1,63 @@
|
||||
---
|
||||
matrix_backup_borg_enabled: true
|
||||
|
||||
matrix_backup_borg_container_image_self_build: false
|
||||
matrix_backup_borg_docker_repo: "https://github.com/borgmatic-collective/docker-borgmatic"
|
||||
matrix_backup_borg_docker_src_files_path: "{{ matrix_base_data_path }}/borg/docker-src"
|
||||
|
||||
matrix_backup_borg_version: latest
|
||||
matrix_backup_borg_docker_image: "{{ matrix_backup_borg_docker_image_name_prefix }}etke.cc/borgmatic:{{ matrix_backup_borg_version }}"
|
||||
matrix_backup_borg_docker_image_name_prefix: "{{ 'localhost/' if matrix_backup_borg_container_image_self_build else 'registry.gitlab.com/' }}"
|
||||
matrix_backup_borg_docker_image_force_pull: "{{ matrix_backup_borg_docker_image.endswith(':latest') }}"
|
||||
|
||||
matrix_backup_borg_base_path: "{{ matrix_base_data_path }}/backup-borg"
|
||||
matrix_backup_borg_config_path: "{{ matrix_backup_borg_base_path }}/config"
|
||||
|
||||
# A list of extra arguments to pass to the container
|
||||
matrix_backup_borg_container_extra_arguments: []
|
||||
|
||||
# List of systemd services that matrix-backup-borg.service depends on
|
||||
matrix_backup_borg_systemd_required_services_list: ['docker.service']
|
||||
|
||||
# List of systemd services that matrix-backup-borg.service wants
|
||||
matrix_backup_borg_systemd_wanted_services_list: []
|
||||
|
||||
# systemd calendar configuration for backup job
|
||||
matrix_backup_borg_schedule: "*-*-* 04:00:00"
|
||||
|
||||
# what directories should be added to backup
|
||||
matrix_backup_borg_location_source_directories: []
|
||||
|
||||
# target repositories
|
||||
matrix_backup_borg_location_repositories: []
|
||||
|
||||
# exclude following paths:
|
||||
matrix_backup_borg_location_exclude_patterns: []
|
||||
|
||||
# borg encryption mode, only repokey-* is supported
|
||||
matrix_backup_borg_encryption: repokey-blake2
|
||||
|
||||
# private ssh key used to connect to the borg repo
|
||||
matrix_backup_borg_ssh_key_private: ""
|
||||
|
||||
# borg ssh command with ssh key
|
||||
matrix_backup_borg_storage_ssh_command: ssh -o "StrictHostKeyChecking accept-new" -i /etc/borgmatic.d/sshkey
|
||||
|
||||
# compression algorithm
|
||||
matrix_backup_borg_storage_compression: lz4
|
||||
|
||||
# archive name format
|
||||
matrix_backup_borg_storage_archive_name_format: "matrix-{now:%Y-%m-%d-%H%M%S}"
|
||||
|
||||
# repository passphrase
|
||||
matrix_backup_borg_storage_encryption_passphrase: ""
|
||||
|
||||
# retention configuration
|
||||
matrix_backup_borg_retention_keep_hourly: 0
|
||||
matrix_backup_borg_retention_keep_daily: 7
|
||||
matrix_backup_borg_retention_keep_weekly: 4
|
||||
matrix_backup_borg_retention_keep_monthly: 12
|
||||
matrix_backup_borg_retention_keep_yearly: 2
|
||||
|
||||
# retention prefix
|
||||
matrix_backup_borg_retention_prefix: "matrix-"
|
||||
@ -0,0 +1,4 @@
|
||||
---
|
||||
- set_fact:
|
||||
matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-backup-borg.service', 'matrix-backup-borg.timer'] }}"
|
||||
when: matrix_backup_borg_enabled|bool
|
||||
@ -0,0 +1,23 @@
|
||||
---
|
||||
|
||||
- import_tasks: "{{ role_path }}/tasks/init.yml"
|
||||
tags:
|
||||
- always
|
||||
|
||||
- import_tasks: "{{ role_path }}/tasks/validate_config.yml"
|
||||
when: "run_setup|bool and matrix_backup_borg_enabled|bool"
|
||||
tags:
|
||||
- setup-all
|
||||
- setup-backup-borg
|
||||
|
||||
- import_tasks: "{{ role_path }}/tasks/setup_install.yml"
|
||||
when: "run_setup|bool and matrix_backup_borg_enabled|bool"
|
||||
tags:
|
||||
- setup-all
|
||||
- setup-backup-borg
|
||||
|
||||
- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml"
|
||||
when: "run_setup|bool and not matrix_backup_borg_enabled|bool"
|
||||
tags:
|
||||
- setup-all
|
||||
- setup-backup-borg
|
||||
@ -0,0 +1,97 @@
|
||||
---
|
||||
- name: Ensure borg paths exist
|
||||
file:
|
||||
path: "{{ item.path }}"
|
||||
state: directory
|
||||
mode: 0750
|
||||
owner: "{{ matrix_user_username }}"
|
||||
group: "{{ matrix_user_groupname }}"
|
||||
with_items:
|
||||
- {path: "{{ matrix_backup_borg_config_path }}", when: true}
|
||||
- {path: "{{ matrix_backup_borg_docker_src_files_path }}", when: true}
|
||||
when: "item.when|bool"
|
||||
|
||||
- name: Ensure borg config is created
|
||||
template:
|
||||
src: "{{ role_path }}/templates/config.yaml.j2"
|
||||
dest: "{{ matrix_backup_borg_config_path }}/config.yaml"
|
||||
owner: "{{ matrix_user_username }}"
|
||||
group: "{{ matrix_user_groupname }}"
|
||||
mode: 0640
|
||||
|
||||
- name: Ensure borg passwd is created
|
||||
template:
|
||||
src: "{{ role_path }}/templates/passwd.j2"
|
||||
dest: "{{ matrix_backup_borg_config_path }}/passwd"
|
||||
owner: "{{ matrix_user_username }}"
|
||||
group: "{{ matrix_user_groupname }}"
|
||||
mode: 0640
|
||||
|
||||
- name: Ensure borg ssh key is created
|
||||
template:
|
||||
src: "{{ role_path }}/templates/sshkey.j2"
|
||||
dest: "{{ matrix_backup_borg_config_path }}/sshkey"
|
||||
owner: "{{ matrix_user_username }}"
|
||||
group: "{{ matrix_user_groupname }}"
|
||||
mode: 0600
|
||||
|
||||
- name: Ensure borg image is pulled
|
||||
docker_image:
|
||||
name: "{{ matrix_backup_borg_docker_image }}"
|
||||
source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
|
||||
force_source: "{{ matrix_backup_borg_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
|
||||
force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_backup_borg_docker_image_force_pull }}"
|
||||
when: "not matrix_backup_borg_container_image_self_build|bool"
|
||||
register: result
|
||||
retries: "{{ matrix_container_retries_count }}"
|
||||
delay: "{{ matrix_container_retries_delay }}"
|
||||
until: result is not failed
|
||||
|
||||
- name: Ensure borg repository is present on self-build
|
||||
git:
|
||||
repo: "{{ matrix_backup_borg_docker_repo }}"
|
||||
dest: "{{ matrix_backup_borg_docker_src_files_path }}"
|
||||
force: "yes"
|
||||
register: matrix_backup_borg_git_pull_results
|
||||
when: "matrix_backup_borg_container_image_self_build|bool"
|
||||
|
||||
- name: Ensure borg image is built
|
||||
docker_image:
|
||||
name: "{{ matrix_backup_borg_docker_image }}"
|
||||
source: build
|
||||
force_source: "{{ matrix_backup_borg_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
|
||||
force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mailer_git_pull_results.changed }}"
|
||||
build:
|
||||
dockerfile: Dockerfile
|
||||
path: "{{ matrix_backup_borg_docker_src_files_path }}"
|
||||
pull: true
|
||||
when: "matrix_backup_borg_container_image_self_build|bool"
|
||||
|
||||
- name: Ensure matrix-backup-borg.service installed
|
||||
template:
|
||||
src: "{{ role_path }}/templates/systemd/matrix-backup-borg.service.j2"
|
||||
dest: "{{ matrix_systemd_path }}/matrix-backup-borg.service"
|
||||
mode: 0644
|
||||
register: matrix_backup_borg_systemd_service_result
|
||||
|
||||
- name: Ensure matrix-backup-borg.timer installed
|
||||
template:
|
||||
src: "{{ role_path }}/templates/systemd/matrix-backup-borg.timer.j2"
|
||||
dest: "{{ matrix_systemd_path }}/matrix-backup-borg.timer"
|
||||
mode: 0644
|
||||
register: matrix_backup_borg_systemd_timer_result
|
||||
|
||||
- name: Ensure systemd reloaded after matrix-backup-borg.service installation
|
||||
service:
|
||||
daemon_reload: true
|
||||
when: "matrix_backup_borg_systemd_service_result.changed|bool"
|
||||
|
||||
- name: Ensure matrix-backup-borg.service enabled
|
||||
service:
|
||||
enabled: true
|
||||
name: matrix-backup-borg.service
|
||||
|
||||
- name: Ensure matrix-backup-borg.timer enabled
|
||||
service:
|
||||
enabled: true
|
||||
name: matrix-backup-borg.timer
|
||||
@ -0,0 +1,41 @@
|
||||
---
|
||||
- name: Check existence of matrix-backup-borg service
|
||||
stat:
|
||||
path: "{{ matrix_systemd_path }}/matrix-backup-borg.service"
|
||||
register: matrix_backup_borg_service_stat
|
||||
|
||||
- name: Ensure matrix-backup-borg is stopped
|
||||
service:
|
||||
name: matrix-backup-borg
|
||||
state: stopped
|
||||
enabled: false
|
||||
daemon_reload: true
|
||||
register: stopping_result
|
||||
when: "matrix_backup_borg_service_stat.stat.exists|bool"
|
||||
|
||||
- name: Ensure matrix-backup-borg.service doesn't exist
|
||||
file:
|
||||
path: "{{ matrix_systemd_path }}/matrix-backup-borg.service"
|
||||
state: absent
|
||||
when: "matrix_backup_borg_service_stat.stat.exists|bool"
|
||||
|
||||
- name: Ensure matrix-backup-borg.timer doesn't exist
|
||||
file:
|
||||
path: "{{ matrix_systemd_path }}/matrix-backup-borg.timer"
|
||||
state: absent
|
||||
when: "matrix_backup_borg_service_stat.stat.exists|bool"
|
||||
|
||||
- name: Ensure systemd reloaded after matrix-backup-borg.service removal
|
||||
service:
|
||||
daemon_reload: true
|
||||
when: "matrix_backup_borg_service_stat.stat.exists|bool"
|
||||
|
||||
- name: Ensure Matrix borg paths don't exist
|
||||
file:
|
||||
path: "{{ matrix_backup_borg_base_path }}"
|
||||
state: absent
|
||||
|
||||
- name: Ensure borg Docker image doesn't exist
|
||||
docker_image:
|
||||
name: "{{ matrix_backup_borg_docker_image }}"
|
||||
state: absent
|
||||
@ -0,0 +1,10 @@
|
||||
---
|
||||
- name: Fail if required settings not defined
|
||||
fail:
|
||||
msg: >-
|
||||
You need to define a required configuration setting (`{{ item }}`).
|
||||
when: "vars[item] == ''"
|
||||
with_items:
|
||||
- "matrix_backup_borg_ssh_key_private"
|
||||
- "matrix_backup_borg_location_repositories"
|
||||
- "matrix_backup_borg_storage_encryption_passphrase"
|
||||
@ -0,0 +1,32 @@
|
||||
#jinja2: lstrip_blocks: "True", trim_blocks: "True"
|
||||
|
||||
location:
|
||||
source_directories: {{ matrix_backup_borg_location_source_directories|to_json }}
|
||||
repositories: {{ matrix_backup_borg_location_repositories|to_json }}
|
||||
one_file_system: true
|
||||
exclude_patterns: {{ matrix_backup_borg_location_exclude_patterns|to_json }}
|
||||
|
||||
storage:
|
||||
compression: {{ matrix_backup_borg_storage_compression }}
|
||||
ssh_command: {{ matrix_backup_borg_storage_ssh_command }}
|
||||
archive_name_format: '{{ matrix_backup_borg_storage_archive_name_format }}'
|
||||
encryption_passphrase: {{ matrix_backup_borg_storage_encryption_passphrase }}
|
||||
|
||||
retention:
|
||||
keep_hourly: {{ matrix_backup_borg_retention_keep_hourly }}
|
||||
keep_daily: {{ matrix_backup_borg_retention_keep_daily }}
|
||||
keep_weekly: {{ matrix_backup_borg_retention_keep_weekly }}
|
||||
keep_monthly: {{ matrix_backup_borg_retention_keep_monthly }}
|
||||
keep_yearly: {{ matrix_backup_borg_retention_keep_yearly }}
|
||||
prefix: '{{ matrix_backup_borg_retention_prefix }}'
|
||||
|
||||
consistency:
|
||||
checks:
|
||||
- repository
|
||||
- archives
|
||||
|
||||
hooks:
|
||||
after_backup:
|
||||
- echo "Backup created."
|
||||
on_error:
|
||||
- echo "Error while creating a backup."
|
||||
@ -0,0 +1,29 @@
|
||||
{# the passwd file with correct username, UID and GID is mandatory to work with borg over ssh, otherwise ssh connections will fail #}
|
||||
root:x:0:0:root:/root:/bin/ash
|
||||
bin:x:1:1:bin:/bin:/sbin/nologin
|
||||
daemon:x:2:2:daemon:/sbin:/sbin/nologin
|
||||
adm:x:3:4:adm:/var/adm:/sbin/nologin
|
||||
lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin
|
||||
sync:x:5:0:sync:/sbin:/bin/sync
|
||||
shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown
|
||||
halt:x:7:0:halt:/sbin:/sbin/halt
|
||||
mail:x:8:12:mail:/var/mail:/sbin/nologin
|
||||
news:x:9:13:news:/usr/lib/news:/sbin/nologin
|
||||
uucp:x:10:14:uucp:/var/spool/uucppublic:/sbin/nologin
|
||||
operator:x:11:0:operator:/root:/sbin/nologin
|
||||
man:x:13:15:man:/usr/man:/sbin/nologin
|
||||
postmaster:x:14:12:postmaster:/var/mail:/sbin/nologin
|
||||
cron:x:16:16:cron:/var/spool/cron:/sbin/nologin
|
||||
ftp:x:21:21::/var/lib/ftp:/sbin/nologin
|
||||
sshd:x:22:22:sshd:/dev/null:/sbin/nologin
|
||||
at:x:25:25:at:/var/spool/cron/atjobs:/sbin/nologin
|
||||
squid:x:31:31:Squid:/var/cache/squid:/sbin/nologin
|
||||
xfs:x:33:33:X Font Server:/etc/X11/fs:/sbin/nologin
|
||||
games:x:35:35:games:/usr/games:/sbin/nologin
|
||||
cyrus:x:85:12::/usr/cyrus:/sbin/nologin
|
||||
vpopmail:x:89:89::/var/vpopmail:/sbin/nologin
|
||||
ntp:x:123:123:NTP:/var/empty:/sbin/nologin
|
||||
smmsp:x:209:209:smmsp:/var/spool/mqueue:/sbin/nologin
|
||||
guest:x:405:100:guest:/dev/null:/sbin/nologin
|
||||
{{ matrix_user_username }}:x:{{ matrix_user_uid }}:{{ matrix_user_gid }}:Matrix:/tmp:/bin/ash
|
||||
nobody:x:65534:65534:nobody:/:/sbin/nologin
|
||||
@ -0,0 +1 @@
|
||||
{{ matrix_backup_borg_ssh_key_private }}
|
||||
@ -0,0 +1,58 @@
|
||||
#jinja2: lstrip_blocks: "True"
|
||||
[Unit]
|
||||
Description=Matrix Borg Backup
|
||||
{% for service in matrix_backup_borg_systemd_required_services_list %}
|
||||
Requires={{ service }}
|
||||
After={{ service }}
|
||||
{% endfor %}
|
||||
{% for service in matrix_backup_borg_systemd_wanted_services_list %}
|
||||
Wants={{ service }}
|
||||
{% endfor %}
|
||||
DefaultDependencies=no
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
Environment="HOME={{ matrix_systemd_unit_home_path }}"
|
||||
ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-backup-borg 2>/dev/null'
|
||||
ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-backup-borg 2>/dev/null'
|
||||
ExecStartPre=-{{ matrix_host_command_docker }} run --rm --name matrix-backup-borg \
|
||||
--log-driver=none \
|
||||
--cap-drop=ALL \
|
||||
--read-only \
|
||||
--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
|
||||
--network={{ matrix_docker_network }} \
|
||||
--tmpfs=/tmp:rw,noexec,nosuid,size=100m \
|
||||
--mount type=bind,src={{ matrix_backup_borg_config_path }}/passwd,dst=/etc/passwd,ro \
|
||||
--mount type=bind,src={{ matrix_backup_borg_config_path }},dst=/etc/borgmatic.d,ro \
|
||||
{% for source in matrix_backup_borg_location_source_directories %}
|
||||
--mount type=bind,src={{ source }},dst={{ source }},ro \
|
||||
{% endfor %}
|
||||
{% for arg in matrix_backup_borg_container_extra_arguments %}
|
||||
{{ arg }} \
|
||||
{% endfor %}
|
||||
{{ matrix_backup_borg_docker_image }} \
|
||||
sh -c "borgmatic --init --encryption {{ matrix_backup_borg_encryption }}"
|
||||
|
||||
ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-backup-borg \
|
||||
--log-driver=none \
|
||||
--cap-drop=ALL \
|
||||
--read-only \
|
||||
--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
|
||||
--network={{ matrix_docker_network }} \
|
||||
--tmpfs=/tmp:rw,noexec,nosuid,size=100m \
|
||||
--mount type=bind,src={{ matrix_backup_borg_config_path }}/passwd,dst=/etc/passwd,ro \
|
||||
--mount type=bind,src={{ matrix_backup_borg_config_path }},dst=/etc/borgmatic.d,ro \
|
||||
{% for source in matrix_backup_borg_location_source_directories %}
|
||||
--mount type=bind,src={{ source }},dst={{ source }},ro \
|
||||
{% endfor %}
|
||||
{% for arg in matrix_backup_borg_container_extra_arguments %}
|
||||
{{ arg }} \
|
||||
{% endfor %}
|
||||
{{ matrix_backup_borg_docker_image }}
|
||||
|
||||
ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-backup-borg 2>/dev/null'
|
||||
ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-backup-borg 2>/dev/null'
|
||||
SyslogIdentifier=matrix-backup-borg
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Matrix Borg Backup timer
|
||||
|
||||
[Timer]
|
||||
Unit=matrix-backup-borg.service
|
||||
OnCalendar={{ matrix_backup_borg_schedule }}
|
||||
RandomizedDelaySec=2h
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
Loading…
Reference in new issue