Merge pull request #2561 from etkecc/synapse-auto-compressor
add synapse_auto_compressorhousekeeping^2
Slavi Pantaleev
2 years ago
committed by
GitHub
commit
e5c5a9b02b
@ -0,0 +1,36 @@
|
||||
# Setting up synapse_auto_compressor
|
||||
|
||||
The playbook can install and configure [synapse_auto_compressor](https://github.com/matrix-org/rust-synapse-compress-state/#automated-tool-synapse_auto_compressor) for you.
|
||||
|
||||
It's a CLI tool that automatically compresses Synapse's `state_groups` database table in the background.
|
||||
|
||||
See the project's [documentation](https://github.com/matrix-org/rust-synapse-compress-state/#automated-tool-synapse_auto_compressor) to learn what it does and why it might be useful to you.
|
||||
|
||||
|
||||
## Adjusting the playbook configuration
|
||||
|
||||
Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars.yml` file:
|
||||
|
||||
```yaml
|
||||
matrix_synapse_auto_compressor_enabled: true
|
||||
```
|
||||
|
||||
|
||||
## Installing
|
||||
|
||||
After configuring the playbook, run the [installation](installing.md) command again:
|
||||
|
||||
```
|
||||
ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start
|
||||
```
|
||||
|
||||
|
||||
## Usage
|
||||
|
||||
After installation, `synapse_auto_compressor` will run automatically every day at `00:00:00` (as defined in `matrix_synapse_auto_compressor_calendar` by default).
|
||||
|
||||
## Manually start the tool
|
||||
|
||||
For testing your setup it can be helpful to not wait until 00:00. If you want to run the tool immediately, log onto the server
|
||||
and run `systemctl start matrix-synapse-auto-compressor`. Running this command will not return control to your terminal until the compression run is done, which may take a long time.
|
||||
Consider using [tmux](https://en.wikipedia.org/wiki/Tmux) if your SSH connection is unstable.
|
||||
@ -0,0 +1,64 @@
|
||||
---
|
||||
|
||||
# synapse_auto_compressor tool
|
||||
# Project source code URL: https://github.com/matrix-org/rust-synapse-compress-state
|
||||
|
||||
matrix_synapse_auto_compressor_enabled: true
|
||||
|
||||
matrix_synapse_auto_compressor_version: v0.1.3
|
||||
|
||||
matrix_synapse_auto_compressor_base_path: "{{ matrix_base_data_path }}/synapse-auto-compressor"
|
||||
matrix_synapse_auto_compressor_container_src_files_path: "{{ matrix_synapse_auto_compressor_base_path }}/container-src"
|
||||
|
||||
matrix_synapse_auto_compressor_container_image_self_build: false
|
||||
matrix_synapse_auto_compressor_container_image_self_build_repo: "https://github.com/matrix-org/rust-synapse-compress-state.git"
|
||||
matrix_synapse_auto_compressor_container_image_self_build_version: "{{ 'main' if matrix_synapse_auto_compressor_version == 'latest' else matrix_synapse_auto_compressor_version }}"
|
||||
|
||||
matrix_synapse_auto_compressor_container_image: "{{ matrix_synapse_auto_compressor_container_image_name_prefix }}etke.cc/rust-synapse-compress-state:{{ matrix_synapse_auto_compressor_version }}"
|
||||
matrix_synapse_auto_compressor_container_image_name_prefix: "{{ 'localhost/' if matrix_synapse_auto_compressor_container_image_self_build else 'registry.gitlab.com/' }}"
|
||||
matrix_synapse_auto_compressor_container_image_force_pull: "{{ matrix_synapse_auto_compressor_container_image.endswith(':latest') }}"
|
||||
|
||||
# The base container network. It will be auto-created by this role if it doesn't exist already.
|
||||
matrix_synapse_auto_compressor_container_network: matrix-synapse-auto-compressor
|
||||
|
||||
# A list of additional container networks that the container would be connected to.
|
||||
# The role does not create these networks, so make sure they already exist.
|
||||
# Use this to expose this container to another reverse proxy, which runs in a different container network.
|
||||
matrix_synapse_auto_compressor_container_additional_networks: []
|
||||
|
||||
# A list of extra arguments to pass to the container
|
||||
matrix_synapse_auto_compressor_container_extra_arguments: []
|
||||
|
||||
# List of systemd services that matrix-synapse-auto-compressor.service depends on
|
||||
matrix_synapse_auto_compressor_systemd_required_services_list: ['docker.service']
|
||||
|
||||
# List of systemd services that matrix-synapse-auto-compressor.service wants
|
||||
matrix_synapse_auto_compressor_systemd_wanted_services_list: []
|
||||
|
||||
matrix_synapse_auto_compressor_database_username: 'synapse'
|
||||
matrix_synapse_auto_compressor_database_password: ''
|
||||
matrix_synapse_auto_compressor_database_hostname: ''
|
||||
matrix_synapse_auto_compressor_database_port: 5432
|
||||
matrix_synapse_auto_compressor_database_name: 'synapse'
|
||||
|
||||
# connection string to synapse database (postgres only)
|
||||
matrix_synapse_auto_compressor_synapse_database: 'postgres://{{ matrix_synapse_auto_compressor_database_username }}:{{ matrix_synapse_auto_compressor_database_password }}@{{ matrix_synapse_auto_compressor_database_hostname }}:{{ matrix_synapse_auto_compressor_database_port }}/{{ matrix_synapse_auto_compressor_database_name }}'
|
||||
|
||||
# systemd calendar configuration for the compressor job
|
||||
matrix_synapse_auto_compressor_calendar: "*-*-* 00:00:00"
|
||||
|
||||
# The number of state groups to work on at once.
|
||||
# All of the entries from state_groups_state are requested from the database for state groups that are worked on.
|
||||
# Therefore small chunk sizes may be needed on machines with low memory.
|
||||
# Note: if the compressor fails to find space savings on the chunk as a whole
|
||||
# (which may well happen in rooms with lots of backfill in) then the entire chunk is skipped.
|
||||
matrix_synapse_auto_compressor_chunk_size: 500
|
||||
|
||||
# CHUNKS_TO_COMPRESS chunks of size CHUNK_SIZE will be compressed.
|
||||
# The higher this number is set to, the longer the compressor will run for.
|
||||
matrix_synapse_auto_compressor_chunks_to_compress: 100
|
||||
|
||||
matrix_synapse_auto_compressor_command: "synapse_auto_compressor -p $POSTGRES_LOCATION -c {{ matrix_synapse_auto_compressor_chunk_size }} -n {{ matrix_synapse_auto_compressor_chunks_to_compress }}"
|
||||
|
||||
# Controls the POSTGRES_LOCATION environment variable
|
||||
matrix_synapse_auto_compressor_environment_variable_postgres_location: "{{ matrix_synapse_auto_compressor_synapse_database }}"
|
||||
@ -0,0 +1,72 @@
|
||||
---
|
||||
|
||||
- name: Ensure synapse-auto-compressor paths exist
|
||||
ansible.builtin.file:
|
||||
path: "{{ item.path }}"
|
||||
state: directory
|
||||
mode: 0750
|
||||
owner: "{{ matrix_user_username }}"
|
||||
group: "{{ matrix_user_groupname }}"
|
||||
when: item.when | bool
|
||||
with_items:
|
||||
- path: "{{ matrix_synapse_auto_compressor_base_path }}"
|
||||
when: true
|
||||
- path: "{{ matrix_synapse_auto_compressor_container_src_files_path }}"
|
||||
when: "{{ matrix_synapse_auto_compressor_container_image_self_build }}"
|
||||
|
||||
- name: Ensure synapse-auto-compressor labels installed
|
||||
ansible.builtin.template:
|
||||
src: "{{ role_path }}/templates/env.j2"
|
||||
dest: "{{ matrix_synapse_auto_compressor_base_path }}/env"
|
||||
mode: 0640
|
||||
owner: "{{ matrix_user_username }}"
|
||||
group: "{{ matrix_user_groupname }}"
|
||||
|
||||
- name: Ensure synapse-auto-compressor image is pulled
|
||||
community.docker.docker_image:
|
||||
name: "{{ matrix_synapse_auto_compressor_container_image }}"
|
||||
source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
|
||||
force_source: "{{ matrix_synapse_auto_compressor_container_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
|
||||
force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_synapse_auto_compressor_container_image_force_pull }}"
|
||||
when: "not matrix_synapse_auto_compressor_container_image_self_build | bool"
|
||||
register: result
|
||||
retries: "{{ devture_playbook_help_container_retries_count }}"
|
||||
delay: "{{ devture_playbook_help_container_retries_delay }}"
|
||||
until: result is not failed
|
||||
|
||||
- name: Ensure synapse-auto-compressor repository is present on self-build
|
||||
ansible.builtin.git:
|
||||
repo: "{{ matrix_synapse_auto_compressor_container_image_self_build_repo }}"
|
||||
version: "{{ matrix_synapse_auto_compressor_container_image_self_build_version }}"
|
||||
dest: "{{ matrix_synapse_auto_compressor_container_src_files_path }}"
|
||||
force: "yes"
|
||||
become: true
|
||||
become_user: "{{ matrix_user_username }}"
|
||||
register: matrix_synapse_auto_compressor_git_pull_results
|
||||
when: "matrix_synapse_auto_compressor_container_image_self_build | bool"
|
||||
|
||||
- name: Ensure synapse-auto-compressor image is built
|
||||
community.docker.docker_image:
|
||||
name: "{{ matrix_synapse_auto_compressor_container_image }}"
|
||||
source: build
|
||||
force_source: "{{ matrix_synapse_auto_compressor_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
|
||||
force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mailer_git_pull_results.changed }}"
|
||||
build:
|
||||
dockerfile: Dockerfile
|
||||
path: "{{ matrix_synapse_auto_compressor_container_src_files_path }}"
|
||||
pull: true
|
||||
when: "matrix_synapse_auto_compressor_container_image_self_build | bool"
|
||||
|
||||
- name: Ensure matrix-synapse-auto-compressor container network is created
|
||||
community.general.docker_network:
|
||||
name: "{{ matrix_synapse_auto_compressor_container_network }}"
|
||||
driver: bridge
|
||||
|
||||
- name: Ensure matrix-synapse-auto-compressor systemd service and timer are installed
|
||||
ansible.builtin.template:
|
||||
src: "{{ role_path }}/templates/matrix-synapse-auto-compressor.{{ item }}.j2"
|
||||
dest: "{{ devture_systemd_docker_base_systemd_path }}/matrix-synapse-auto-compressor.{{ item }}"
|
||||
mode: 0644
|
||||
with_items:
|
||||
- service
|
||||
- timer
|
||||
@ -0,0 +1,20 @@
|
||||
---
|
||||
|
||||
- block:
|
||||
- when: matrix_synapse_auto_compressor_enabled | bool
|
||||
ansible.builtin.include_tasks: "{{ role_path }}/tasks/validate_config.yml"
|
||||
|
||||
- when: matrix_synapse_auto_compressor_enabled | bool
|
||||
ansible.builtin.include_tasks: "{{ role_path }}/tasks/install.yml"
|
||||
tags:
|
||||
- setup-all
|
||||
- setup-synapse-auto-compressor
|
||||
- install-all
|
||||
- install-synapse-auto-compressor
|
||||
|
||||
- block:
|
||||
- when: not matrix_synapse_auto_compressor_enabled | bool
|
||||
ansible.builtin.include_tasks: "{{ role_path }}/tasks/uninstall.yml"
|
||||
tags:
|
||||
- setup-all
|
||||
- setup-synapse-auto-compressor
|
||||
@ -0,0 +1,29 @@
|
||||
---
|
||||
- name: Check existence of matrix-synapse-auto-compressor service
|
||||
ansible.builtin.stat:
|
||||
path: "{{ devture_systemd_docker_base_systemd_path }}/matrix-synapse-auto-compressor.service"
|
||||
register: matrix_synapse_auto_compressor_service_stat
|
||||
|
||||
- when: matrix_synapse_auto_compressor_service_stat.stat.exists | bool
|
||||
block:
|
||||
- name: Ensure matrix-synapse-auto-compressor is stopped
|
||||
ansible.builtin.service:
|
||||
name: matrix-synapse-auto-compressor
|
||||
state: stopped
|
||||
enabled: false
|
||||
daemon_reload: true
|
||||
|
||||
- name: Ensure matrix-synapse-auto-compressor.service doesn't exist
|
||||
ansible.builtin.file:
|
||||
path: "{{ devture_systemd_docker_base_systemd_path }}/matrix-synapse-auto-compressor.service"
|
||||
state: absent
|
||||
|
||||
- name: Ensure matrix-synapse-auto-compressor.timer doesn't exist
|
||||
ansible.builtin.file:
|
||||
path: "{{ devture_systemd_docker_base_systemd_path }}/matrix-synapse-auto-compressor.timer"
|
||||
state: absent
|
||||
|
||||
- name: Ensure Matrix synapse-auto-compressor paths don't exist
|
||||
ansible.builtin.file:
|
||||
path: "{{ matrix_synapse_auto_compressor_base_path }}"
|
||||
state: absent
|
||||
@ -0,0 +1,10 @@
|
||||
---
|
||||
|
||||
- name: Fail if required matrix-synapse-auto-compressor settings not defined
|
||||
ansible.builtin.fail:
|
||||
msg: >
|
||||
You need to define a required configuration setting (`{{ item }}`).
|
||||
when: "vars[item] == ''"
|
||||
with_items:
|
||||
- matrix_synapse_auto_compressor_database_hostname
|
||||
- matrix_synapse_auto_compressor_database_password
|
||||
@ -0,0 +1 @@
|
||||
POSTGRES_LOCATION={{ matrix_synapse_auto_compressor_environment_variable_postgres_location }}
|
||||
@ -0,0 +1,46 @@
|
||||
#jinja2: lstrip_blocks: "True"
|
||||
[Unit]
|
||||
Description=Synapse State Auto Compressor
|
||||
{% for service in matrix_synapse_auto_compressor_systemd_required_services_list %}
|
||||
Requires={{ service }}
|
||||
After={{ service }}
|
||||
{% endfor %}
|
||||
{% for service in matrix_synapse_auto_compressor_systemd_wanted_services_list %}
|
||||
Wants={{ service }}
|
||||
{% endfor %}
|
||||
DefaultDependencies=no
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
Environment="HOME={{ devture_systemd_docker_base_systemd_unit_home_path }}"
|
||||
ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} kill matrix-synapse-auto-compressor 2>/dev/null || true'
|
||||
ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} rm matrix-synapse-auto-compressor 2>/dev/null || true'
|
||||
|
||||
ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} create \
|
||||
--rm \
|
||||
--name=matrix-synapse-auto-compressor \
|
||||
--log-driver=none \
|
||||
--cap-drop=ALL \
|
||||
--read-only \
|
||||
--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
|
||||
--network={{ matrix_synapse_auto_compressor_container_network }} \
|
||||
--env-file={{ matrix_synapse_auto_compressor_base_path }}/env \
|
||||
--entrypoint=/bin/sh \
|
||||
{% for arg in matrix_synapse_auto_compressor_container_extra_arguments %}
|
||||
{{ arg }} \
|
||||
{% endfor %}
|
||||
{{ matrix_synapse_auto_compressor_container_image }} \
|
||||
-c '{{ matrix_synapse_auto_compressor_command }}'
|
||||
|
||||
{% for network in matrix_synapse_auto_compressor_container_additional_networks %}
|
||||
ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} network connect {{ network }} matrix-synapse-auto-compressor
|
||||
{% endfor %}
|
||||
|
||||
ExecStart={{ devture_systemd_docker_base_host_command_docker }} start --attach matrix-synapse-auto-compressor
|
||||
|
||||
ExecStop=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} kill matrix-synapse-auto-compressor 2>/dev/null || true'
|
||||
ExecStop=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} rm matrix-synapse-auto-compressor 2>/dev/null || true'
|
||||
SyslogIdentifier=matrix-synapse-auto-compressor
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@ -0,0 +1,9 @@
|
||||
[Unit]
|
||||
Description=Synapse State Auto Compressor Timer
|
||||
|
||||
[Timer]
|
||||
Unit=matrix-synapse-auto-compressor.service
|
||||
OnCalendar={{ matrix_synapse_auto_compressor_calendar }}
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
Loading…
Reference in new issue