Slavi Pantaleev
5 years ago
parent
631a14bf0c
commit
0ca21d80d7
@ -0,0 +1,72 @@
|
||||
# Synapse maintenance
|
||||
|
||||
This document shows you how to perform various maintenance tasks related to the Synapse chat server.
|
||||
|
||||
Table of contents:
|
||||
|
||||
- [Purging unused data with synapse-janitor](#purging-unused-data-with-synapse-janitor), for when you wish to delete unused data from the Synapse database
|
||||
|
||||
- [Purging old data with the Purge History API](#purging-old-data-with-the-purge-history-api), for when you wish to delete in-use (but old) data from the Synapse database
|
||||
|
||||
- [Compressing state with rust-synapse-compress-state](#compressing-state-with-rust-synapse-compress-state), for when you wish to compress some Synapse state tables using the [rust-synapse-compress-state](https://github.com/matrix-org/rust-synapse-compress-state) tool
|
||||
|
||||
|
||||
## Purging unused data with synapse-janitor
|
||||
|
||||
When you **leave** and **forget** a room, Synapse can clean up its data, but currently doesn't.
|
||||
This **unused and unreachable data** remains in your database forever.
|
||||
|
||||
There are external tools (like [synapse-janitor](https://github.com/xwiki-labs/synapse_scripts)), which are meant to solve this problem.
|
||||
|
||||
To ask the playbook to run synapse-janitor, execute:
|
||||
|
||||
```bash
|
||||
ansible-playbook -i inventory/hosts setup.yml --tags=run-postgres-synapse-janitor
|
||||
```
|
||||
|
||||
**Note**: this will automatically stop Synapse temporarily and restart it later.
|
||||
|
||||
|
||||
### Vacuuming Postgres
|
||||
|
||||
Running synapse-janitor potentially deletes a lot of data from the Postgres database.
|
||||
However, disk space only ever gets released after a [`FULL` Postgres `VACUUM`](./maintenance-postgres.md#vacuuming-postgresql).
|
||||
|
||||
It's easiest if you ask the playbook to run both synapse-janitor and a `VACUUM FULL` in one call:
|
||||
|
||||
```bash
|
||||
ansible-playbook -i inventory/hosts setup.yml --tags=run-postgres-synapse-janitor,run-postgres-vacuum
|
||||
```
|
||||
|
||||
**Note**: this will automatically stop Synapse temporarily and restart it later. You'll also need plenty of available disk space in your Postgres data directory (usually `/matrix/postgres/data`).
|
||||
|
||||
|
||||
## Purging old data with the Purge History API
|
||||
|
||||
If [purging unused and unreachable data](#purging-unused-data-with-synapse-janitor) is not enough for you, you can start deleting in-use (but old) data.
|
||||
|
||||
**This is destructive** (especially for non-federated rooms), because it means **people will no longer have access to history past a certain point**.
|
||||
|
||||
Synapse provides a [Purge History API](https://github.com/matrix-org/synapse/blob/master/docs/admin_api/purge_history_api.rst) that you can use to purge on a per-room basis.
|
||||
|
||||
To make use of this API, **you'll need an admin access token** first. You can find your access token in the setting of some clients (like riot-web).
|
||||
Alternatively, you can log in and obtain a new access token like this:
|
||||
|
||||
```
|
||||
curl \
|
||||
--data '{"identifier": {"type": "m.id.user", "user": "YOUR_MATRIX_USERNAME" }, "password": "YOUR_MATRIX_PASSWORD", "type": "m.login.password", "device_id": "Synapse-Purge-History-API"}' \
|
||||
https://matrix.DOMAIN/_matrix/client/r0/login
|
||||
```
|
||||
|
||||
Follow the [Purge History API](https://github.com/matrix-org/synapse/blob/master/docs/admin_api/purge_history_api.rst) documentation page for the actual purging instructions.
|
||||
|
||||
Don't forget that disk space only ever gets released after a [`FULL` Postgres `VACUUM`](./maintenance-postgres.md#vacuuming-postgresql) - something the playbook can help you with.
|
||||
|
||||
|
||||
## Compressing state with rust-synapse-compress-state
|
||||
|
||||
[rust-synapse-compress-state](https://github.com/matrix-org/rust-synapse-compress-state) can be used to optimize some `_state` tables used by Synapse.
|
||||
|
||||
Unfortunately, at this time the playbook can't help you run this **experimental tool**.
|
||||
|
||||
Since it's also experimental, you may wish to stay away from it, or at least [make Postgres backups](./docs/maintenance-postgres.md#backing-up-postgresql) first.
|
||||
@ -0,0 +1,110 @@
|
||||
---
|
||||
|
||||
# Pre-checks
|
||||
|
||||
- name: Fail if Postgres not enabled
|
||||
fail:
|
||||
msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot run synapse-janitor."
|
||||
when: "not matrix_postgres_enabled|bool"
|
||||
|
||||
|
||||
# Defaults
|
||||
|
||||
- name: Set postgres_start_wait_time, if not provided
|
||||
set_fact:
|
||||
postgres_start_wait_time: 15
|
||||
when: "postgres_start_wait_time|default('') == ''"
|
||||
|
||||
- name: Set postgres_synapse_janitor_wait_time, if not provided
|
||||
set_fact:
|
||||
postgres_synapse_janitor_wait_time: "{{ 7 * 86400 }}"
|
||||
when: "postgres_synapse_janitor_wait_time|default('') == ''"
|
||||
|
||||
- name: Set postgres_synapse_janitor_tool_path, if not provided
|
||||
set_fact:
|
||||
postgres_synapse_janitor_tool_path: "{{ matrix_postgres_base_path }}/synapse_janitor.sql"
|
||||
when: "postgres_synapse_janitor_tool_path|default('') == ''"
|
||||
|
||||
|
||||
# Actual janitor work
|
||||
|
||||
- name: Download synapse-janitor tool
|
||||
get_url:
|
||||
url: "{{ matrix_postgres_tool_synapse_janitor }}"
|
||||
dest: "{{ postgres_synapse_janitor_tool_path }}"
|
||||
force: true
|
||||
mode: 0550
|
||||
owner: "{{ matrix_user_username }}"
|
||||
group: "{{ matrix_user_username }}"
|
||||
|
||||
- name: Ensure matrix-postgres is started
|
||||
service:
|
||||
name: matrix-postgres
|
||||
state: started
|
||||
daemon_reload: yes
|
||||
|
||||
- name: Wait a bit, so that Postgres can start
|
||||
wait_for:
|
||||
timeout: "{{ postgres_start_wait_time }}"
|
||||
delegate_to: 127.0.0.1
|
||||
become: false
|
||||
|
||||
- import_tasks: tasks/util/detect_existing_postgres_version.yml
|
||||
|
||||
- name: Abort, if no existing Postgres version detected
|
||||
fail:
|
||||
msg: "Could not find existing Postgres installation"
|
||||
when: "not matrix_postgres_detected_existing|bool"
|
||||
|
||||
- name: Generate Postgres database synapse-janitor command
|
||||
set_fact:
|
||||
matrix_postgres_synapse_janitor_command: >-
|
||||
/usr/bin/docker run --rm --name matrix-postgres-synapse-janitor
|
||||
--user={{ matrix_user_uid }}:{{ matrix_user_gid }}
|
||||
--cap-drop=ALL
|
||||
--network={{ matrix_docker_network }}
|
||||
--env-file={{ matrix_postgres_base_path }}/env-postgres-psql
|
||||
--mount type=bind,src={{ postgres_synapse_janitor_tool_path }},dst=/synapse_janitor.sql,ro=true
|
||||
{{ matrix_postgres_docker_image_latest }}
|
||||
psql -v ON_ERROR_STOP=1 -h matrix-postgres {{ matrix_synapse_database_database }} -f /synapse_janitor.sql
|
||||
|
||||
- name: Note about Postgres purging alternative
|
||||
debug:
|
||||
msg: >-
|
||||
Running synapse-janitor with the following Postgres command: `{{ matrix_postgres_synapse_janitor_command }}`.
|
||||
If this crashes, you can stop all processes (`systemctl stop matrix-*`),
|
||||
start Postgres only (`systemctl start matrix-postgres`)
|
||||
and manually run the above command directly on the server.
|
||||
|
||||
- name: Populate service facts
|
||||
service_facts:
|
||||
|
||||
- set_fact:
|
||||
matrix_postgres_synapse_was_running: "{{ ansible_facts.services['matrix-synapse.service']|default(none) is not none and ansible_facts.services['matrix-synapse.service'].state == 'running' }}"
|
||||
|
||||
- name: Ensure matrix-synapse is stopped
|
||||
service:
|
||||
name: matrix-synapse
|
||||
state: stopped
|
||||
daemon_reload: yes
|
||||
|
||||
- name: Run synapse-janitor
|
||||
command: "{{ matrix_postgres_synapse_janitor_command }}"
|
||||
async: "{{ postgres_synapse_janitor_wait_time }}"
|
||||
poll: 10
|
||||
register: matrix_postgres_synapse_janitor_result
|
||||
|
||||
# Intentionally show the results
|
||||
- debug: var="matrix_postgres_synapse_janitor_result"
|
||||
|
||||
- name: Ensure matrix-synapse is started, if it previously was
|
||||
service:
|
||||
name: matrix-synapse
|
||||
state: started
|
||||
daemon_reload: yes
|
||||
when: "matrix_postgres_synapse_was_running|bool"
|
||||
|
||||
- name: Delete synapse-janitor tool
|
||||
file:
|
||||
path: "{{ postgres_synapse_janitor_tool_path }}"
|
||||
state: absent
|
||||
@ -0,0 +1,90 @@
|
||||
---
|
||||
|
||||
# Pre-checks
|
||||
|
||||
- name: Fail if Postgres not enabled
|
||||
fail:
|
||||
msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot run vacuum."
|
||||
when: "not matrix_postgres_enabled|bool"
|
||||
|
||||
|
||||
# Defaults
|
||||
|
||||
- name: Set postgres_start_wait_time, if not provided
|
||||
set_fact:
|
||||
postgres_start_wait_time: 15
|
||||
when: "postgres_start_wait_time|default('') == ''"
|
||||
|
||||
- name: Set postgres_vacuum_wait_time, if not provided
|
||||
set_fact:
|
||||
postgres_vacuum_wait_time: "{{ 7 * 86400 }}"
|
||||
when: "postgres_vacuum_wait_time|default('') == ''"
|
||||
|
||||
|
||||
# Actual vacuuming work
|
||||
|
||||
- name: Ensure matrix-postgres is started
|
||||
service:
|
||||
name: matrix-postgres
|
||||
state: started
|
||||
daemon_reload: yes
|
||||
|
||||
- name: Wait a bit, so that Postgres can start
|
||||
wait_for:
|
||||
timeout: "{{ postgres_start_wait_time }}"
|
||||
delegate_to: 127.0.0.1
|
||||
become: false
|
||||
|
||||
- import_tasks: tasks/util/detect_existing_postgres_version.yml
|
||||
|
||||
- name: Abort, if no existing Postgres version detected
|
||||
fail:
|
||||
msg: "Could not find existing Postgres installation"
|
||||
when: "not matrix_postgres_detected_existing|bool"
|
||||
|
||||
- name: Generate Postgres database vacuum command
|
||||
set_fact:
|
||||
matrix_postgres_vacuum_command: >-
|
||||
/usr/bin/docker run --rm --name matrix-postgres-synapse-vacuum
|
||||
--user={{ matrix_user_uid }}:{{ matrix_user_gid }}
|
||||
--cap-drop=ALL
|
||||
--network={{ matrix_docker_network }}
|
||||
--env-file={{ matrix_postgres_base_path }}/env-postgres-psql
|
||||
{{ matrix_postgres_docker_image_latest }}
|
||||
psql -v ON_ERROR_STOP=1 -h matrix-postgres {{ matrix_synapse_database_database }} -c 'VACUUM FULL VERBOSE'
|
||||
|
||||
- name: Note about Postgres vacuum alternative
|
||||
debug:
|
||||
msg: >-
|
||||
Running vacuum with the following Postgres command: `{{ matrix_postgres_vacuum_command }}`.
|
||||
If this crashes, you can stop all processes (`systemctl stop matrix-*`),
|
||||
start Postgres only (`systemctl start matrix-postgres`)
|
||||
and manually run the above command directly on the server.
|
||||
|
||||
- name: Populate service facts
|
||||
service_facts:
|
||||
|
||||
- set_fact:
|
||||
matrix_postgres_synapse_was_running: "{{ ansible_facts.services['matrix-synapse.service']|default(none) is not none and ansible_facts.services['matrix-synapse.service'].state == 'running' }}"
|
||||
|
||||
- name: Ensure matrix-synapse is stopped
|
||||
service:
|
||||
name: matrix-synapse
|
||||
state: stopped
|
||||
daemon_reload: yes
|
||||
|
||||
- name: Run Postgres vacuum command
|
||||
command: "{{ matrix_postgres_vacuum_command }}"
|
||||
async: "{{ postgres_vacuum_wait_time }}"
|
||||
poll: 10
|
||||
register: matrix_postgres_synapse_vacuum_result
|
||||
|
||||
# Intentionally show the results
|
||||
- debug: var="matrix_postgres_synapse_vacuum_result"
|
||||
|
||||
- name: Ensure matrix-synapse is started, if it previously was
|
||||
service:
|
||||
name: matrix-synapse
|
||||
state: started
|
||||
daemon_reload: yes
|
||||
when: "matrix_postgres_synapse_was_running|bool"
|
||||
Loading…
Reference in new issue