From 109e1addb882304a4fa31db7ae253a72c1c867a8 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 15 Sep 2022 10:45:03 +0300 Subject: [PATCH] Improve reliability when using more than 1 media_repository worker --- CHANGELOG.md | 8 ++++++++ roles/matrix-synapse/defaults/main.yml | 5 +++++ roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 | 4 ++++ 3 files changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06cac7a7..95356d8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ People who are interested in running a Synapse worker setup should know that **o - we've added support for [running background tasks on a worker](#background-tasks-can-run-on-a-worker) - we've restored support for [`appservice` workers](#appservice-worker-support-is-back) - we've restored support for [`user_dir` workers](#user-directory-worker-support-is-back) +- we've made it possible to [reliably use more than 1 `media_repository` worker](#using-more-than-1-media-repository-worker-is-now-more-reliable) - see the [Potential Backward Incompatibilities after these Synapse worker changes](#potential-backward-incompatibilities-after-these-synapse-worker-changes) ### Stream writers support @@ -68,6 +69,13 @@ From now on, we have support for this. With `matrix_synapse_workers_preset: one-of-each`, you'll get one `user_dir` worker automatically. You can also control the `user_dir` workers count with `matrix_synapse_workers_user_dir_workers_count`. Only `0` or `1` workers of this type are supported by Synapse. +### Using more than 1 media repository worker is now more reliable + +With `matrix_synapse_workers_preset: one-of-each`, we only launch one `media_repository` worker. + +If you've been configuring `matrix_synapse_workers_media_repository_workers_count` manually, you may have increased that to more workers. +When multiple media repository workers are in use, background tasks related to the media repository must always be configured to run on a single `media_repository` worker via `media_instance_running_background_jobs`. Until now, we weren't doing this correctly, but we now are. + ### Potential Backward Incompatibilities after these Synapse worker changes Below we'll discuss **potential backward incompatibilities**. diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 672285cc..dab098fb 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -535,6 +535,11 @@ matrix_synapse_workers_media_repository_workers_metrics_range_start: 19551 # Adjusting this value manually is generally not necessary. matrix_synapse_enable_media_repo: "{{ not matrix_synapse_workers_enabled or (matrix_synapse_workers_enabled_list | selectattr('type', 'equalto', 'media_repository') | list | length > 0) }}" +# matrix_synapse_media_instance_running_background_jobs populates the `media_instance_running_background_jobs` Synapse configuration used when Synapse workers are in use (`matrix_synapse_workers_enabled`). +# `media_instance_running_background_jobs` is meant to point to a single media-repository worker, which is dedicated to running background tasks that maintain the media repository. +# Multiple `media_repository` workers may be enabled. We always pick the first one as the background tasks worker. +matrix_synapse_media_instance_running_background_jobs: "{{ (matrix_synapse_workers_enabled_list | selectattr('type', 'equalto', 'media_repository') | list)[0].name if (matrix_synapse_workers_enabled and matrix_synapse_workers_enabled_list | selectattr('type', 'equalto', 'media_repository') | list | length > 0) else '' }}" + # matrix_synapse_workers_appservice_workers_count can only be 0 or 1. More instances are not supported. # appservice workers were deprecated since Synapse v1.59 (see: https://github.com/matrix-org/synapse/blob/v1.59.0/docs/upgrade.md#deprecation-of-the-synapseappappservice-and-synapseappuser_dir-worker-application-types). # Our implementation uses generic worker services and assigns them to perform appservice work using the `notify_appservices_from_worker` Synapse option. diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index 3cd76bf9..ae71b7ae 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -2894,6 +2894,10 @@ update_user_directory_from_worker: {{ matrix_synapse_update_user_directory_from_ run_background_tasks_on: {{ matrix_synapse_run_background_tasks_on | to_json }} {% endif %} +{% if matrix_synapse_media_instance_running_background_jobs != '' %} +media_instance_running_background_jobs: {{ matrix_synapse_media_instance_running_background_jobs | to_json }} +{% endif %} + # A shared secret used by the replication APIs to authenticate HTTP requests # from workers. #