From 353bc7c3624f6b7186d65757e7b66bc8e6e56e2c Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Fri, 10 Apr 2020 23:44:53 +0200 Subject: [PATCH 01/70] Add initial support for synapse workers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit · needs documentation; no checks yet for port clashes or typos in worker name · according to https://github.com/matrix-org/synapse/wiki/Workers-setup-with-nginx#results about 90% of requests go to the synchrotron endpoint · thus, the synchrotron worker is especially suited to be load-balanced · most of the other workers are documented to support only a single instance · https://github.com/matrix-org/synapse/blob/master/docs/workers.md --- roles/matrix-synapse/defaults/main.yml | 37 +++++++++++++++++ roles/matrix-synapse/tasks/setup_synapse.yml | 2 + roles/matrix-synapse/tasks/workers/setup.yml | 7 ++++ .../tasks/workers/setup_install.yml | 41 +++++++++++++++++++ .../tasks/workers/setup_uninstall.yml | 34 +++++++++++++++ .../templates/synapse/homeserver.yaml.j2 | 38 +++++++++++++++++ .../systemd/matrix-synapse-worker@.service.j2 | 29 +++++++++++++ .../templates/synapse/worker.yaml.j2 | 29 +++++++++++++ 8 files changed, 217 insertions(+) create mode 100644 roles/matrix-synapse/tasks/workers/setup.yml create mode 100644 roles/matrix-synapse/tasks/workers/setup_install.yml create mode 100644 roles/matrix-synapse/tasks/workers/setup_uninstall.yml create mode 100644 roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 create mode 100644 roles/matrix-synapse/templates/synapse/worker.yaml.j2 diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index f117fe93..dcd42ab8 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -258,6 +258,43 @@ matrix_synapse_metrics_port: 9100 # See https://github.com/matrix-org/synapse/blob/master/docs/manhole.md matrix_synapse_manhole_enabled: false +# Enable support for Synapse workers +matrix_synapse_workers_enabled: false + +# List of workers to spawn +matrix_synapse_workers_enabled_list: [] + +# Default list of workers to spawn +matrix_synapse_workers_enabled_list: + - { worker: synchrotron, port: 8082 } + - { worker: synchrotron, port: 8083 } + - { worker: synchrotron, port: 8084 } + - { worker: appservice, port: 8085 } + - { worker: client_reader, port: 8086 } + - { worker: event_creator, port: 8087 } + - { worker: federation_reader, port: 8088 } + - { worker: federation_sender, port: 8089 } + - { worker: frontend_proxy, port: 8090 } + - { worker: media_repository, port: 8091 } + - { worker: pusher, port: 8092 } + - { worker: user_dir, port: 8093 } + +# The list of available workers (2020-04-14) +matrix_synapse_workers_avail_list: + - appservice + - client_reader + - event_creator + - federation_reader + - federation_sender + - frontend_proxy + - media_repository + - pusher + - synchrotron + - user_dir + +# Ports used for communication between main synapse process and workers +matrix_synapse_replication_tcp_port: 9092 +matrix_synapse_replication_http_port: 9093 # Send ERROR logs to sentry.io for easier tracking # To set this up: go to sentry.io, create a python project, and set diff --git a/roles/matrix-synapse/tasks/setup_synapse.yml b/roles/matrix-synapse/tasks/setup_synapse.yml index c40ae170..ad1b24e6 100644 --- a/roles/matrix-synapse/tasks/setup_synapse.yml +++ b/roles/matrix-synapse/tasks/setup_synapse.yml @@ -19,6 +19,8 @@ - import_tasks: "{{ role_path }}/tasks/ext/setup.yml" +- import_tasks: "{{ role_path }}/tasks/workers/setup.yml" + - import_tasks: "{{ role_path }}/tasks/synapse/setup.yml" - import_tasks: "{{ role_path }}/tasks/goofys/setup.yml" diff --git a/roles/matrix-synapse/tasks/workers/setup.yml b/roles/matrix-synapse/tasks/workers/setup.yml new file mode 100644 index 00000000..faf2899e --- /dev/null +++ b/roles/matrix-synapse/tasks/workers/setup.yml @@ -0,0 +1,7 @@ +--- + +- import_tasks: "{{ role_path }}/tasks/workers/setup_install.yml" + when: "matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool" + +- import_tasks: "{{ role_path }}/tasks/workers/setup_uninstall.yml" + when: "not matrix_synapse_workers_enabled|bool" diff --git a/roles/matrix-synapse/tasks/workers/setup_install.yml b/roles/matrix-synapse/tasks/workers/setup_install.yml new file mode 100644 index 00000000..1f23d0c5 --- /dev/null +++ b/roles/matrix-synapse/tasks/workers/setup_install.yml @@ -0,0 +1,41 @@ +--- + +- name: Ensure synapse worker base service file installed + template: + src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker@.service.j2" + dest: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service" + mode: 0644 + register: matrix_synapse_worker_systemd_service_result + +- name: Ensure previous worker service symlinks are cleaned (FIXME) + file: + path: "{{ item.root + '/' + item.path }}" + state: absent + when: + - item.state == 'link' + - item.path is match('matrix-synapse-worker@*.service') + with_filetree: + - "{{ matrix_systemd_path }}/matrix-synapse.service.wants" + +- name: Ensure systemd reloaded the worker service unit + service: + daemon_reload: yes + +- name: Ensure individual worker service symlinks exist + service: + name: "matrix-synapse-worker@{{ item.worker }}:{{ item.port }}.service" + enabled: true + with_items: "{{ matrix_synapse_workers_enabled_list }}" + +- name: Ensure creation of specific worker configs + template: + src: "{{ role_path }}/templates/synapse/worker.yaml.j2" + dest: "{{ matrix_synapse_config_dir_path }}/worker.{{ item.worker }}:{{ item.port }}.yaml" + with_list: "{{ matrix_synapse_workers_enabled_list }}" + +- name: Add workers to synapse.wants list + set_fact: + matrix_synapse_systemd_wanted_services_list: > + {{ matrix_synapse_systemd_wanted_services_list + + ['matrix-synapse-worker@' + item.worker + ':' + item.port|string + '.service'] }} + with_items: "{{ matrix_synapse_workers_enabled_list }}" diff --git a/roles/matrix-synapse/tasks/workers/setup_uninstall.yml b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml new file mode 100644 index 00000000..86430879 --- /dev/null +++ b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml @@ -0,0 +1,34 @@ +--- + +- name: Ensure individual worker services are stopped + service: + name: "matrix-synapse-worker@{{ item.worker }}:{{ item.port }}.service" + state: stopped + with_items: "{{ matrix_synapse_workers_enabled_list }}" + +# As we cannot know the ports of workers removed from the enabled_list.. +# => .. just kill them all (FIXME?) +- name: Ensure previous worker service symlinks are cleaned + file: + path: "{{ item.root + '/' + item.path }}" + state: absent + when: + - item.state == 'link' + - item.path is match('matrix-synapse-worker@*.service') + with_filetree: + - "{{ matrix_systemd_path }}/matrix-synapse.service.wants" + +- name: Ensure synapse worker base service file gets removed + file: + path: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service" + state: absent + register: matrix_synapse_worker_systemd_service_result + +- name: Remove workers from synapse.wants list + set_fact: + matrix_synapse_systemd_wanted_services_list: "{{ matrix_synapse_systemd_wanted_services_list | reject('search', item) | list }}" + with_items: "{{ matrix_synapse_workers_avail_list }}" + +- name: Ensure systemd noticed removal of worker service units + service: + daemon_reload: yes diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index d41313e3..fde097f0 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -251,6 +251,44 @@ listeners: type: manhole {% endif %} +{% if matrix_synapse_workers_enabled %} + # c.f. https://github.com/matrix-org/synapse/tree/master/docs/workers.md + # TCP replication: streaming data from the master to the workers + - port: {{ matrix_synapse_replication_tcp_port }} + bind_addresses: ['0.0.0.0'] + type: replication + + # HTTP replication: for the workers to send data to the main synapse process + - port: {{ matrix_synapse_replication_http_port }} + bind_addresses: ['0.0.0.0'] + type: http + resources: + - names: [replication] + +# c.f. https://github.com/matrix-org/synapse/tree/master/contrib/systemd-with-workers/README.md +worker_app: synapse.app.homeserver + +# thx https://oznetnerd.com/2017/04/18/jinja2-selectattr-filter/ +# reduce the main worker's offerings to core homeserver business +{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'appservice')|list %} +notify_appservices: false +{% endif %} +{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_sender')|list %} +send_federation: false +{% endif %} +{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|list %} +enable_media_repo: false +{% endif %} +{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'pusher')|list %} +start_pushers: false +{% endif %} +{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|list %} +update_user_directory: false +{% endif %} + +# rather let systemd handle the forking +daemonize: false +{% endif %} ## Homeserver blocking ## diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 new file mode 100644 index 00000000..a46517c4 --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 @@ -0,0 +1,29 @@ +#jinja2: lstrip_blocks: "True" +# c.f. https://github.com/matrix-org/synapse/pull/4662 +[Unit] +Description=Synapse Matrix Worker +After=matrix-synapse.service +BindsTo=matrix-synapse.service + +[Service] +Type=simple + +# Intentional delay, so that the homeserver (we likely depend on) can manage to start. +ExecStartPre=/bin/sleep 5 + +# systemd ftw 🤦‍♂️ +# https://github.com/systemd/systemd/issues/14895#issuecomment-594123923 +ExecStart=/bin/sh -c "WORKER=%i; WORKER=$${WORKER%%:*}; \ + exec /usr/bin/docker exec \ + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + matrix-synapse \ + python -m synapse.app.$${WORKER} -c /data/homeserver.yaml -c /data/worker.%i.yaml" + +ExecStop=/usr/bin/docker exec matrix-synapse pkill -f %i +KillMode=process +Restart=always +RestartSec=10 +SyslogIdentifier=matrix-synapse-%i + +[Install] +WantedBy=matrix-synapse.service diff --git a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 new file mode 100644 index 00000000..37a5f87b --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 @@ -0,0 +1,29 @@ +#jinja2: lstrip_blocks: "True" +worker_app: synapse.app.{{ item.worker }} + +worker_replication_host: 127.0.0.1 +worker_replication_port: {{ matrix_synapse_replication_tcp_port }} +worker_replication_http_port: {{ matrix_synapse_replication_http_port }} + +{% if item.worker not in [ 'appservice', 'federation_sender', 'pusher' ] %} +worker_listeners: + - type: http + port: {{ item.port }} + resources: + - names: +{% if item.worker in [ 'synchrotron', 'client_reader', 'event_creator', 'frontend_proxy', 'user_dir' ] %} + - client +{% elif item.worker in [ 'federation_reader' ] %} + - federation +{% elif item.worker in [ 'media_repository' ] %} + - media +{% endif %} +{% endif %} + +{% if item.worker == 'frontend_proxy' %} +worker_main_http_uri: http://127.0.0.1:8008 +{% endif %} + +worker_daemonize: false +worker_pid_file: /matrix-run/{{ item.worker }}.port{{ item.port }}.pid +worker_log_config: /data/{{ matrix_server_fqn_matrix }}.log.config From a14b9c09adee37282b2deb7d0ff63594b5174107 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Sat, 11 Apr 2020 03:08:59 +0200 Subject: [PATCH 02/70] Add to synapse nginx template conditional URL rewrites for workers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit · 😅 How to keep this in sync with the matrix-synapse documentation? · regex location matching is expensive · nginx syntax limit: one location only per block / statement · thus, lots of duplicate statements in this file --- .../nginx/conf.d/matrix-synapse.conf.j2 | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 99662515..0f4982cc 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -101,6 +101,93 @@ } {% endif %} + {% if synchrotron_workers %} + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L134 #} + location /_matrix/client/r0/sync { + proxy_pass http://synchrotron$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + location /_matrix/client/r0/events { + proxy_pass http://synchrotron$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + location /_matrix/client/r0/initialSync { + proxy_pass http://synchrotron$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + location ~ ^/_matrix/client/r0/rooms/[^/]+/initialSync$ { + proxy_pass http://synchrotron$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} + + {% set client_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'client_reader')|first %} + {% if client_reader_worker %} + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L252 #} + location ^/_matrix/client/(versions$|(api/v1|r0|unstable)/(publicRooms$|rooms/.*/joined_me|rooms/.*/context/.|rooms/.*/members$|rooms/.*/messages$|rooms/.*/state$|login$|account/3pid$|keys/query$|keys/changes$|voip/turnServer$|joined_groups$|publicised_groups$|publicised_groups/|pushrules/.*$|groups/.*$|register$|auth/.*/fallback/web$)) { + proxy_pass http://127.0.0.1:{{ client_reader_worker.port }}$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} + + {% set media_repository_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|first %} + {% if media_repository_worker %} + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L219 #} + location /_matrix/media/ { + proxy_pass http://127.0.0.1:{{ media_repository_worker.port }}$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L223 #} + location ~ ^/_synapse/admin/v1/(purge_media_cache|room/.*/media.*|user/.*/media.*|media/.*|quarantine_media/.*)$ { + proxy_pass http://127.0.0.1:{{ media_repository_worker.port }}$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} + + {% set event_creator_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'event_creator')|first %} + {% if event_creator_worker %} + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L323 #} + location ~ ^/_matrix/client/(api/v1|r0|unstable)/(rooms/.*/send|rooms/.*/state/|rooms/.*/(join|invite|leave|ban|unban|kick)$|join/|profile/) { + proxy_pass http://127.0.0.1:{{ event_creator_worker.port }}$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} + + {% set frontend_proxy_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'frontend_proxy')|first %} + {% if frontend_proxy_worker %} + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L302 #} + location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload { + proxy_pass http://127.0.0.1:{{ frontend_proxy_worker.port }}$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% if not matrix_synapse_use_presence %} + location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status { + proxy_pass http://127.0.0.1:{{ frontend_proxy_worker.port }}$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} + {% endif %} + + {% set user_dir_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|first %} + {% if user_dir_worker %} + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L290 #} + location ~ ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$ { + proxy_pass http://127.0.0.1:{{ user_dir_worker.port }}$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} + {% for configuration_block in matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks %} {{- configuration_block }} {% endfor %} @@ -174,6 +261,19 @@ } {% endmacro %} +{% set synchrotron_workers = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'synchrotron')|list %} +{% if synchrotron_workers %} +upstream synchrotron { + # ensures that requests from the same client will always be passed + # to the same server (except when this server is unavailable) + ip_hash; + + {% for worker in synchrotron_workers %} + server 127.0.0.1:{{ worker.port }}; + {% endfor %} +} +{% endif %} + server { listen {{ 8080 if matrix_nginx_proxy_enabled else 80 }}; server_name {{ matrix_nginx_proxy_proxy_matrix_hostname }}; @@ -255,6 +355,16 @@ server { ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH"; {% endif %} + {% set federation_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_reader')|first %} + {% if federation_worker %} + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L160 #} + location ~ ^(/_matrix/federation/v1/event/|/_matrix/federation/v1/state/|/_matrix/federation/v1/state_ids/|/_matrix/federation/v1/backfill/|/_matrix/federation/v1/get_missing_events/|/_matrix/federation/v1/publicRooms|/_matrix/federation/v1/query/|/_matrix/federation/v1/make_join/|/_matrix/federation/v1/make_leave/|/_matrix/federation/v1/send_join/|/_matrix/federation/v2/send_join/|/_matrix/federation/v1/send_leave/|/_matrix/federation/v2/send_leave/|/_matrix/federation/v1/invite/|/_matrix/federation/v2/invite/|/_matrix/federation/v1/query_auth/|/_matrix/federation/v1/event_auth/|/_matrix/federation/v1/exchange_third_party_invite/|/_matrix/federation/v1/user/devices/|/_matrix/federation/v1/send/|/_matrix/federation/v1/get_groups_publicised$|/_matrix/key/v2/query|/_matrix/federation/v1/groups/) { + proxy_pass http://127.0.0.1:{{ federation_worker.port }}$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} + location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} From cf452fdf0a1ece84ae4c54e6a73a8618114c5132 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Sun, 19 Apr 2020 10:16:01 +0200 Subject: [PATCH 03/70] Fix corner-cases found through testing (aka ansible is nuts) --- roles/matrix-synapse/tasks/workers/setup.yml | 1 + roles/matrix-synapse/tasks/workers/setup_install.yml | 3 ++- .../matrix-synapse/tasks/workers/setup_uninstall.yml | 12 ++++++++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/roles/matrix-synapse/tasks/workers/setup.yml b/roles/matrix-synapse/tasks/workers/setup.yml index faf2899e..4951ac2d 100644 --- a/roles/matrix-synapse/tasks/workers/setup.yml +++ b/roles/matrix-synapse/tasks/workers/setup.yml @@ -1,4 +1,5 @@ --- +# a negative when condition will not actually prevent ansible from executing loops in imported tasks! - import_tasks: "{{ role_path }}/tasks/workers/setup_install.yml" when: "matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool" diff --git a/roles/matrix-synapse/tasks/workers/setup_install.yml b/roles/matrix-synapse/tasks/workers/setup_install.yml index 1f23d0c5..0031c236 100644 --- a/roles/matrix-synapse/tasks/workers/setup_install.yml +++ b/roles/matrix-synapse/tasks/workers/setup_install.yml @@ -12,8 +12,9 @@ path: "{{ item.root + '/' + item.path }}" state: absent when: + - matrix_synapse_workers_enabled|bool - item.state == 'link' - - item.path is match('matrix-synapse-worker@*.service') + - item.path is match('matrix-synapse-worker@.*\\.service') with_filetree: - "{{ matrix_systemd_path }}/matrix-synapse.service.wants" diff --git a/roles/matrix-synapse/tasks/workers/setup_uninstall.yml b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml index 86430879..d1e7e3b5 100644 --- a/roles/matrix-synapse/tasks/workers/setup_uninstall.yml +++ b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml @@ -1,10 +1,13 @@ --- -- name: Ensure individual worker services are stopped +- name: Populate service facts + service_facts: + +- name: Ensure any worker services are stopped service: - name: "matrix-synapse-worker@{{ item.worker }}:{{ item.port }}.service" + name: "{{ item.key }}" state: stopped - with_items: "{{ matrix_synapse_workers_enabled_list }}" + with_dict: "{{ ansible_facts.services|default({})|dict2items|selectattr('key', 'match', 'matrix-synapse-worker@.+\\.service')|list|items2dict }}" # As we cannot know the ports of workers removed from the enabled_list.. # => .. just kill them all (FIXME?) @@ -13,8 +16,9 @@ path: "{{ item.root + '/' + item.path }}" state: absent when: + - not matrix_synapse_workers_enabled|bool - item.state == 'link' - - item.path is match('matrix-synapse-worker@*.service') + - item.path is match('matrix-synapse-worker@.*\\.service') with_filetree: - "{{ matrix_systemd_path }}/matrix-synapse.service.wants" From 66a40735126c361e396dde47e9f96c9fe9664ee4 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Sun, 19 Apr 2020 17:42:51 +0200 Subject: [PATCH 04/70] Publish synapse worker ports, need to be accessible to nginx --- .../templates/synapse/systemd/matrix-synapse.service.j2 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 index 0bd2c25d..700e4134 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 @@ -44,6 +44,11 @@ ExecStart=/usr/bin/docker run --rm --name matrix-synapse \ {% if matrix_synapse_manhole_enabled and matrix_synapse_container_manhole_api_host_bind_port %} -p {{ matrix_synapse_container_manhole_api_host_bind_port }}:9000 \ {% endif %} + {% if matrix_synapse_workers_enabled %} + {% for worker in matrix_synapse_workers_enabled_list %} + -p {{ worker.port }}:{{ worker.port }} \ + {% endfor %} + {% endif %} -v {{ matrix_synapse_config_dir_path }}:/data:ro \ -v {{ matrix_synapse_run_path }}:/matrix-run:rw \ -v {{ matrix_synapse_storage_path }}:/matrix-media-store-parent:slave \ From e4763c21bc4a2e3a053629618d7ba7c78e120f92 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Sun, 19 Apr 2020 18:41:27 +0200 Subject: [PATCH 05/70] nginx config: route traffic to workers on matrix-synapse FIXME: horrid duplication in template file --- .../nginx/conf.d/matrix-synapse.conf.j2 | 133 +++++++++++------- 1 file changed, 80 insertions(+), 53 deletions(-) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 0f4982cc..d7a1e6b3 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -127,65 +127,86 @@ {% set client_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'client_reader')|first %} {% if client_reader_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L252 #} - location ^/_matrix/client/(versions$|(api/v1|r0|unstable)/(publicRooms$|rooms/.*/joined_me|rooms/.*/context/.|rooms/.*/members$|rooms/.*/messages$|rooms/.*/state$|login$|account/3pid$|keys/query$|keys/changes$|voip/turnServer$|joined_groups$|publicised_groups$|publicised_groups/|pushrules/.*$|groups/.*$|register$|auth/.*/fallback/web$)) { - proxy_pass http://127.0.0.1:{{ client_reader_worker.port }}$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L252 #} + location ^/_matrix/client/(versions$|(api/v1|r0|unstable)/(publicRooms$|rooms/.*/joined_me|rooms/.*/context/.|rooms/.*/members$|rooms/.*/messages$|rooms/.*/state$|login$|account/3pid$|keys/query$|keys/changes$|voip/turnServer$|joined_groups$|publicised_groups$|publicised_groups/|pushrules/.*$|groups/.*$|register$|auth/.*/fallback/web$)) { + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-synapse:{{ client_reader_worker.port }}" + proxy_pass http://$backend$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } {% endif %} {% set media_repository_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|first %} {% if media_repository_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L219 #} - location /_matrix/media/ { - proxy_pass http://127.0.0.1:{{ media_repository_worker.port }}$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L223 #} - location ~ ^/_synapse/admin/v1/(purge_media_cache|room/.*/media.*|user/.*/media.*|media/.*|quarantine_media/.*)$ { - proxy_pass http://127.0.0.1:{{ media_repository_worker.port }}$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L219 #} + location /_matrix/media/ { + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-synapse:{{ media_repository_worker.port }}" + proxy_pass http://$backend$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L223 #} + location ~ ^/_synapse/admin/v1/(purge_media_cache|room/.*/media.*|user/.*/media.*|media/.*|quarantine_media/.*)$ { + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-synapse:{{ media_repository_worker.port }}" + proxy_pass http://$backend$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } {% endif %} {% set event_creator_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'event_creator')|first %} {% if event_creator_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L323 #} - location ~ ^/_matrix/client/(api/v1|r0|unstable)/(rooms/.*/send|rooms/.*/state/|rooms/.*/(join|invite|leave|ban|unban|kick)$|join/|profile/) { - proxy_pass http://127.0.0.1:{{ event_creator_worker.port }}$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L323 #} + location ~ ^/_matrix/client/(api/v1|r0|unstable)/(rooms/.*/send|rooms/.*/state/|rooms/.*/(join|invite|leave|ban|unban|kick)$|join/|profile/) { + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-synapse:{{ event_creator_worker.port }}" + proxy_pass http://$backend$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } {% endif %} {% set frontend_proxy_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'frontend_proxy')|first %} {% if frontend_proxy_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L302 #} - location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload { - proxy_pass http://127.0.0.1:{{ frontend_proxy_worker.port }}$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {% if not matrix_synapse_use_presence %} - location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status { - proxy_pass http://127.0.0.1:{{ frontend_proxy_worker.port }}$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {% endif %} + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L302 #} + location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload { + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}" + proxy_pass http://$backend$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% if not matrix_synapse_use_presence %} + location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status { + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}" + proxy_pass http://$backend$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} {% endif %} {% set user_dir_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|first %} {% if user_dir_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L290 #} - location ~ ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$ { - proxy_pass http://127.0.0.1:{{ user_dir_worker.port }}$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L290 #} + location ~ ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$ { + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-synapse:{{ user_dir_worker.port }}" + proxy_pass http://$backend$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } {% endif %} {% for configuration_block in matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks %} @@ -268,8 +289,11 @@ upstream synchrotron { # to the same server (except when this server is unavailable) ip_hash; - {% for worker in synchrotron_workers %} - server 127.0.0.1:{{ worker.port }}; + {% for synchrotron_worker in synchrotron_workers %} + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-synapse:{{ synchrotron_worker.port }}" + server $backend:{{ synchrotron_worker.port }}; {% endfor %} } {% endif %} @@ -355,14 +379,17 @@ server { ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH"; {% endif %} - {% set federation_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_reader')|first %} - {% if federation_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L160 #} - location ~ ^(/_matrix/federation/v1/event/|/_matrix/federation/v1/state/|/_matrix/federation/v1/state_ids/|/_matrix/federation/v1/backfill/|/_matrix/federation/v1/get_missing_events/|/_matrix/federation/v1/publicRooms|/_matrix/federation/v1/query/|/_matrix/federation/v1/make_join/|/_matrix/federation/v1/make_leave/|/_matrix/federation/v1/send_join/|/_matrix/federation/v2/send_join/|/_matrix/federation/v1/send_leave/|/_matrix/federation/v2/send_leave/|/_matrix/federation/v1/invite/|/_matrix/federation/v2/invite/|/_matrix/federation/v1/query_auth/|/_matrix/federation/v1/event_auth/|/_matrix/federation/v1/exchange_third_party_invite/|/_matrix/federation/v1/user/devices/|/_matrix/federation/v1/send/|/_matrix/federation/v1/get_groups_publicised$|/_matrix/key/v2/query|/_matrix/federation/v1/groups/) { - proxy_pass http://127.0.0.1:{{ federation_worker.port }}$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } + {% set federation_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_reader')|first %} + {% if federation_reader_worker %} + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L160 #} + location ~ ^(/_matrix/federation/v1/event/|/_matrix/federation/v1/state/|/_matrix/federation/v1/state_ids/|/_matrix/federation/v1/backfill/|/_matrix/federation/v1/get_missing_events/|/_matrix/federation/v1/publicRooms|/_matrix/federation/v1/query/|/_matrix/federation/v1/make_join/|/_matrix/federation/v1/make_leave/|/_matrix/federation/v1/send_join/|/_matrix/federation/v2/send_join/|/_matrix/federation/v1/send_leave/|/_matrix/federation/v2/send_leave/|/_matrix/federation/v1/invite/|/_matrix/federation/v2/invite/|/_matrix/federation/v1/query_auth/|/_matrix/federation/v1/event_auth/|/_matrix/federation/v1/exchange_third_party_invite/|/_matrix/federation/v1/user/devices/|/_matrix/federation/v1/send/|/_matrix/federation/v1/get_groups_publicised$|/_matrix/key/v2/query|/_matrix/federation/v1/groups/) { + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-synapse:{{ federation_reader_worker.port }}" + proxy_pass http://$backend$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } {% endif %} location / { From 5f63d287b7dba70830c0218f18a873e02b9bd79c Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Sun, 19 Apr 2020 18:57:00 +0200 Subject: [PATCH 06/70] Move synapse worker ports up 10k --- roles/matrix-synapse/defaults/main.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index dcd42ab8..9e65d421 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -266,18 +266,18 @@ matrix_synapse_workers_enabled_list: [] # Default list of workers to spawn matrix_synapse_workers_enabled_list: - - { worker: synchrotron, port: 8082 } - - { worker: synchrotron, port: 8083 } - - { worker: synchrotron, port: 8084 } - - { worker: appservice, port: 8085 } - - { worker: client_reader, port: 8086 } - - { worker: event_creator, port: 8087 } - - { worker: federation_reader, port: 8088 } - - { worker: federation_sender, port: 8089 } - - { worker: frontend_proxy, port: 8090 } - - { worker: media_repository, port: 8091 } - - { worker: pusher, port: 8092 } - - { worker: user_dir, port: 8093 } + - { worker: synchrotron, port: 18082 } + - { worker: synchrotron, port: 18083 } + - { worker: synchrotron, port: 18084 } + - { worker: appservice, port: 18085 } + - { worker: client_reader, port: 18086 } + - { worker: event_creator, port: 18087 } + - { worker: federation_reader, port: 18088 } + - { worker: federation_sender, port: 18089 } + - { worker: frontend_proxy, port: 18090 } + - { worker: media_repository, port: 18091 } + - { worker: pusher, port: 18092 } + - { worker: user_dir, port: 18093 } # The list of available workers (2020-04-14) matrix_synapse_workers_avail_list: From 765c046bebcd071cde1e24d5852f8235cfe17642 Mon Sep 17 00:00:00 2001 From: Christoph Johannes Kleine Date: Sun, 19 Apr 2020 19:50:42 +0200 Subject: [PATCH 07/70] add missing ; to matrix-synapse.conf.j2 --- .../nginx/conf.d/matrix-synapse.conf.j2 | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index d7a1e6b3..71605105 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -131,7 +131,7 @@ location ^/_matrix/client/(versions$|(api/v1|r0|unstable)/(publicRooms$|rooms/.*/joined_me|rooms/.*/context/.|rooms/.*/members$|rooms/.*/messages$|rooms/.*/state$|login$|account/3pid$|keys/query$|keys/changes$|voip/turnServer$|joined_groups$|publicised_groups$|publicised_groups/|pushrules/.*$|groups/.*$|register$|auth/.*/fallback/web$)) { {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ client_reader_worker.port }}" + set $backend "matrix-synapse:{{ client_reader_worker.port }}"; proxy_pass http://$backend$request_uri; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; @@ -144,7 +144,7 @@ location /_matrix/media/ { {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ media_repository_worker.port }}" + set $backend "matrix-synapse:{{ media_repository_worker.port }}"; proxy_pass http://$backend$request_uri; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; @@ -153,7 +153,7 @@ location ~ ^/_synapse/admin/v1/(purge_media_cache|room/.*/media.*|user/.*/media.*|media/.*|quarantine_media/.*)$ { {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ media_repository_worker.port }}" + set $backend "matrix-synapse:{{ media_repository_worker.port }}"; proxy_pass http://$backend$request_uri; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; @@ -166,7 +166,7 @@ location ~ ^/_matrix/client/(api/v1|r0|unstable)/(rooms/.*/send|rooms/.*/state/|rooms/.*/(join|invite|leave|ban|unban|kick)$|join/|profile/) { {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ event_creator_worker.port }}" + set $backend "matrix-synapse:{{ event_creator_worker.port }}"; proxy_pass http://$backend$request_uri; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; @@ -179,7 +179,7 @@ location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload { {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}" + set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}"; proxy_pass http://$backend$request_uri; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; @@ -188,7 +188,7 @@ location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status { {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}" + set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}"; proxy_pass http://$backend$request_uri; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; @@ -202,7 +202,7 @@ location ~ ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$ { {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ user_dir_worker.port }}" + set $backend "matrix-synapse:{{ user_dir_worker.port }}"; proxy_pass http://$backend$request_uri; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; @@ -292,7 +292,7 @@ upstream synchrotron { {% for synchrotron_worker in synchrotron_workers %} {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ synchrotron_worker.port }}" + set $backend "matrix-synapse:{{ synchrotron_worker.port }}"; server $backend:{{ synchrotron_worker.port }}; {% endfor %} } @@ -385,7 +385,7 @@ server { location ~ ^(/_matrix/federation/v1/event/|/_matrix/federation/v1/state/|/_matrix/federation/v1/state_ids/|/_matrix/federation/v1/backfill/|/_matrix/federation/v1/get_missing_events/|/_matrix/federation/v1/publicRooms|/_matrix/federation/v1/query/|/_matrix/federation/v1/make_join/|/_matrix/federation/v1/make_leave/|/_matrix/federation/v1/send_join/|/_matrix/federation/v2/send_join/|/_matrix/federation/v1/send_leave/|/_matrix/federation/v2/send_leave/|/_matrix/federation/v1/invite/|/_matrix/federation/v2/invite/|/_matrix/federation/v1/query_auth/|/_matrix/federation/v1/event_auth/|/_matrix/federation/v1/exchange_third_party_invite/|/_matrix/federation/v1/user/devices/|/_matrix/federation/v1/send/|/_matrix/federation/v1/get_groups_publicised$|/_matrix/key/v2/query|/_matrix/federation/v1/groups/) { {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ federation_reader_worker.port }}" + set $backend "matrix-synapse:{{ federation_reader_worker.port }}"; proxy_pass http://$backend$request_uri; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; From 46984a4f99f1b3aece5fbce171feb43bf8e4a726 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Sun, 19 Apr 2020 19:55:43 +0200 Subject: [PATCH 08/70] Nginx conf: more testing less b0rk --- .../templates/nginx/conf.d/matrix-synapse.conf.j2 | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 71605105..68edab1d 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -290,10 +290,7 @@ upstream synchrotron { ip_hash; {% for synchrotron_worker in synchrotron_workers %} - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ synchrotron_worker.port }}"; - server $backend:{{ synchrotron_worker.port }}; + server "matrix-synapse:{{ synchrotron_worker.port }}"; {% endfor %} } {% endif %} From 59d1fb76b65112c5e6ac8731003fe2b92642c43b Mon Sep 17 00:00:00 2001 From: Max Klenk Date: Thu, 27 Aug 2020 15:25:32 +0200 Subject: [PATCH 09/70] only apply worker redirects if workers are enabled --- .../templates/nginx/conf.d/matrix-synapse.conf.j2 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 99ba7f3d..bb67ff56 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -101,6 +101,7 @@ } {% endif %} + {% if matrix_synapse_workers_enabled %} {% if synchrotron_workers %} {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L134 #} location /_matrix/client/r0/sync { @@ -208,6 +209,7 @@ proxy_set_header X-Forwarded-For $remote_addr; } {% endif %} + {% endif %} {% for configuration_block in matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks %} {{- configuration_block }} @@ -282,6 +284,7 @@ } {% endmacro %} +{% if matrix_synapse_workers_enabled %} {% set synchrotron_workers = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'synchrotron')|list %} {% if synchrotron_workers %} upstream synchrotron { @@ -294,6 +297,7 @@ upstream synchrotron { {% endfor %} } {% endif %} +{% endif %} server { listen {{ 8080 if matrix_nginx_proxy_enabled else 80 }}; @@ -376,6 +380,7 @@ server { ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH"; {% endif %} + {% if matrix_synapse_workers_enabled } {% set federation_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_reader')|first %} {% if federation_reader_worker %} {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L160 #} @@ -388,6 +393,7 @@ server { proxy_set_header X-Forwarded-For $remote_addr; } {% endif %} + {% endif %} location / { {% if matrix_nginx_proxy_enabled %} From 53ccc783b74510061d3c67a92e8ee903eb15b231 Mon Sep 17 00:00:00 2001 From: Max Klenk Date: Thu, 27 Aug 2020 15:26:46 +0200 Subject: [PATCH 10/70] remove duplicated key --- roles/matrix-synapse/defaults/main.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 4c403954..efddec1b 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -264,9 +264,6 @@ matrix_synapse_manhole_enabled: false matrix_synapse_workers_enabled: false # List of workers to spawn -matrix_synapse_workers_enabled_list: [] - -# Default list of workers to spawn matrix_synapse_workers_enabled_list: - { worker: synchrotron, port: 18082 } - { worker: synchrotron, port: 18083 } From 06bc430c7c59cff0235b98cce37e49ceb6eeaad7 Mon Sep 17 00:00:00 2001 From: Max Klenk Date: Fri, 28 Aug 2020 13:53:39 +0200 Subject: [PATCH 11/70] refactor to use new workers and routes they serve --- roles/matrix-nginx-proxy/defaults/main.yml | 78 ++++++ .../nginx/conf.d/matrix-synapse.conf.j2 | 234 ++++++++---------- roles/matrix-synapse/defaults/main.yml | 37 ++- .../templates/synapse/worker.yaml.j2 | 5 +- 4 files changed, 202 insertions(+), 152 deletions(-) diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 8ba0c532..d6a3d3a7 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -247,3 +247,81 @@ matrix_ssl_log_dir_path: "{{ matrix_ssl_base_path }}/log" # nginx status page configurations. matrix_nginx_proxy_proxy_matrix_nginx_status_enabled: false matrix_nginx_proxy_proxy_matrix_nginx_status_allowed_addresses: ['{{ ansible_default_ipv4.address }}'] + + +# worker +matrix_nginx_proxy_synapse_workers_enabled: "{{ matrix_synapse_workers_enabled }}" +matrix_nginx_proxy_synapse_workers_enabled_list: "{{ matrix_synapse_workers_enabled_list }}" +matrix_nginx_proxy_synapse_generic_worker_locations: [ + # Sync requests + '^/_matrix/client/(v2_alpha|r0)/sync$', + '^/_matrix/client/(api/v1|v2_alpha|r0)/events$', + '^/_matrix/client/(api/v1|r0)/initialSync$', + '^/_matrix/client/(api/v1|r0)/rooms/[^/]+/initialSync$', + + # Federation requests + '^/_matrix/federation/v1/event/', + '^/_matrix/federation/v1/state/', + '^/_matrix/federation/v1/state_ids/', + '^/_matrix/federation/v1/backfill/', + '^/_matrix/federation/v1/get_missing_events/', + '^/_matrix/federation/v1/publicRooms', + '^/_matrix/federation/v1/query/', + '^/_matrix/federation/v1/make_join/', + '^/_matrix/federation/v1/make_leave/', + '^/_matrix/federation/v1/send_join/', + '^/_matrix/federation/v2/send_join/', + '^/_matrix/federation/v1/send_leave/', + '^/_matrix/federation/v2/send_leave/', + '^/_matrix/federation/v1/invite/', + '^/_matrix/federation/v2/invite/', + '^/_matrix/federation/v1/query_auth/', + '^/_matrix/federation/v1/event_auth/', + '^/_matrix/federation/v1/exchange_third_party_invite/', + '^/_matrix/federation/v1/user/devices/', + '^/_matrix/federation/v1/get_groups_publicised$', + '^/_matrix/key/v2/query', + + # Inbound federation transaction request + '^/_matrix/federation/v1/send/', + + # Client API requests + '^/_matrix/client/(api/v1|r0|unstable)/publicRooms$', + '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/joined_members$', + '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/context/.*$', + '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/members$', + '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state$', + '^/_matrix/client/(api/v1|r0|unstable)/account/3pid$', + '^/_matrix/client/(api/v1|r0|unstable)/keys/query$', + '^/_matrix/client/(api/v1|r0|unstable)/keys/changes$', + '^/_matrix/client/versions$', + '^/_matrix/client/(api/v1|r0|unstable)/voip/turnServer$', + '^/_matrix/client/(api/v1|r0|unstable)/joined_groups$', + '^/_matrix/client/(api/v1|r0|unstable)/publicised_groups$', + '^/_matrix/client/(api/v1|r0|unstable)/publicised_groups/', + + # Registration/login requests + '^/_matrix/client/(api/v1|r0|unstable)/login$', + '^/_matrix/client/(r0|unstable)/register$', + '^/_matrix/client/(r0|unstable)/auth/.*/fallback/web$', + + # Event sending requests + '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send', + '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state/', + '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$', + '^/_matrix/client/(api/v1|r0|unstable)/join/', + '^/_matrix/client/(api/v1|r0|unstable)/profile/', +] + +matrix_nginx_proxy_synapse_media_repository_locations: [ + '^/_matrix/media/*$', + '^/_synapse/admin/v1/purge_media_cache$', + '^/_synapse/admin/v1/room/.*/media.*$', + '^/_synapse/admin/v1/user/.*/media.*$', + '^/_synapse/admin/v1/media/.*$', + '^/_synapse/admin/v1/quarantine_media/.*$', +] + +matrix_nginx_proxy_synapse_user_dir_locations: [ + 'matrix_nginx_proxy_synapse_media_workers_endpoints', +] diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index bb67ff56..a49bd8b6 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -101,115 +101,60 @@ } {% endif %} - {% if matrix_synapse_workers_enabled %} - {% if synchrotron_workers %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L134 #} - location /_matrix/client/r0/sync { - proxy_pass http://synchrotron$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - location /_matrix/client/r0/events { - proxy_pass http://synchrotron$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - location /_matrix/client/r0/initialSync { - proxy_pass http://synchrotron$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - location ~ ^/_matrix/client/r0/rooms/[^/]+/initialSync$ { - proxy_pass http://synchrotron$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {% endif %} - - {% set client_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'client_reader')|first %} - {% if client_reader_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L252 #} - location ^/_matrix/client/(versions$|(api/v1|r0|unstable)/(publicRooms$|rooms/.*/joined_me|rooms/.*/context/.|rooms/.*/members$|rooms/.*/messages$|rooms/.*/state$|login$|account/3pid$|keys/query$|keys/changes$|voip/turnServer$|joined_groups$|publicised_groups$|publicised_groups/|pushrules/.*$|groups/.*$|register$|auth/.*/fallback/web$)) { - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ client_reader_worker.port }}"; - proxy_pass http://$backend$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {% endif %} + {% if matrix_nginx_proxy_synapse_workers_enabled %} + {# Synapse Workers #} + + {% if generic_worker_workers %} + {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappgeneric_worker #} + {% for location in matrix_nginx_proxy_synapse_generic_worker_locations %} + location ~ {{ location }} { + proxy_pass http://generic_worker_upstream$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endfor %} + {# ToDo: add GET ^/_matrix/federation/v1/groups/ #} + {% endif %} - {% set media_repository_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|first %} - {% if media_repository_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L219 #} - location /_matrix/media/ { - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ media_repository_worker.port }}"; - proxy_pass http://$backend$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L223 #} - location ~ ^/_synapse/admin/v1/(purge_media_cache|room/.*/media.*|user/.*/media.*|media/.*|quarantine_media/.*)$ { - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ media_repository_worker.port }}"; - proxy_pass http://$backend$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {% endif %} + {% if media_repository_workers %} + {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappmedia_repository #} + {% for location in matrix_nginx_proxy_synapse_media_repository_locations %} + location ~ {{ location }} { + proxy_pass http://media_repository_upstream$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endfor %} + {% endif %} - {% set event_creator_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'event_creator')|first %} - {% if event_creator_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L323 #} - location ~ ^/_matrix/client/(api/v1|r0|unstable)/(rooms/.*/send|rooms/.*/state/|rooms/.*/(join|invite|leave|ban|unban|kick)$|join/|profile/) { - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ event_creator_worker.port }}"; - proxy_pass http://$backend$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {% endif %} + {% if user_dir_workers %} + {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappuser_dir #} + {% for location in matrix_nginx_proxy_synapse_user_dir_locations %} + location ~ {{ location }} { + proxy_pass http://user_dir_upstream$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endfor %} + {% endif %} - {% set frontend_proxy_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'frontend_proxy')|first %} - {% if frontend_proxy_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L302 #} - location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload { - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}"; - proxy_pass http://$backend$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {% if not matrix_synapse_use_presence %} - location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status { - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}"; - proxy_pass http://$backend$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {% endif %} + {% if frontend_proxy_workers %} + {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappfrontend_proxy #} + location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload { + proxy_pass http://frontend_proxy_upstream$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% if not matrix_synapse_use_presence %} + location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status { + proxy_pass http://frontend_proxy_upstream$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} + {% endif %} {% endif %} - {% set user_dir_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|first %} - {% if user_dir_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L290 #} - location ~ ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$ { - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ user_dir_worker.port }}"; - proxy_pass http://$backend$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {% endif %} - {% endif %} {% for configuration_block in matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks %} {{- configuration_block }} @@ -284,19 +229,51 @@ } {% endmacro %} -{% if matrix_synapse_workers_enabled %} -{% set synchrotron_workers = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'synchrotron')|list %} -{% if synchrotron_workers %} -upstream synchrotron { - # ensures that requests from the same client will always be passed - # to the same server (except when this server is unavailable) - ip_hash; +{% set generic_worker_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'generic_worker')|list %} +{% set media_repository_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|list %} +{% set user_dir_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|list %} +{% set frontend_proxy_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'frontend_proxy')|list %} +{% if matrix_nginx_proxy_synapse_workers_enabled %} + {# Setup upstream for groups of workers #} + + {% if generic_worker_workers %} + upstream generic_worker_upstream { + # ensures that requests from the same client will always be passed + # to the same server (except when this server is unavailable) + ip_hash; + + {% for worker in generic_worker_workers %} + server "matrix-synapse:{{ worker.port }}"; + {% endfor %} + } + {% endif %} - {% for synchrotron_worker in synchrotron_workers %} - server "matrix-synapse:{{ synchrotron_worker.port }}"; - {% endfor %} -} -{% endif %} + {% if frontend_proxy_workers %} + upstream frontend_proxy_upstream { + # Round Robin + {% for worker in frontend_proxy_workers %} + server "matrix-synapse:{{ worker.port }}"; + {% endfor %} + } + {% endif %} + + {% if media_repository_workers %} + upstream media_repository_upstream { + # Round Robin + {% for worker in media_repository_workers %} + server "matrix-synapse:{{ worker.port }}"; + {% endfor %} + } + {% endif %} + + {% if user_dir_workers %} + upstream user_dir_upstream { + # Round Robin + {% for worker in user_dir_workers %} + server "matrix-synapse:{{ worker.port }}"; + {% endfor %} + } + {% endif %} {% endif %} server { @@ -380,19 +357,18 @@ server { ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH"; {% endif %} - {% if matrix_synapse_workers_enabled } - {% set federation_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_reader')|first %} - {% if federation_reader_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L160 #} - location ~ ^(/_matrix/federation/v1/event/|/_matrix/federation/v1/state/|/_matrix/federation/v1/state_ids/|/_matrix/federation/v1/backfill/|/_matrix/federation/v1/get_missing_events/|/_matrix/federation/v1/publicRooms|/_matrix/federation/v1/query/|/_matrix/federation/v1/make_join/|/_matrix/federation/v1/make_leave/|/_matrix/federation/v1/send_join/|/_matrix/federation/v2/send_join/|/_matrix/federation/v1/send_leave/|/_matrix/federation/v2/send_leave/|/_matrix/federation/v1/invite/|/_matrix/federation/v2/invite/|/_matrix/federation/v1/query_auth/|/_matrix/federation/v1/event_auth/|/_matrix/federation/v1/exchange_third_party_invite/|/_matrix/federation/v1/user/devices/|/_matrix/federation/v1/send/|/_matrix/federation/v1/get_groups_publicised$|/_matrix/key/v2/query|/_matrix/federation/v1/groups/) { - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ federation_reader_worker.port }}"; - proxy_pass http://$backend$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {% endif %} + {% if matrix_nginx_proxy_synapse_workers_enabled %} + {% if generic_worker_workers %} + {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappgeneric_worker #} + {% for location in matrix_nginx_proxy_synapse_generic_worker_locations %} + location ~ {{ location }} { + proxy_pass http://generic_worker_upstream$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endfor %} + {# ToDo: add GET ^/_matrix/federation/v1/groups/ #} + {% endif %} {% endif %} location / { diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index efddec1b..aad9fad3 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -265,31 +265,28 @@ matrix_synapse_workers_enabled: false # List of workers to spawn matrix_synapse_workers_enabled_list: - - { worker: synchrotron, port: 18082 } - - { worker: synchrotron, port: 18083 } - - { worker: synchrotron, port: 18084 } - - { worker: appservice, port: 18085 } - - { worker: client_reader, port: 18086 } - - { worker: event_creator, port: 18087 } - - { worker: federation_reader, port: 18088 } - - { worker: federation_sender, port: 18089 } - - { worker: frontend_proxy, port: 18090 } - - { worker: media_repository, port: 18091 } - - { worker: pusher, port: 18092 } - - { worker: user_dir, port: 18093 } - -# The list of available workers (2020-04-14) + - { worker: generic_worker, port: 18101 } + - { worker: generic_worker, port: 18102 } + - { worker: generic_worker, port: 18103 } + - { worker: generic_worker, port: 18104 } + - { worker: generic_worker, port: 18105 } + - { worker: generic_worker, port: 18106 } + - { worker: pusher, port: 18201 } + - { worker: appservice, port: 18301 } + - { worker: federation_sender, port: 18401 } + - { worker: media_repository, port: 18501 } + - { worker: user_dir, port: 18502 } + - { worker: frontend_proxy, port: 18503 } + +# The list of available workers (2020-08-28) matrix_synapse_workers_avail_list: + - generic_worker + - pusher - appservice - - client_reader - - event_creator - - federation_reader - federation_sender - - frontend_proxy - media_repository - - pusher - - synchrotron - user_dir + - frontend_proxy # Ports used for communication between main synapse process and workers matrix_synapse_replication_tcp_port: 9092 diff --git a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 index 37a5f87b..d5f78fdb 100644 --- a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 @@ -11,9 +11,8 @@ worker_listeners: port: {{ item.port }} resources: - names: -{% if item.worker in [ 'synchrotron', 'client_reader', 'event_creator', 'frontend_proxy', 'user_dir' ] %} +{% if item.worker in [ 'generic_worker', 'frontend_proxy', 'user_dir' ] %} - client -{% elif item.worker in [ 'federation_reader' ] %} - federation {% elif item.worker in [ 'media_repository' ] %} - media @@ -23,7 +22,7 @@ worker_listeners: {% if item.worker == 'frontend_proxy' %} worker_main_http_uri: http://127.0.0.1:8008 {% endif %} - + worker_daemonize: false worker_pid_file: /matrix-run/{{ item.worker }}.port{{ item.port }}.pid worker_log_config: /data/{{ matrix_server_fqn_matrix }}.log.config From a25a429a52baadedaaa77309ccf284618d781a92 Mon Sep 17 00:00:00 2001 From: Max Klenk Date: Thu, 10 Sep 2020 13:39:00 +0200 Subject: [PATCH 12/70] add redis support --- group_vars/matrix_servers | 21 ++++ roles/matrix-redis/defaults/main.yml | 22 +++++ roles/matrix-redis/tasks/init.yml | 3 + roles/matrix-redis/tasks/main.yml | 9 ++ roles/matrix-redis/tasks/setup_redis.yml | 99 +++++++++++++++++++ roles/matrix-redis/templates/redis.conf.j2 | 2 + .../templates/systemd/matrix-redis.service.j2 | 36 +++++++ roles/matrix-synapse/defaults/main.yml | 13 ++- .../templates/synapse/homeserver.yaml.j2 | 13 +-- .../templates/synapse/worker.yaml.j2 | 1 - setup.yml | 1 + 11 files changed, 206 insertions(+), 14 deletions(-) create mode 100644 roles/matrix-redis/defaults/main.yml create mode 100644 roles/matrix-redis/tasks/init.yml create mode 100644 roles/matrix-redis/tasks/main.yml create mode 100644 roles/matrix-redis/tasks/setup_redis.yml create mode 100644 roles/matrix-redis/templates/redis.conf.j2 create mode 100644 roles/matrix-redis/templates/systemd/matrix-redis.service.j2 diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 4a989f03..d80072ef 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -864,6 +864,22 @@ matrix_postgres_db_name: "homeserver" +###################################################################### +# +# matrix-redis +# +###################################################################### + +matrix_redis_enabled: "{{ matrix_synapse_workers_enabled }}" + +###################################################################### +# +# /matrix-redis +# +###################################################################### + + + ###################################################################### # # matrix-client-element @@ -1003,6 +1019,11 @@ matrix_synapse_systemd_wanted_services_list: | (['matrix-mailer.service'] if matrix_mailer_enabled else []) }} +# Worker support with redis +matrix_synapse_redis_enabled: "{{ matrix_redis_enabled }}" +matrix_synapse_redis_host: "{{ 'matrix-redis' if matrix_redis_enabled else '' }}" +matrix_synapse_redis_password: "{{ matrix_redis_connection_password if matrix_redis_enabled else '' }}" + ###################################################################### # # /matrix-synapse diff --git a/roles/matrix-redis/defaults/main.yml b/roles/matrix-redis/defaults/main.yml new file mode 100644 index 00000000..f48ea542 --- /dev/null +++ b/roles/matrix-redis/defaults/main.yml @@ -0,0 +1,22 @@ +matrix_redis_enabled: true + +matrix_redis_connection_password: "" + +matrix_redis_base_path: "{{ matrix_base_data_path }}/redis" +matrix_redis_data_path: "{{ matrix_redis_base_path }}/data" + +matrix_redis_docker_image_v5: "redis:5.0-alpine" +matrix_redis_docker_image_v6: "redis:6.0-alpine" +matrix_redis_docker_image_latest: "{{ matrix_redis_docker_image_v6 }}" +matrix_redis_docker_image_to_use: '{{ matrix_redis_docker_image_latest }}' + +matrix_redis_docker_image_force_pull: "{{ matrix_redis_docker_image_to_use.endswith(':latest') }}" + +# A list of extra arguments to pass to the container +matrix_redis_container_extra_arguments: [] + +# Controls whether the matrix-redis container exposes a port (tcp/6379 in the container) +# that can be used to access redis from outside the container +# +# Takes an ":" or "" value (e.g. "127.0.0.1:6379"), or empty string to not expose. +matrix_redis_container_redis_bind_port: "" diff --git a/roles/matrix-redis/tasks/init.yml b/roles/matrix-redis/tasks/init.yml new file mode 100644 index 00000000..49068851 --- /dev/null +++ b/roles/matrix-redis/tasks/init.yml @@ -0,0 +1,3 @@ +- set_fact: + matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-redis'] }}" + when: matrix_redis_enabled|bool diff --git a/roles/matrix-redis/tasks/main.yml b/roles/matrix-redis/tasks/main.yml new file mode 100644 index 00000000..595b09f5 --- /dev/null +++ b/roles/matrix-redis/tasks/main.yml @@ -0,0 +1,9 @@ +- import_tasks: "{{ role_path }}/tasks/init.yml" + tags: + - always + +- import_tasks: "{{ role_path }}/tasks/setup_redis.yml" + when: run_setup|bool + tags: + - setup-all + - setup-redis diff --git a/roles/matrix-redis/tasks/setup_redis.yml b/roles/matrix-redis/tasks/setup_redis.yml new file mode 100644 index 00000000..6f00282b --- /dev/null +++ b/roles/matrix-redis/tasks/setup_redis.yml @@ -0,0 +1,99 @@ +--- + +# +# Tasks related to setting up an internal redis server +# + +- name: Ensure redis Docker image is pulled + docker_image: + name: "{{ matrix_redis_docker_image_to_use }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_redis_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_redis_docker_image_force_pull }}" + when: matrix_redis_enabled|bool + +- name: Ensure redis paths exist + file: + path: "{{ item }}" + state: directory + mode: 0700 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_username }}" + with_items: + - "{{ matrix_redis_base_path }}" + - "{{ matrix_redis_data_path }}" + when: matrix_redis_enabled|bool + +# We do this as a separate task, because: +# - we'd like to do it for the data path only, not for the base path (which contains root-owned environment variable files we'd like to leave as-is) +# - we need to do it without `mode`, or we risk making certain `.conf` and other files's executable bit to flip to true +- name: Ensure redis data path ownership is correct + file: + path: "{{ matrix_redis_data_path }}" + state: directory + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_username }}" + recurse: yes + when: matrix_redis_enabled|bool + +- name: Ensure redis environment variables file created + template: + src: "{{ role_path }}/templates/{{ item }}.j2" + dest: "{{ matrix_redis_base_path }}/{{ item }}" + mode: 0644 + with_items: + - "redis.conf" + when: matrix_redis_enabled|bool + +- name: Ensure matrix-redis.service installed + template: + src: "{{ role_path }}/templates/systemd/matrix-redis.service.j2" + dest: "{{ matrix_systemd_path }}/matrix-redis.service" + mode: 0644 + register: matrix_redis_systemd_service_result + when: matrix_redis_enabled|bool + +- name: Ensure systemd reloaded after matrix-redis.service installation + service: + daemon_reload: yes + when: "matrix_redis_enabled|bool and matrix_redis_systemd_service_result.changed" + +# +# Tasks related to getting rid of the internal redis server (if it was previously enabled) +# + +- name: Check existence of matrix-redis service + stat: + path: "{{ matrix_systemd_path }}/matrix-redis.service" + register: matrix_redis_service_stat + when: "not matrix_redis_enabled|bool" + +- name: Ensure matrix-redis is stopped + service: + name: matrix-redis + state: stopped + daemon_reload: yes + when: "not matrix_redis_enabled|bool and matrix_redis_service_stat.stat.exists" + +- name: Ensure matrix-redis.service doesn't exist + file: + path: "{{ matrix_systemd_path }}/matrix-redis.service" + state: absent + when: "not matrix_redis_enabled|bool and matrix_redis_service_stat.stat.exists" + +- name: Ensure systemd reloaded after matrix-redis.service removal + service: + daemon_reload: yes + when: "not matrix_redis_enabled|bool and matrix_redis_service_stat.stat.exists" + +- name: Check existence of matrix-redis local data path + stat: + path: "{{ matrix_redis_data_path }}" + register: matrix_redis_data_path_stat + when: "not matrix_redis_enabled|bool" + +# We just want to notify the user. Deleting data is too destructive. +- name: Notify if matrix-redis local data remains + debug: + msg: "Note: You are not using a local redis instance, but some old data remains from before in `{{ matrix_redis_data_path }}`. Feel free to delete it." + when: "not matrix_redis_enabled|bool and matrix_redis_data_path_stat.stat.exists" diff --git a/roles/matrix-redis/templates/redis.conf.j2 b/roles/matrix-redis/templates/redis.conf.j2 new file mode 100644 index 00000000..23b07929 --- /dev/null +++ b/roles/matrix-redis/templates/redis.conf.j2 @@ -0,0 +1,2 @@ +#jinja2: lstrip_blocks: "True" +requirepass {{ matrix_redis_connection_password }} diff --git a/roles/matrix-redis/templates/systemd/matrix-redis.service.j2 b/roles/matrix-redis/templates/systemd/matrix-redis.service.j2 new file mode 100644 index 00000000..0752d23b --- /dev/null +++ b/roles/matrix-redis/templates/systemd/matrix-redis.service.j2 @@ -0,0 +1,36 @@ +#jinja2: lstrip_blocks: "True" +[Unit] +Description=Matrix Redis server +After=docker.service +Requires=docker.service + +[Service] +Type=simple +ExecStartPre=-/usr/bin/docker stop matrix-redis +ExecStartPre=-/usr/bin/docker rm matrix-redis + +ExecStart=/usr/bin/docker run --rm --name matrix-redis \ + --log-driver=none \ + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + --cap-drop=ALL \ + --read-only \ + --tmpfs=/tmp:rw,noexec,nosuid,size=100m \ + --network={{ matrix_docker_network }} \ + {% if matrix_redis_container_redis_bind_port %} + -p {{ matrix_redis_container_redis_bind_port }}:6379 \ + {% endif %} + -v {{ matrix_redis_base_path }}/redis.conf:/usr/local/etc/redis/redis.conf \ + {% for arg in matrix_redis_container_extra_arguments %} + {{ arg }} \ + {% endfor %} + {{ matrix_redis_docker_image_to_use }} \ + redis-server /usr/local/etc/redis/redis.conf + +ExecStop=-/usr/bin/docker stop matrix-redis +ExecStop=-/usr/bin/docker rm matrix-redis +Restart=always +RestartSec=30 +SyslogIdentifier=matrix-redis + +[Install] +WantedBy=multi-user.target diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index aad9fad3..c390421d 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -275,8 +275,8 @@ matrix_synapse_workers_enabled_list: - { worker: appservice, port: 18301 } - { worker: federation_sender, port: 18401 } - { worker: media_repository, port: 18501 } - - { worker: user_dir, port: 18502 } - - { worker: frontend_proxy, port: 18503 } + - { worker: user_dir, port: 18601 } + - { worker: frontend_proxy, port: 18701 } # The list of available workers (2020-08-28) matrix_synapse_workers_avail_list: @@ -288,8 +288,13 @@ matrix_synapse_workers_avail_list: - user_dir - frontend_proxy -# Ports used for communication between main synapse process and workers -matrix_synapse_replication_tcp_port: 9092 +# Redis information +matrix_synapse_redis_enabled: false +matrix_synapse_redis_host: "" +matrix_synapse_redis_port: 6379 +matrix_synapse_redis_password: "" + +# Port used for communication between main synapse process and workers matrix_synapse_replication_http_port: 9093 # Send ERROR logs to sentry.io for easier tracking diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index 9b92e05e..9bdc97d9 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -225,11 +225,6 @@ listeners: {% if matrix_synapse_workers_enabled %} # c.f. https://github.com/matrix-org/synapse/tree/master/docs/workers.md - # TCP replication: streaming data from the master to the workers - - port: {{ matrix_synapse_replication_tcp_port }} - bind_addresses: ['0.0.0.0'] - type: replication - # HTTP replication: for the workers to send data to the main synapse process - port: {{ matrix_synapse_replication_http_port }} bind_addresses: ['0.0.0.0'] @@ -2464,16 +2459,16 @@ opentracing: redis: # Uncomment the below to enable Redis support. # - #enabled: true + enabled: {{ matrix_synapse_redis_enabled }} # Optional host and port to use to connect to redis. Defaults to # localhost and 6379 # - #host: localhost - #port: 6379 + host: {{ matrix_synapse_redis_host }} + port: {{ matrix_synapse_redis_port }} # Optional password if configured on the Redis instance # - #password: + password: {{ matrix_synapse_redis_password }} # vim:ft=yaml diff --git a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 index d5f78fdb..c99e97cd 100644 --- a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 @@ -2,7 +2,6 @@ worker_app: synapse.app.{{ item.worker }} worker_replication_host: 127.0.0.1 -worker_replication_port: {{ matrix_synapse_replication_tcp_port }} worker_replication_http_port: {{ matrix_synapse_replication_http_port }} {% if item.worker not in [ 'appservice', 'federation_sender', 'pusher' ] %} diff --git a/setup.yml b/setup.yml index 1c19d442..db9fbf88 100755 --- a/setup.yml +++ b/setup.yml @@ -7,6 +7,7 @@ - matrix-base - matrix-mailer - matrix-postgres + - matrix-redis - matrix-corporal - matrix-bridge-appservice-discord - matrix-bridge-appservice-slack From 4fdfc0a34f71e7c88e3402920bb05a658d4c752c Mon Sep 17 00:00:00 2001 From: Max Klenk Date: Fri, 11 Sep 2020 09:46:20 +0200 Subject: [PATCH 13/70] add missing ratelimiting options required for load testing --- roles/matrix-synapse/defaults/main.yml | 12 ++++++++++++ .../templates/synapse/homeserver.yaml.j2 | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index c390421d..c6f24fb1 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -111,6 +111,18 @@ matrix_synapse_rc_login: per_second: 0.17 burst_count: 3 +matrix_synapse_rc_admin_redaction: + per_second: 1 + burst_count: 50 + +matrix_synapse_rc_joins: + local: + per_second: 0.1 + burst_count: 3 + remote: + per_second: 0.01 + burst_count: 3 + matrix_synapse_rc_federation: window_size: 1000 sleep_limit: 10 diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index 9bdc97d9..508da379 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -772,6 +772,7 @@ rc_login: {{ matrix_synapse_rc_login|to_json }} #rc_admin_redaction: # per_second: 1 # burst_count: 50 +rc_admin_redaction: {{ matrix_synapse_rc_admin_redaction|to_json }} # #rc_joins: # local: @@ -780,7 +781,7 @@ rc_login: {{ matrix_synapse_rc_login|to_json }} # remote: # per_second: 0.01 # burst_count: 3 - +rc_joins: {{ matrix_synapse_rc_joins|to_json }} # Ratelimiting settings for incoming federation # From 880025324a71c199d2471f19fca23876cac64f92 Mon Sep 17 00:00:00 2001 From: Max Klenk Date: Fri, 11 Sep 2020 10:35:50 +0200 Subject: [PATCH 14/70] fix redis config if no password is set --- roles/matrix-redis/templates/redis.conf.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/matrix-redis/templates/redis.conf.j2 b/roles/matrix-redis/templates/redis.conf.j2 index 23b07929..34371356 100644 --- a/roles/matrix-redis/templates/redis.conf.j2 +++ b/roles/matrix-redis/templates/redis.conf.j2 @@ -1,2 +1,4 @@ #jinja2: lstrip_blocks: "True" +{% if matrix_redis_connection_password %} requirepass {{ matrix_redis_connection_password }} +{% endif %} From 1e68d8b2e598bb2c52ac68280fa15e74d8e39f6b Mon Sep 17 00:00:00 2001 From: Max Klenk Date: Fri, 11 Sep 2020 14:29:10 +0200 Subject: [PATCH 15/70] allow to pass arguments to the postgres process --- roles/matrix-postgres/defaults/main.yml | 4 ++++ .../templates/systemd/matrix-postgres.service.j2 | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index b5c6f6a8..6b824a23 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -22,6 +22,10 @@ matrix_postgres_docker_image_force_pull: "{{ matrix_postgres_docker_image_to_use # A list of extra arguments to pass to the container matrix_postgres_container_extra_arguments: [] +# A list of extra arguments to pass to the postgres process +# e.g. "-c 'max_connections=200'" +matrix_postgres_process_extra_arguments: [] + # Controls whether the matrix-postgres container exposes a port (tcp/5432 in the # container) that can be used to access the database from outside the container (e.g. with psql) # diff --git a/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2 b/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2 index 8e6392c1..f1c751a6 100644 --- a/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2 +++ b/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2 @@ -26,7 +26,11 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-postgres \ {% for arg in matrix_postgres_container_extra_arguments %} {{ arg }} \ {% endfor %} - {{ matrix_postgres_docker_image_to_use }} + {{ matrix_postgres_docker_image_to_use }} \ + postgres \ + {% for arg in matrix_postgres_process_extra_arguments %} + {{ arg }} \ + {% endfor %} ExecStop=-{{ matrix_host_command_docker }} stop matrix-postgres ExecStop=-{{ matrix_host_command_docker }} rm matrix-postgres From 132daba1af645aa007f6abdd1c5fa981545d4348 Mon Sep 17 00:00:00 2001 From: Max Klenk Date: Fri, 18 Sep 2020 10:17:43 +0200 Subject: [PATCH 16/70] fix worker routes --- roles/matrix-nginx-proxy/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index d6a3d3a7..0df7ce68 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -314,7 +314,7 @@ matrix_nginx_proxy_synapse_generic_worker_locations: [ ] matrix_nginx_proxy_synapse_media_repository_locations: [ - '^/_matrix/media/*$', + '^/_matrix/media/$', '^/_synapse/admin/v1/purge_media_cache$', '^/_synapse/admin/v1/room/.*/media.*$', '^/_synapse/admin/v1/user/.*/media.*$', @@ -323,5 +323,5 @@ matrix_nginx_proxy_synapse_media_repository_locations: [ ] matrix_nginx_proxy_synapse_user_dir_locations: [ - 'matrix_nginx_proxy_synapse_media_workers_endpoints', + '^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$', ] From fc2edcbecf0788191c90fdff8193998c8f6adbf9 Mon Sep 17 00:00:00 2001 From: Max Klenk Date: Fri, 18 Sep 2020 10:45:01 +0200 Subject: [PATCH 17/70] fix media routing --- roles/matrix-nginx-proxy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 0df7ce68..4ae22f23 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -314,7 +314,7 @@ matrix_nginx_proxy_synapse_generic_worker_locations: [ ] matrix_nginx_proxy_synapse_media_repository_locations: [ - '^/_matrix/media/$', + '^/_matrix/media/', '^/_synapse/admin/v1/purge_media_cache$', '^/_synapse/admin/v1/room/.*/media.*$', '^/_synapse/admin/v1/user/.*/media.*$', From 40024e9b81539e942bc544c798304883085d75c7 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Tue, 29 Sep 2020 11:14:39 +0200 Subject: [PATCH 18/70] Prevent workers failing if their config doesn't exist - cherry-pick "Ensure worker config exists in systemd service (#7528)" from synapse d74cdc1a42e8b487d74c214b1d0ca575429d546a: "check that the worker config file exists instead of silently failing." --- .../templates/synapse/systemd/matrix-synapse-worker@.service.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 index a46517c4..d82564c4 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 @@ -2,6 +2,7 @@ # c.f. https://github.com/matrix-org/synapse/pull/4662 [Unit] Description=Synapse Matrix Worker +AssertPathExists={{matrix_synapse_config_dir_path }}/worker.%i.yaml After=matrix-synapse.service BindsTo=matrix-synapse.service From e9241f5fb9ee5603bbabea76ca5f8f5b3f92e470 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Sun, 11 Oct 2020 16:42:45 +0200 Subject: [PATCH 19/70] Improve synapse-workers systemd service template Is the PID magic gonna work? or will it need an ExecStartPost hack.. --- .../synapse/systemd/matrix-synapse-worker@.service.j2 | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 index d82564c4..d14b2557 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 @@ -1,4 +1,6 @@ #jinja2: lstrip_blocks: "True" +# Instantiable worker service, running inside the synapse container +# alongside the homeserver main process. # c.f. https://github.com/matrix-org/synapse/pull/4662 [Unit] Description=Synapse Matrix Worker @@ -12,15 +14,18 @@ Type=simple # Intentional delay, so that the homeserver (we likely depend on) can manage to start. ExecStartPre=/bin/sleep 5 -# systemd ftw 🤦‍♂️ -# https://github.com/systemd/systemd/issues/14895#issuecomment-594123923 +# no sane way of instancing more than one variable (systemd "cant-fix" 🤦) +# c.f. https://github.com/systemd/systemd/issues/14895#issuecomment-594123923 +# So use good ol' shell parameter expansion to get the worker type.. ExecStart=/bin/sh -c "WORKER=%i; WORKER=$${WORKER%%:*}; \ exec /usr/bin/docker exec \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ matrix-synapse \ python -m synapse.app.$${WORKER} -c /data/homeserver.yaml -c /data/worker.%i.yaml" +ExecReload=/bin/kill -HUP $MAINPID ExecStop=/usr/bin/docker exec matrix-synapse pkill -f %i +PIDFile=/matrix-run/{{ item.worker }}.port{{ item.port }}.pid KillMode=process Restart=always RestartSec=10 From 36e9be60929a35957a4c38969d3337d61afbc791 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Sun, 11 Oct 2020 21:31:18 +0200 Subject: [PATCH 20/70] matrix_synapse_workers_{avail,enabled}_list: sort non-generic workers .. alphabetically and put those not documented as multi-instance capable on ports ending on zero. --- roles/matrix-synapse/defaults/main.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 8ea12850..4fb843cb 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -275,7 +275,8 @@ matrix_synapse_manhole_enabled: false # Enable support for Synapse workers matrix_synapse_workers_enabled: false -# List of workers to spawn +# Default list of workers to spawn +# (worker with ports ending on 00 are single-instance) matrix_synapse_workers_enabled_list: - { worker: generic_worker, port: 18101 } - { worker: generic_worker, port: 18102 } @@ -283,22 +284,22 @@ matrix_synapse_workers_enabled_list: - { worker: generic_worker, port: 18104 } - { worker: generic_worker, port: 18105 } - { worker: generic_worker, port: 18106 } - - { worker: pusher, port: 18201 } - - { worker: appservice, port: 18301 } - - { worker: federation_sender, port: 18401 } + - { worker: appservice, port: 18200 } + - { worker: federation_sender, port: 18301 } + - { worker: frontend_proxy, port: 18400 } - { worker: media_repository, port: 18501 } - - { worker: user_dir, port: 18601 } - - { worker: frontend_proxy, port: 18701 } + - { worker: pusher, port: 18600 } + - { worker: user_dir, port: 18700 } # The list of available workers (2020-08-28) matrix_synapse_workers_avail_list: - generic_worker - - pusher - appservice - federation_sender + - frontend_proxy - media_repository + - pusher - user_dir - - frontend_proxy # Redis information matrix_synapse_redis_enabled: false From d2e61af2243e2fa3ca95f15210ffa41d66fe2463 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Wed, 9 Sep 2020 19:57:49 +0200 Subject: [PATCH 21/70] Add worker_name to synapse worker config template & restrict federation listener; frontend_proxy / user_dir don't need it --- roles/matrix-synapse/templates/synapse/worker.yaml.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 index c99e97cd..319f5708 100644 --- a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 @@ -1,5 +1,6 @@ #jinja2: lstrip_blocks: "True" worker_app: synapse.app.{{ item.worker }} +worker_name: {{ item.worker ~ '_' ~ item.port }} worker_replication_host: 127.0.0.1 worker_replication_http_port: {{ matrix_synapse_replication_http_port }} @@ -12,6 +13,8 @@ worker_listeners: - names: {% if item.worker in [ 'generic_worker', 'frontend_proxy', 'user_dir' ] %} - client +{% endif %} +{% if item.worker in [ 'generic_worker' ] %} - federation {% elif item.worker in [ 'media_repository' ] %} - media From 501efee07e621eb626f5ed032210d49f30c04408 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Thu, 22 Oct 2020 20:53:41 +0200 Subject: [PATCH 22/70] synapse workers: supply systemd with actual worker PIDs (requires jq) also, worker.yaml.j2: - hone worker_name - remove worker_pid_file entry (would only be used if worker_daemonize set to true; also, synapse only knows about the container namespace and thus can not provide the required host-view PID) --- .../matrix-synapse-worker-write-pid | 30 +++++++++++++++++++ .../tasks/workers/setup_install.yml | 6 ++++ .../tasks/workers/setup_uninstall.yml | 5 ++++ .../systemd/matrix-synapse-worker@.service.j2 | 12 +++++--- .../templates/synapse/worker.yaml.j2 | 3 +- 5 files changed, 50 insertions(+), 6 deletions(-) create mode 100644 roles/matrix-synapse/files/usr-local-bin/matrix-synapse-worker-write-pid diff --git a/roles/matrix-synapse/files/usr-local-bin/matrix-synapse-worker-write-pid b/roles/matrix-synapse/files/usr-local-bin/matrix-synapse-worker-write-pid new file mode 100644 index 00000000..02c5ba09 --- /dev/null +++ b/roles/matrix-synapse/files/usr-local-bin/matrix-synapse-worker-write-pid @@ -0,0 +1,30 @@ +#!/bin/bash +# Find a synapse worker's PID and write it to a file so systemd can manage it as a service + +# example invocation: +# matrix-synapse-worker-write-pid user_dir:18700 /run/matrix-synapse-worker.user_dir:18700.pid + +docker_api_call() { curl --silent --unix-socket /var/run/docker.sock ${@}; } + +TARGETCONTAINER=matrix-synapse +TARGETWORKER=${1} +PIDFILE=${2} + +# get ID list of subprocesses executed in $TARGETCONTAINER, and for each.. +for EXECID in $(docker_api_call http://localhost/containers/${TARGETCONTAINER}/json | jq --raw-output '.ExecIDs[]') +do + # fetch detailed process info + EXECINFO=$(docker_api_call http://localhost/exec/${EXECID}/json) + + # extract config file path from last command argument + WORKERCONFIGFILE=$(echo ${EXECINFO} | jq --raw-output .ProcessConfig.arguments[-1]) + + # reconstruct worker name + WORKERNAME=${WORKERCONFIGFILE#*/worker.} + WORKERNAME=${WORKERNAME%.yaml} + + # if name matches the target worker: write out most recent PID & quit + [ "${WORKERNAME}" = "${TARGETWORKER}" ] \ + && echo ${EXECINFO} | jq --raw-output .Pid > ${PIDFILE} \ + && exit 0 +done diff --git a/roles/matrix-synapse/tasks/workers/setup_install.yml b/roles/matrix-synapse/tasks/workers/setup_install.yml index 0031c236..44d59495 100644 --- a/roles/matrix-synapse/tasks/workers/setup_install.yml +++ b/roles/matrix-synapse/tasks/workers/setup_install.yml @@ -40,3 +40,9 @@ {{ matrix_synapse_systemd_wanted_services_list + ['matrix-synapse-worker@' + item.worker + ':' + item.port|string + '.service'] }} with_items: "{{ matrix_synapse_workers_enabled_list }}" + +- name: Ensure matrix-synapse-worker-write-pid script is created + copy: + src: "{{ role_path }}/files/usr-local-bin/matrix-synapse-worker-write-pid" + dest: "{{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid" + mode: 0750 diff --git a/roles/matrix-synapse/tasks/workers/setup_uninstall.yml b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml index d1e7e3b5..0571114c 100644 --- a/roles/matrix-synapse/tasks/workers/setup_uninstall.yml +++ b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml @@ -36,3 +36,8 @@ - name: Ensure systemd noticed removal of worker service units service: daemon_reload: yes + +- name: Ensure matrix-synapse-worker-write-pid script is removed + file: + path: "{{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid" + state: absent diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 index d14b2557..2c82873d 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 @@ -3,8 +3,8 @@ # alongside the homeserver main process. # c.f. https://github.com/matrix-org/synapse/pull/4662 [Unit] -Description=Synapse Matrix Worker -AssertPathExists={{matrix_synapse_config_dir_path }}/worker.%i.yaml +Description=Matrix worker synapse.app.%i +AssertPathExists={{ matrix_synapse_config_dir_path }}/worker.%i.yaml After=matrix-synapse.service BindsTo=matrix-synapse.service @@ -23,9 +23,13 @@ ExecStart=/bin/sh -c "WORKER=%i; WORKER=$${WORKER%%:*}; \ matrix-synapse \ python -m synapse.app.$${WORKER} -c /data/homeserver.yaml -c /data/worker.%i.yaml" +# wait for worker startup & write out PID of actual worker process so systemd can handle it +ExecStartPost=/bin/sleep 5 +ExecStartPost=/usr/local/bin/matrix-synapse-worker-write-pid %i /run/matrix-synapse-worker.%i.pid + ExecReload=/bin/kill -HUP $MAINPID -ExecStop=/usr/bin/docker exec matrix-synapse pkill -f %i -PIDFile=/matrix-run/{{ item.worker }}.port{{ item.port }}.pid +ExecStop=/bin/kill $MAINPID +PIDFile=/run/matrix-synapse-worker.%i.pid KillMode=process Restart=always RestartSec=10 diff --git a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 index 319f5708..0a282ba7 100644 --- a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 @@ -1,6 +1,6 @@ #jinja2: lstrip_blocks: "True" worker_app: synapse.app.{{ item.worker }} -worker_name: {{ item.worker ~ '_' ~ item.port }} +worker_name: {{ item.worker ~ ':' ~ item.port }} worker_replication_host: 127.0.0.1 worker_replication_http_port: {{ matrix_synapse_replication_http_port }} @@ -26,5 +26,4 @@ worker_main_http_uri: http://127.0.0.1:8008 {% endif %} worker_daemonize: false -worker_pid_file: /matrix-run/{{ item.worker }}.port{{ item.port }}.pid worker_log_config: /data/{{ matrix_server_fqn_matrix }}.log.config From a4125d544623312397c0760f43ac91fd261fa96b Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Fri, 23 Oct 2020 20:49:53 +0200 Subject: [PATCH 23/70] synapse workers: polishing, cleansing and installation of jq dependency --- roles/matrix-synapse/tasks/workers/setup.yml | 5 ++- .../tasks/workers/setup_install.yml | 31 +++++++++++++++++-- .../tasks/workers/setup_uninstall.yml | 12 ++++--- .../systemd/matrix-synapse-worker@.service.j2 | 10 +++--- 4 files changed, 44 insertions(+), 14 deletions(-) diff --git a/roles/matrix-synapse/tasks/workers/setup.yml b/roles/matrix-synapse/tasks/workers/setup.yml index 4951ac2d..083da807 100644 --- a/roles/matrix-synapse/tasks/workers/setup.yml +++ b/roles/matrix-synapse/tasks/workers/setup.yml @@ -1,8 +1,7 @@ --- -# a negative when condition will not actually prevent ansible from executing loops in imported tasks! -- import_tasks: "{{ role_path }}/tasks/workers/setup_install.yml" +- include_tasks: "{{ role_path }}/tasks/workers/setup_install.yml" when: "matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool" -- import_tasks: "{{ role_path }}/tasks/workers/setup_uninstall.yml" +- include_tasks: "{{ role_path }}/tasks/workers/setup_uninstall.yml" when: "not matrix_synapse_workers_enabled|bool" diff --git a/roles/matrix-synapse/tasks/workers/setup_install.yml b/roles/matrix-synapse/tasks/workers/setup_install.yml index 44d59495..3f1f8ac0 100644 --- a/roles/matrix-synapse/tasks/workers/setup_install.yml +++ b/roles/matrix-synapse/tasks/workers/setup_install.yml @@ -7,12 +7,11 @@ mode: 0644 register: matrix_synapse_worker_systemd_service_result -- name: Ensure previous worker service symlinks are cleaned (FIXME) +- name: Ensure previous worker service symlinks are cleaned file: path: "{{ item.root + '/' + item.path }}" state: absent when: - - matrix_synapse_workers_enabled|bool - item.state == 'link' - item.path is match('matrix-synapse-worker@.*\\.service') with_filetree: @@ -28,6 +27,13 @@ enabled: true with_items: "{{ matrix_synapse_workers_enabled_list }}" +- name: Ensure previous worker configs are cleaned + file: + path: "{{ item }}" + state: absent + with_fileglob: + - "{{ matrix_synapse_config_dir_path }}/worker.*.yaml" + - name: Ensure creation of specific worker configs template: src: "{{ role_path }}/templates/synapse/worker.yaml.j2" @@ -46,3 +52,24 @@ src: "{{ role_path }}/files/usr-local-bin/matrix-synapse-worker-write-pid" dest: "{{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid" mode: 0750 + +- name: Ensure jq is installed (Archlinux) + pacman: + name: + - jq + state: present + when: (ansible_distribution == 'Archlinux') + +- name: Ensure jq is installed (CentOS) + yum: + name: + - jq + state: present + when: (ansible_distribution == 'CentOS') + +- name: Ensure jq is installed (Debian) + apt: + name: + - jq + state: present + when: (ansible_os_family == 'Debian') diff --git a/roles/matrix-synapse/tasks/workers/setup_uninstall.yml b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml index 0571114c..58b64760 100644 --- a/roles/matrix-synapse/tasks/workers/setup_uninstall.yml +++ b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml @@ -9,14 +9,11 @@ state: stopped with_dict: "{{ ansible_facts.services|default({})|dict2items|selectattr('key', 'match', 'matrix-synapse-worker@.+\\.service')|list|items2dict }}" -# As we cannot know the ports of workers removed from the enabled_list.. -# => .. just kill them all (FIXME?) -- name: Ensure previous worker service symlinks are cleaned +- name: Ensure worker service symlinks are cleaned file: path: "{{ item.root + '/' + item.path }}" state: absent when: - - not matrix_synapse_workers_enabled|bool - item.state == 'link' - item.path is match('matrix-synapse-worker@.*\\.service') with_filetree: @@ -28,6 +25,13 @@ state: absent register: matrix_synapse_worker_systemd_service_result +- name: Ensure worker configs are cleaned + file: + path: "{{ item.root + '/' + item.path }}" + state: absent + with_fileglob: + - "{{ matrix_synapse_config_dir_path }}/worker.*.yaml" + - name: Remove workers from synapse.wants list set_fact: matrix_synapse_systemd_wanted_services_list: "{{ matrix_synapse_systemd_wanted_services_list | reject('search', item) | list }}" diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 index 2c82873d..983426ba 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 @@ -11,21 +11,21 @@ BindsTo=matrix-synapse.service [Service] Type=simple -# Intentional delay, so that the homeserver (we likely depend on) can manage to start. -ExecStartPre=/bin/sleep 5 +# Intentional delay, so that the homeserver can manage to start. +ExecStartPre={{ matrix_host_command_sleep }} 5 # no sane way of instancing more than one variable (systemd "cant-fix" 🤦) # c.f. https://github.com/systemd/systemd/issues/14895#issuecomment-594123923 # So use good ol' shell parameter expansion to get the worker type.. ExecStart=/bin/sh -c "WORKER=%i; WORKER=$${WORKER%%:*}; \ - exec /usr/bin/docker exec \ + exec {{ matrix_host_command_docker }} exec \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ matrix-synapse \ python -m synapse.app.$${WORKER} -c /data/homeserver.yaml -c /data/worker.%i.yaml" # wait for worker startup & write out PID of actual worker process so systemd can handle it -ExecStartPost=/bin/sleep 5 -ExecStartPost=/usr/local/bin/matrix-synapse-worker-write-pid %i /run/matrix-synapse-worker.%i.pid +ExecStartPost={{ matrix_host_command_sleep }} 5 +ExecStartPost={{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid %i /run/matrix-synapse-worker.%i.pid ExecReload=/bin/kill -HUP $MAINPID ExecStop=/bin/kill $MAINPID From 2d1b9f2dbf33f4178d75c385c0264093ad842ec7 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Wed, 28 Oct 2020 07:13:19 +0100 Subject: [PATCH 24/70] synapse workers: reworkings + get endpoints from upstream docs via awk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (yes, a bit awkward and brittle… xD) --- group_vars/matrix_servers | 11 +- roles/matrix-nginx-proxy/defaults/main.yml | 83 +---------- .../nginx/conf.d/matrix-synapse.conf.j2 | 47 +++--- roles/matrix-synapse/defaults/main.yml | 39 ++--- .../files/workers-doc-to-yaml.awk | 137 ++++++++++++++++++ .../tasks/workers/setup_install.yml | 19 ++- .../templates/synapse/homeserver.yaml.j2 | 10 +- .../synapse/systemd/matrix-synapse.service.j2 | 5 +- .../templates/synapse/worker.yaml.j2 | 14 +- 9 files changed, 226 insertions(+), 139 deletions(-) create mode 100755 roles/matrix-synapse/files/workers-doc-to-yaml.awk diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index c9ab5ef7..891cf6af 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -799,6 +799,15 @@ matrix_nginx_proxy_proxy_matrix_user_directory_search_addr_sans_container: "{{ m matrix_nginx_proxy_self_check_validate_certificates: "{{ false if matrix_ssl_retrieval_method == 'self-signed' else true }}" +matrix_nginx_proxy_synapse_presence_disabled: "{{ not matrix_synapse_use_presence }}" + +matrix_nginx_proxy_synapse_workers_enabled: "{{ matrix_synapse_workers_enabled }}" +matrix_nginx_proxy_synapse_workers_list: "{{ matrix_synapse_workers_enabled_list }}" +matrix_nginx_proxy_synapse_generic_worker_locations: "{{ matrix_synapse_workers_generic_worker_endpoints }}" +matrix_nginx_proxy_synapse_media_repository_locations: "{{ matrix_synapse_workers_media_repository_endpoints }}" +matrix_nginx_proxy_synapse_user_dir_locations: "{{ matrix_synapse_workers_user_dir_endpoints }}" +matrix_nginx_proxy_synapse_frontend_proxy_locations: "{{ matrix_synapse_workers_frontend_proxy_endpoints }}" + matrix_nginx_proxy_systemd_wanted_services_list: | {{ (['matrix-synapse.service']) @@ -1017,7 +1026,7 @@ matrix_synapse_systemd_wanted_services_list: | (['matrix-mailer.service'] if matrix_mailer_enabled else []) }} -# Worker support with redis +# Synapse workers (used for parallel load-scaling) need Redis for IPC. matrix_synapse_redis_enabled: "{{ matrix_redis_enabled }}" matrix_synapse_redis_host: "{{ 'matrix-redis' if matrix_redis_enabled else '' }}" matrix_synapse_redis_password: "{{ matrix_redis_connection_password if matrix_redis_enabled else '' }}" diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 718a040e..b01be702 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -249,79 +249,10 @@ matrix_nginx_proxy_proxy_matrix_nginx_status_enabled: false matrix_nginx_proxy_proxy_matrix_nginx_status_allowed_addresses: ['{{ ansible_default_ipv4.address }}'] -# worker -matrix_nginx_proxy_synapse_workers_enabled: "{{ matrix_synapse_workers_enabled }}" -matrix_nginx_proxy_synapse_workers_enabled_list: "{{ matrix_synapse_workers_enabled_list }}" -matrix_nginx_proxy_synapse_generic_worker_locations: [ - # Sync requests - '^/_matrix/client/(v2_alpha|r0)/sync$', - '^/_matrix/client/(api/v1|v2_alpha|r0)/events$', - '^/_matrix/client/(api/v1|r0)/initialSync$', - '^/_matrix/client/(api/v1|r0)/rooms/[^/]+/initialSync$', - - # Federation requests - '^/_matrix/federation/v1/event/', - '^/_matrix/federation/v1/state/', - '^/_matrix/federation/v1/state_ids/', - '^/_matrix/federation/v1/backfill/', - '^/_matrix/federation/v1/get_missing_events/', - '^/_matrix/federation/v1/publicRooms', - '^/_matrix/federation/v1/query/', - '^/_matrix/federation/v1/make_join/', - '^/_matrix/federation/v1/make_leave/', - '^/_matrix/federation/v1/send_join/', - '^/_matrix/federation/v2/send_join/', - '^/_matrix/federation/v1/send_leave/', - '^/_matrix/federation/v2/send_leave/', - '^/_matrix/federation/v1/invite/', - '^/_matrix/federation/v2/invite/', - '^/_matrix/federation/v1/query_auth/', - '^/_matrix/federation/v1/event_auth/', - '^/_matrix/federation/v1/exchange_third_party_invite/', - '^/_matrix/federation/v1/user/devices/', - '^/_matrix/federation/v1/get_groups_publicised$', - '^/_matrix/key/v2/query', - - # Inbound federation transaction request - '^/_matrix/federation/v1/send/', - - # Client API requests - '^/_matrix/client/(api/v1|r0|unstable)/publicRooms$', - '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/joined_members$', - '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/context/.*$', - '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/members$', - '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state$', - '^/_matrix/client/(api/v1|r0|unstable)/account/3pid$', - '^/_matrix/client/(api/v1|r0|unstable)/keys/query$', - '^/_matrix/client/(api/v1|r0|unstable)/keys/changes$', - '^/_matrix/client/versions$', - '^/_matrix/client/(api/v1|r0|unstable)/voip/turnServer$', - '^/_matrix/client/(api/v1|r0|unstable)/joined_groups$', - '^/_matrix/client/(api/v1|r0|unstable)/publicised_groups$', - '^/_matrix/client/(api/v1|r0|unstable)/publicised_groups/', - - # Registration/login requests - '^/_matrix/client/(api/v1|r0|unstable)/login$', - '^/_matrix/client/(r0|unstable)/register$', - '^/_matrix/client/(r0|unstable)/auth/.*/fallback/web$', - - # Event sending requests - '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send', - '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state/', - '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$', - '^/_matrix/client/(api/v1|r0|unstable)/join/', - '^/_matrix/client/(api/v1|r0|unstable)/profile/', -] - -matrix_nginx_proxy_synapse_media_repository_locations: [ - '^/_matrix/media/', - '^/_synapse/admin/v1/purge_media_cache$', - '^/_synapse/admin/v1/room/.*/media.*$', - '^/_synapse/admin/v1/user/.*/media.*$', - '^/_synapse/admin/v1/media/.*$', - '^/_synapse/admin/v1/quarantine_media/.*$', -] - -matrix_nginx_proxy_synapse_user_dir_locations: [ - '^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$', -] +# synapse worker activation and endpoint mappings +matrix_nginx_proxy_synapse_workers_enabled: false +matrix_nginx_proxy_synapse_workers_list: [] +matrix_nginx_proxy_synapse_generic_worker_locations: [] +matrix_nginx_proxy_synapse_media_repository_locations: [] +matrix_nginx_proxy_synapse_user_dir_locations: [] +matrix_nginx_proxy_synapse_frontend_proxy_locations: [] diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index a49bd8b6..f39c2c34 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -68,6 +68,7 @@ {% endif %} {% if matrix_nginx_proxy_proxy_matrix_user_directory_search_enabled %} + # FIXME: if this is enabled, user_dir_workers should be disabled location /_matrix/client/r0/user_directory/search { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} @@ -102,10 +103,10 @@ {% endif %} {% if matrix_nginx_proxy_synapse_workers_enabled %} - {# Synapse Workers #} + # Workers redirects BEGIN - {% if generic_worker_workers %} - {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappgeneric_worker #} + {% if generic_workers %} + # https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappgeneric_worker {% for location in matrix_nginx_proxy_synapse_generic_worker_locations %} location ~ {{ location }} { proxy_pass http://generic_worker_upstream$request_uri; @@ -113,11 +114,11 @@ proxy_set_header X-Forwarded-For $remote_addr; } {% endfor %} - {# ToDo: add GET ^/_matrix/federation/v1/groups/ #} + # FIXME: add GET ^/_matrix/federation/v1/groups/ {% endif %} {% if media_repository_workers %} - {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappmedia_repository #} + # https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappmedia_repository {% for location in matrix_nginx_proxy_synapse_media_repository_locations %} location ~ {{ location }} { proxy_pass http://media_repository_upstream$request_uri; @@ -128,7 +129,8 @@ {% endif %} {% if user_dir_workers %} - {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappuser_dir #} + # FIXME: obsolete if matrix_nginx_proxy_proxy_matrix_user_directory_search_enabled is set + # https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappuser_dir {% for location in matrix_nginx_proxy_synapse_user_dir_locations %} location ~ {{ location }} { proxy_pass http://user_dir_upstream$request_uri; @@ -139,13 +141,16 @@ {% endif %} {% if frontend_proxy_workers %} - {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappfrontend_proxy #} - location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload { + # https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappfrontend_proxy + {% for location in matrix_nginx_proxy_synapse_frontend_proxy_locations %} + location ~ {{ location }} { proxy_pass http://frontend_proxy_upstream$request_uri; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; } - {% if not matrix_synapse_use_presence %} + {% endfor %} + {% if matrix_nginx_proxy_synapse_presence_disabled %} + # FIXME: keep in sync with synapse workers documentation manually location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status { proxy_pass http://frontend_proxy_upstream$request_uri; proxy_set_header Host $host; @@ -153,6 +158,7 @@ } {% endif %} {% endif %} + # Workers redirects END {% endif %} @@ -229,20 +235,20 @@ } {% endmacro %} -{% set generic_worker_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'generic_worker')|list %} -{% set media_repository_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|list %} -{% set user_dir_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|list %} -{% set frontend_proxy_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'frontend_proxy')|list %} +{% set generic_workers = matrix_nginx_proxy_synapse_workers_list|selectattr('type', 'equalto', 'generic_worker')|list %} +{% set media_repository_workers = matrix_nginx_proxy_synapse_workers_list|selectattr('type', 'equalto', 'media_repository')|list %} +{% set user_dir_workers = matrix_nginx_proxy_synapse_workers_list|selectattr('type', 'equalto', 'user_dir')|list %} +{% set frontend_proxy_workers = matrix_nginx_proxy_synapse_workers_list|selectattr('type', 'equalto', 'frontend_proxy')|list %} {% if matrix_nginx_proxy_synapse_workers_enabled %} - {# Setup upstream for groups of workers #} + # Round Robin "upstream" pools for workers - {% if generic_worker_workers %} + {% if generic_workers %} upstream generic_worker_upstream { # ensures that requests from the same client will always be passed # to the same server (except when this server is unavailable) ip_hash; - {% for worker in generic_worker_workers %} + {% for worker in generic_workers %} server "matrix-synapse:{{ worker.port }}"; {% endfor %} } @@ -250,7 +256,6 @@ {% if frontend_proxy_workers %} upstream frontend_proxy_upstream { - # Round Robin {% for worker in frontend_proxy_workers %} server "matrix-synapse:{{ worker.port }}"; {% endfor %} @@ -259,7 +264,6 @@ {% if media_repository_workers %} upstream media_repository_upstream { - # Round Robin {% for worker in media_repository_workers %} server "matrix-synapse:{{ worker.port }}"; {% endfor %} @@ -268,7 +272,6 @@ {% if user_dir_workers %} upstream user_dir_upstream { - # Round Robin {% for worker in user_dir_workers %} server "matrix-synapse:{{ worker.port }}"; {% endfor %} @@ -358,8 +361,8 @@ server { {% endif %} {% if matrix_nginx_proxy_synapse_workers_enabled %} - {% if generic_worker_workers %} - {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappgeneric_worker #} + {% if generic_workers %} + # https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappgeneric_worker {% for location in matrix_nginx_proxy_synapse_generic_worker_locations %} location ~ {{ location }} { proxy_pass http://generic_worker_upstream$request_uri; @@ -367,7 +370,7 @@ server { proxy_set_header X-Forwarded-For $remote_addr; } {% endfor %} - {# ToDo: add GET ^/_matrix/federation/v1/groups/ #} + # FIXME: add GET ^/_matrix/federation/v1/groups/ {% endif %} {% endif %} diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 2cdd839c..47287f28 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -275,31 +275,22 @@ matrix_synapse_manhole_enabled: false # Enable support for Synapse workers matrix_synapse_workers_enabled: false -# Default list of workers to spawn -# (worker with ports ending on 00 are single-instance) +# Default list of workers to spawn (order in accord to docs) +# - no endpoints / doesn't need port mapping if port ends on 0 +# - single-instance-only if 2nd last digit of port number is 0 matrix_synapse_workers_enabled_list: - - { worker: generic_worker, port: 18101 } - - { worker: generic_worker, port: 18102 } - - { worker: generic_worker, port: 18103 } - - { worker: generic_worker, port: 18104 } - - { worker: generic_worker, port: 18105 } - - { worker: generic_worker, port: 18106 } - - { worker: appservice, port: 18200 } - - { worker: federation_sender, port: 18301 } - - { worker: frontend_proxy, port: 18400 } - - { worker: media_repository, port: 18501 } - - { worker: pusher, port: 18600 } - - { worker: user_dir, port: 18700 } - -# The list of available workers (2020-08-28) -matrix_synapse_workers_avail_list: - - generic_worker - - appservice - - federation_sender - - frontend_proxy - - media_repository - - pusher - - user_dir + - { type: generic_worker, port: 18111 } + - { type: generic_worker, port: 18112 } + - { type: generic_worker, port: 18113 } + - { type: generic_worker, port: 18114 } + - { type: generic_worker, port: 18115 } + - { type: generic_worker, port: 18116 } + - { type: pusher, port: 00 } + - { type: appservice, port: 00 } + - { type: federation_sender, port: 0 } + - { type: media_repository, port: 18221 } + - { type: user_dir, port: 18331 } + - { type: frontend_proxy, port: 18441 } # Redis information matrix_synapse_redis_enabled: false diff --git a/roles/matrix-synapse/files/workers-doc-to-yaml.awk b/roles/matrix-synapse/files/workers-doc-to-yaml.awk new file mode 100755 index 00000000..e0474645 --- /dev/null +++ b/roles/matrix-synapse/files/workers-doc-to-yaml.awk @@ -0,0 +1,137 @@ +#!/usr/bin/awk +# Hackish approach to get a machine-readable list of current matrix +# synapse REST API endpoints from the official documentation at +# https://github.com/matrix-org/synapse/raw/master/docs/workers.md +# +# invoke in shell with: +# URL=https://github.com/matrix-org/synapse/raw/master/docs/workers.md +# curl -L ${URL} | awk -f parse-workers-docs.awk - + +function worker_stanza_append(string) { + worker_stanza = worker_stanza string +} + +function line_is_endpoint_url(line) { + # probably API endpoint if it starts with white-space and ^ or / + return (line ~ /^ +[\^/].*\//) +} + +# Put YAML marker at beginning of file. +BEGIN { + print "---" + endpoint_conditional_comment = " # FIXME: ADDITIONAL CONDITIONS REQUIRED: to be enabled manually\n" +} + +# Enable further processing after the introductory text. +# Read each synapse worker section as record and its lines as fields. +/Available worker applications/ { + enable_parsing = 1 + # set record separator to markdown section header + RS = "\n### " + # set field separator to newline + FS = "\n" +} + +# Once parsing is active, this will process each section as record. +enable_parsing { + # Each worker section starts with a synapse.app.X headline + if ($1 ~ /synapse\.app\./) { + + # get rid of the backticks and extract worker type from headline + gsub("`", "", $1) + gsub("synapse.app.", "", $1) + worker_type = $1 + + # initialize empty worker stanza + worker_stanza = "" + + # track if any endpoints are mentioned in a specific section + worker_has_urls = 0 + + # some endpoint descriptions contain flag terms + endpoints_seem_conditional = 0 + + # also, collect a list of available workers + workers = (workers ? workers "\n" : "") " - " worker_type + + # loop through the lines (2 - number of fields in record) + for (i = 1; i < NF + 1; i++) { + # copy line for gsub replacements + line = $i + + # end all lines but the last with a linefeed + linefeed = (i < NF - 1) ? "\n" : "" + + # line starts with white-space and a hash: endpoint block headline + if (line ~ /^ +#/) { + + # copy to output verbatim, normalizing white-space + gsub(/^ +/, "", line) + worker_stanza_append(" " line linefeed) + + } else if (line_is_endpoint_url(line)) { + + # mark section for special output formatting + worker_has_urls = 1 + + # remove leading white-space + gsub(/^ +/, "", line) + api_endpoint_regex = line + + # FIXME: https://github.com/matrix-org/synapse/issues/new + # munge inconsistent media_repository endpoint notation + if (api_endpoint_regex == "/_matrix/media/") { + api_endpoint_regex = "^" line + } + + # disable endpoints which specify complications + if (endpoints_seem_conditional) { + # only add notice if previous line didn't match + if (!line_is_endpoint_url($(i - 1))) { + worker_stanza_append(endpoint_conditional_comment) + } + worker_stanza_append(" # " api_endpoint_regex linefeed) + } else { + # output endpoint regex + worker_stanza_append(" - " api_endpoint_regex linefeed) + } + + # white-space only line? + } else if (line ~ /^\w*$/) { + + if (i > 3 && i < NF) { + # print white-space lines unless 1st or last line in section + worker_stanza_append(line linefeed) + } + + # nothing of the above: the line is regular documentation text + } else { + + # include this text line as comment + worker_stanza_append(" # " line linefeed) + + # and take note of words hinting at additional conditions to be met + if (line ~ /\<[Ii]f\>|\<[Ff]or\>/) { + endpoints_seem_conditional = 1 + } + } + } + + if (worker_has_urls) { + print "\nmatrix_synapse_workers_" worker_type "_endpoints:" + print worker_stanza + } else { + # include workers without endpoints as well for reference + print "\n# " worker_type " worker (no API endpoints) [" + print worker_stanza + print "# ]" + } + } +} + +END { + print "\nmatrix_synapse_workers_avail_list:" + print workers | "sort" +} + +# vim: tabstop=4 shiftwidth=4 expandtab autoindent diff --git a/roles/matrix-synapse/tasks/workers/setup_install.yml b/roles/matrix-synapse/tasks/workers/setup_install.yml index 3f1f8ac0..cbd73643 100644 --- a/roles/matrix-synapse/tasks/workers/setup_install.yml +++ b/roles/matrix-synapse/tasks/workers/setup_install.yml @@ -1,5 +1,18 @@ --- +- name: Download synapse workers doc + get_url: + url: https://github.com/matrix-org/synapse/raw/master/docs/workers.md + dest: "{{ role_path }}/files/workers.upstream-documentation.md" + +- name: Download synapse workers doc and convert into YAML + shell: + cmd: "awk -f {{ role_path }}/files/workers-doc-to-yaml.awk -- {{ role_path }}/files/workers.upstream-documentation.md > {{ role_path }}/vars/workers.yml" + creates: "{{ role_path }}/vars/workers.yml" + +- name: Load list of available worker apps and endpoints + include_vars: "{{ role_path }}/vars/workers.yml" + - name: Ensure synapse worker base service file installed template: src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker@.service.j2" @@ -23,7 +36,7 @@ - name: Ensure individual worker service symlinks exist service: - name: "matrix-synapse-worker@{{ item.worker }}:{{ item.port }}.service" + name: "matrix-synapse-worker@{{ item.type }}:{{ item.port }}.service" enabled: true with_items: "{{ matrix_synapse_workers_enabled_list }}" @@ -37,14 +50,14 @@ - name: Ensure creation of specific worker configs template: src: "{{ role_path }}/templates/synapse/worker.yaml.j2" - dest: "{{ matrix_synapse_config_dir_path }}/worker.{{ item.worker }}:{{ item.port }}.yaml" + dest: "{{ matrix_synapse_config_dir_path }}/worker.{{ item.type }}:{{ item.port }}.yaml" with_list: "{{ matrix_synapse_workers_enabled_list }}" - name: Add workers to synapse.wants list set_fact: matrix_synapse_systemd_wanted_services_list: > {{ matrix_synapse_systemd_wanted_services_list + - ['matrix-synapse-worker@' + item.worker + ':' + item.port|string + '.service'] }} + ['matrix-synapse-worker@' + item.type + ':' + item.port|string + '.service'] }} with_items: "{{ matrix_synapse_workers_enabled_list }}" - name: Ensure matrix-synapse-worker-write-pid script is created diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index 5d4dff4e..e69de451 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -249,19 +249,19 @@ worker_app: synapse.app.homeserver # thx https://oznetnerd.com/2017/04/18/jinja2-selectattr-filter/ # reduce the main worker's offerings to core homeserver business -{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'appservice')|list %} +{% if matrix_synapse_workers_enabled_list|selectattr('type', 'equalto', 'appservice')|list %} notify_appservices: false {% endif %} -{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_sender')|list %} +{% if matrix_synapse_workers_enabled_list|selectattr('type', 'equalto', 'federation_sender')|list %} send_federation: false {% endif %} -{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|list %} +{% if matrix_synapse_workers_enabled_list|selectattr('type', 'equalto', 'media_repository')|list %} enable_media_repo: false {% endif %} -{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'pusher')|list %} +{% if matrix_synapse_workers_enabled_list|selectattr('type', 'equalto', 'pusher')|list %} start_pushers: false {% endif %} -{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|list %} +{% if matrix_synapse_workers_enabled_list|selectattr('type', 'equalto', 'user_dir')|list %} update_user_directory: false {% endif %} diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 index 9a5ce8e4..757ef23a 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 @@ -43,9 +43,12 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \ {% if matrix_synapse_manhole_enabled and matrix_synapse_container_manhole_api_host_bind_port %} -p {{ matrix_synapse_container_manhole_api_host_bind_port }}:9000 \ {% endif %} - {% if matrix_synapse_workers_enabled %} + {% if matrix_synapse_workers_enabled and not matrix_nginx_proxy_enabled|default(False) %} + {# Expose worker (by default 18xxx range) ports on host if not using internal nginx proxy #} {% for worker in matrix_synapse_workers_enabled_list %} + {% if worker.port != 0 %} -p {{ worker.port }}:{{ worker.port }} \ + {% endif %} {% endfor %} {% endif %} -v {{ matrix_synapse_config_dir_path }}:/data:ro \ diff --git a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 index 0a282ba7..2ee606dc 100644 --- a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 @@ -1,27 +1,27 @@ #jinja2: lstrip_blocks: "True" -worker_app: synapse.app.{{ item.worker }} -worker_name: {{ item.worker ~ ':' ~ item.port }} +worker_app: synapse.app.{{ item.type }} +worker_name: {{ item.type ~ ':' ~ item.port }} worker_replication_host: 127.0.0.1 worker_replication_http_port: {{ matrix_synapse_replication_http_port }} -{% if item.worker not in [ 'appservice', 'federation_sender', 'pusher' ] %} +{% if item.type not in [ 'appservice', 'federation_sender', 'pusher' ] %} worker_listeners: - type: http port: {{ item.port }} resources: - names: -{% if item.worker in [ 'generic_worker', 'frontend_proxy', 'user_dir' ] %} +{% if item.type in [ 'generic_worker', 'frontend_proxy', 'user_dir' ] %} - client {% endif %} -{% if item.worker in [ 'generic_worker' ] %} +{% if item.type in [ 'generic_worker' ] %} - federation -{% elif item.worker in [ 'media_repository' ] %} +{% elif item.type in [ 'media_repository' ] %} - media {% endif %} {% endif %} -{% if item.worker == 'frontend_proxy' %} +{% if item.type == 'frontend_proxy' %} worker_main_http_uri: http://127.0.0.1:8008 {% endif %} From e078e29ef8266171b6945f388c6504b7f253f33f Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Wed, 28 Oct 2020 08:39:31 +0100 Subject: [PATCH 25/70] synapse workers: fix self name in workers-doc-to-yaml.awk script --- roles/matrix-synapse/files/workers-doc-to-yaml.awk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/matrix-synapse/files/workers-doc-to-yaml.awk b/roles/matrix-synapse/files/workers-doc-to-yaml.awk index e0474645..b3c79eb5 100755 --- a/roles/matrix-synapse/files/workers-doc-to-yaml.awk +++ b/roles/matrix-synapse/files/workers-doc-to-yaml.awk @@ -1,11 +1,11 @@ #!/usr/bin/awk -# Hackish approach to get a machine-readable list of current matrix +# Hackish approach to get a machine-readable list of current matrix # synapse REST API endpoints from the official documentation at # https://github.com/matrix-org/synapse/raw/master/docs/workers.md # # invoke in shell with: # URL=https://github.com/matrix-org/synapse/raw/master/docs/workers.md -# curl -L ${URL} | awk -f parse-workers-docs.awk - +# curl -L ${URL} | awk -f workers-doc-to-yaml.awk - function worker_stanza_append(string) { worker_stanza = worker_stanza string From cce90b187a95fcb9a1d146340e9f5d1ad3460423 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Wed, 28 Oct 2020 23:09:21 +0100 Subject: [PATCH 26/70] synapse workers: fix undefined variable cases when removing workers --- group_vars/matrix_servers | 8 ++++---- roles/matrix-synapse/tasks/workers/setup_uninstall.yml | 5 ++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 891cf6af..3744a65e 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -803,10 +803,10 @@ matrix_nginx_proxy_synapse_presence_disabled: "{{ not matrix_synapse_use_presenc matrix_nginx_proxy_synapse_workers_enabled: "{{ matrix_synapse_workers_enabled }}" matrix_nginx_proxy_synapse_workers_list: "{{ matrix_synapse_workers_enabled_list }}" -matrix_nginx_proxy_synapse_generic_worker_locations: "{{ matrix_synapse_workers_generic_worker_endpoints }}" -matrix_nginx_proxy_synapse_media_repository_locations: "{{ matrix_synapse_workers_media_repository_endpoints }}" -matrix_nginx_proxy_synapse_user_dir_locations: "{{ matrix_synapse_workers_user_dir_endpoints }}" -matrix_nginx_proxy_synapse_frontend_proxy_locations: "{{ matrix_synapse_workers_frontend_proxy_endpoints }}" +matrix_nginx_proxy_synapse_generic_worker_locations: "{{ matrix_synapse_workers_generic_worker_endpoints|default([]) }}" +matrix_nginx_proxy_synapse_media_repository_locations: "{{ matrix_synapse_workers_media_repository_endpoints|default([]) }}" +matrix_nginx_proxy_synapse_user_dir_locations: "{{ matrix_synapse_workers_user_dir_endpoints|default([]) }}" +matrix_nginx_proxy_synapse_frontend_proxy_locations: "{{ matrix_synapse_workers_frontend_proxy_endpoints|default([]) }}" matrix_nginx_proxy_systemd_wanted_services_list: | {{ diff --git a/roles/matrix-synapse/tasks/workers/setup_uninstall.yml b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml index 58b64760..879e6998 100644 --- a/roles/matrix-synapse/tasks/workers/setup_uninstall.yml +++ b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml @@ -27,15 +27,14 @@ - name: Ensure worker configs are cleaned file: - path: "{{ item.root + '/' + item.path }}" + path: "{{ item }}" state: absent with_fileglob: - "{{ matrix_synapse_config_dir_path }}/worker.*.yaml" - name: Remove workers from synapse.wants list set_fact: - matrix_synapse_systemd_wanted_services_list: "{{ matrix_synapse_systemd_wanted_services_list | reject('search', item) | list }}" - with_items: "{{ matrix_synapse_workers_avail_list }}" + matrix_synapse_systemd_wanted_services_list: "{{ matrix_synapse_systemd_wanted_services_list | reject('search', '^matrix-synapse-worker@') | list }}" - name: Ensure systemd noticed removal of worker service units service: From e5072c20d98f13ad27425183c76eb5304a94b0bc Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Tue, 10 Nov 2020 20:35:39 +0100 Subject: [PATCH 27/70] synapse workers/nginx: handle media_repository worker endpoints on federation port to prevent "404 on the federation port for the path `/_matrix/media`, if a remote server is trying to get the media object on federation port, see https://github.com/matrix-org/synapse/issues/8695 " https://github.com/matrix-org/synapse/pull/8701 --- .../templates/nginx/conf.d/matrix-synapse.conf.j2 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index f39c2c34..1dbbb844 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -372,6 +372,16 @@ server { {% endfor %} # FIXME: add GET ^/_matrix/federation/v1/groups/ {% endif %} + {% if media_repository_workers %} + # https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappmedia_repository + {% for location in matrix_nginx_proxy_synapse_media_repository_locations %} + location ~ {{ location }} { + proxy_pass http://media_repository_upstream$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endfor %} + {% endif %} {% endif %} location / { From 1e971312e806123c5677fc7104d435e2b69baedf Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Tue, 10 Nov 2020 21:23:19 +0100 Subject: [PATCH 28/70] synapse workers: handle auth fallback endpoint on main process only (allegedly breaks with SSO enabled) --- roles/matrix-synapse/files/workers-doc-to-yaml.awk | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/roles/matrix-synapse/files/workers-doc-to-yaml.awk b/roles/matrix-synapse/files/workers-doc-to-yaml.awk index b3c79eb5..fe018b6f 100755 --- a/roles/matrix-synapse/files/workers-doc-to-yaml.awk +++ b/roles/matrix-synapse/files/workers-doc-to-yaml.awk @@ -84,6 +84,15 @@ enable_parsing { api_endpoint_regex = "^" line } + # FIXME: https://github.com/matrix-org/synapse/issues/7530 + # https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/456#issuecomment-719015911 + if (api_endpoint_regex == "^/_matrix/client/(r0|unstable)/auth/.*/fallback/web$") { + worker_stanza_append(" # FIXME: possible bug with SSO and multiple generic workers\n") + worker_stanza_append(" # see https://github.com/matrix-org/synapse/issues/7530\n") + worker_stanza_append(" # " api_endpoint_regex linefeed) + continue + } + # disable endpoints which specify complications if (endpoints_seem_conditional) { # only add notice if previous line didn't match From b05d298ae44f2befc227845c9ff45aa64424cf25 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Tue, 10 Nov 2020 21:43:33 +0100 Subject: [PATCH 29/70] synapse workers nginx rule: add client_max_body_size on media endpoints so transfer limits are properly set in accord to the relevant setting https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/456#issuecomment-719996778 --- .../templates/nginx/conf.d/matrix-synapse.conf.j2 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 1dbbb844..86b3762d 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -124,6 +124,10 @@ proxy_pass http://media_repository_upstream$request_uri; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; + + client_body_buffer_size 25M; + client_max_body_size {{ matrix_nginx_proxy_proxy_matrix_client_api_client_max_body_size_mb }}M; + proxy_max_temp_file_size 0; } {% endfor %} {% endif %} @@ -379,6 +383,10 @@ server { proxy_pass http://media_repository_upstream$request_uri; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; + + client_body_buffer_size 25M; + client_max_body_size {{ matrix_nginx_proxy_proxy_matrix_federation_api_client_max_body_size_mb }}M; + proxy_max_temp_file_size 0; } {% endfor %} {% endif %} From dd402bee928997459b0b25254700e2f6b87b0afa Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Tue, 10 Nov 2020 22:22:40 +0100 Subject: [PATCH 30/70] synapse workers: add rudimentary documentation on worker support --- docs/configuring-playbook-synapse.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/configuring-playbook-synapse.md b/docs/configuring-playbook-synapse.md index 019a651f..276baca5 100644 --- a/docs/configuring-playbook-synapse.md +++ b/docs/configuring-playbook-synapse.md @@ -18,6 +18,21 @@ Alternatively, **if there is no pre-defined variable** for a Synapse setting you - or, if extending the configuration is still not powerful enough for your needs, you can **override the configuration completely** using `matrix_synapse_configuration` (or `matrix_synapse_configuration_yaml`). You can find information about this in [`roles/matrix-synapse/defaults/main.yml`](../roles/matrix-synapse/defaults/main.yml). +## Load balancing with workers +To have synapse gracefully handle thousands of users, worker support should be enabled. It factors out some homeserver tasks and spreads the load of incoming client and server-to-server traffic between multiple processes. More information can be found at https://github.com/matrix-org/synapse/blob/master/docs/workers.md (which, coincidentally, also is the file which an awk script extracts the endpoint URLs from when running with tag `setup-synapse`). + +To enable synapse worker support, set + +```yaml +matrix_synapse_workers_enabled: true +``` + +in your `inventory/host_vars/matrix.DOMAIN/vars.yml` file. +There, you can also override the default `matrix_synapse_workers_enabled_list` from [`roles/matrix-synapse/defaults/main.yml`](../roles/matrix-synapse/defaults/main.yml). + +If you are not using the inbuilt nginx proxy container but an instance managed by yourself, you are currently on your own as the template needs yet to be adapted to better support this use case. + + ## Synapse Admin Certain Synapse administration tasks (managing users and rooms, etc.) can be performed via a web user-interace, if you install [Synapse Admin](configuring-playbook-synapse-admin.md). From f3d2797d9ccc4348f131ad35dc1f4220a7b47919 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Tue, 10 Nov 2020 22:40:48 +0100 Subject: [PATCH 31/70] synapse workers: make awk script invocation handle paths with spaces (quoting ftw) --- roles/matrix-synapse/tasks/workers/setup_install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse/tasks/workers/setup_install.yml b/roles/matrix-synapse/tasks/workers/setup_install.yml index cbd73643..6b246a18 100644 --- a/roles/matrix-synapse/tasks/workers/setup_install.yml +++ b/roles/matrix-synapse/tasks/workers/setup_install.yml @@ -7,7 +7,7 @@ - name: Download synapse workers doc and convert into YAML shell: - cmd: "awk -f {{ role_path }}/files/workers-doc-to-yaml.awk -- {{ role_path }}/files/workers.upstream-documentation.md > {{ role_path }}/vars/workers.yml" + cmd: "awk -f '{{ role_path }}/files/workers-doc-to-yaml.awk' -- '{{ role_path }}/files/workers.upstream-documentation.md' > '{{ role_path }}/vars/workers.yml'" creates: "{{ role_path }}/vars/workers.yml" - name: Load list of available worker apps and endpoints From 5598a89ad5b81e9e9ce7e14e4ac81c52a354d6b8 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Tue, 10 Nov 2020 23:00:24 +0100 Subject: [PATCH 32/70] synapse workers doc: link to relevant synapse issue list search --- docs/configuring-playbook-synapse.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuring-playbook-synapse.md b/docs/configuring-playbook-synapse.md index 276baca5..c12386c6 100644 --- a/docs/configuring-playbook-synapse.md +++ b/docs/configuring-playbook-synapse.md @@ -32,6 +32,8 @@ There, you can also override the default `matrix_synapse_workers_enabled_list` f If you are not using the inbuilt nginx proxy container but an instance managed by yourself, you are currently on your own as the template needs yet to be adapted to better support this use case. +In case any problems occur, make sure to have a look at the [list of synapse issues about workers](https://github.com/matrix-org/synapse/issues?q=workers+in%3Atitle) and your `journalctl --unit 'matrix-*'`. + ## Synapse Admin From e314613deddbb7d3b3260c4d819afe9f6be25db8 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Tue, 1 Dec 2020 21:52:59 +0100 Subject: [PATCH 33/70] Add files created by workers-doc-to-yaml.awk to .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 36c65bda..c5279a46 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,7 @@ !/inventory/host_vars/.gitkeep !/inventory/scripts /roles/*/files/scratchpad +/roles/matrix-synapse/files/workers.upstream-documentation.md +/roles/matrix-synapse/vars/workers.yml .DS_Store .python-version From 851c25c47f0fcfca9f701ffa415a1555db9d2c95 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Tue, 1 Dec 2020 21:55:07 +0100 Subject: [PATCH 34/70] matrix-synapse nginx template: fix invalid jinja comment syntax --- .../templates/nginx/conf.d/matrix-synapse.conf.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 36f02b81..54ec11ca 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -105,7 +105,7 @@ {% endif %} {% if matrix_nginx_proxy_synapse_workers_enabled %} - {# Workers redirects BEGIN} + {# Workers redirects BEGIN #} {% if generic_workers %} # https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappgeneric_worker @@ -164,7 +164,7 @@ } {% endif %} {% endif %} - {# Workers redirects END} + {# Workers redirects END #} {% endif %} From d5932ca393d26c6b82dd32077609f19477121e8b Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Tue, 1 Dec 2020 22:18:42 +0100 Subject: [PATCH 35/70] synapse role workers setup: execute the endpoint extraction locally Thanks @maxklenk ! --- roles/matrix-synapse/tasks/workers/setup_install.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/roles/matrix-synapse/tasks/workers/setup_install.yml b/roles/matrix-synapse/tasks/workers/setup_install.yml index 6b246a18..9aa0c4ec 100644 --- a/roles/matrix-synapse/tasks/workers/setup_install.yml +++ b/roles/matrix-synapse/tasks/workers/setup_install.yml @@ -1,14 +1,20 @@ --- - name: Download synapse workers doc - get_url: + local_action: + module: get_url url: https://github.com/matrix-org/synapse/raw/master/docs/workers.md dest: "{{ role_path }}/files/workers.upstream-documentation.md" + vars: + ansible_become: no - name: Download synapse workers doc and convert into YAML - shell: + local_action: + module: shell cmd: "awk -f '{{ role_path }}/files/workers-doc-to-yaml.awk' -- '{{ role_path }}/files/workers.upstream-documentation.md' > '{{ role_path }}/vars/workers.yml'" creates: "{{ role_path }}/vars/workers.yml" + vars: + ansible_become: no - name: Load list of available worker apps and endpoints include_vars: "{{ role_path }}/vars/workers.yml" From 414b812a29b35e0bceb45e17fa7450d44d58dc07 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Tue, 1 Dec 2020 22:20:27 +0100 Subject: [PATCH 36/70] synapse role workers setup: make configs clean action remote compatible Many people probably didn't even know this - that ansible can be quite a bit picky about what it will be willing to work with remotely. Thanks @maxklenk ! --- roles/matrix-synapse/tasks/workers/setup_install.yml | 12 +++++++++--- .../matrix-synapse/tasks/workers/setup_uninstall.yml | 12 +++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/roles/matrix-synapse/tasks/workers/setup_install.yml b/roles/matrix-synapse/tasks/workers/setup_install.yml index 9aa0c4ec..6cd7ae42 100644 --- a/roles/matrix-synapse/tasks/workers/setup_install.yml +++ b/roles/matrix-synapse/tasks/workers/setup_install.yml @@ -46,12 +46,18 @@ enabled: true with_items: "{{ matrix_synapse_workers_enabled_list }}" +- name: Find worker configs to be cleaned + find: + path: "{{ matrix_synapse_config_dir_path }}" + patterns: "worker.*.yaml" + use_regex: true + register: worker_config_files + - name: Ensure previous worker configs are cleaned file: - path: "{{ item }}" + path: "{{ item.path }}" state: absent - with_fileglob: - - "{{ matrix_synapse_config_dir_path }}/worker.*.yaml" + with_items: "{{ worker_config_files.files }}" - name: Ensure creation of specific worker configs template: diff --git a/roles/matrix-synapse/tasks/workers/setup_uninstall.yml b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml index 879e6998..a9884fca 100644 --- a/roles/matrix-synapse/tasks/workers/setup_uninstall.yml +++ b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml @@ -25,12 +25,18 @@ state: absent register: matrix_synapse_worker_systemd_service_result +- name: Find worker configs to be cleaned + find: + path: "{{ matrix_synapse_config_dir_path }}" + patterns: "worker.*.yaml" + use_regex: true + register: worker_config_files + - name: Ensure worker configs are cleaned file: - path: "{{ item }}" + path: "{{ item.path }}" state: absent - with_fileglob: - - "{{ matrix_synapse_config_dir_path }}/worker.*.yaml" + with_items: "{{ worker_config_files.files }}" - name: Remove workers from synapse.wants list set_fact: From af08f1877913210e4c7c9b6f7d2f84107a93d7b9 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Tue, 1 Dec 2020 22:22:04 +0100 Subject: [PATCH 37/70] synapse workers default config: disable user_dir worker for now (until https://github.com/matrix-org/synapse/issues/8787 is resolved) --- roles/matrix-synapse/defaults/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index eb9c40e7..b0e8637e 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -291,7 +291,8 @@ matrix_synapse_workers_enabled_list: - { type: appservice, port: 00 } - { type: federation_sender, port: 0 } - { type: media_repository, port: 18221 } - - { type: user_dir, port: 18331 } +# disable until https://github.com/matrix-org/synapse/issues/8787 resolved +# - { type: user_dir, port: 18331 } - { type: frontend_proxy, port: 18441 } # Redis information From f201bca519b423950a977ea409aa3f2681b1496b Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Tue, 1 Dec 2020 22:49:15 +0100 Subject: [PATCH 38/70] synapse workers: define and expose METRICS port for each worker As seen on TV: https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md#monitoring-workers --- roles/matrix-synapse/defaults/main.yml | 24 +++++++++---------- .../synapse/systemd/matrix-synapse.service.j2 | 10 +++++--- .../templates/synapse/worker.yaml.j2 | 4 ++++ 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index b0e8637e..1a87c149 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -281,19 +281,19 @@ matrix_synapse_workers_enabled: false # - no endpoints / doesn't need port mapping if port ends on 0 # - single-instance-only if 2nd last digit of port number is 0 matrix_synapse_workers_enabled_list: - - { type: generic_worker, port: 18111 } - - { type: generic_worker, port: 18112 } - - { type: generic_worker, port: 18113 } - - { type: generic_worker, port: 18114 } - - { type: generic_worker, port: 18115 } - - { type: generic_worker, port: 18116 } - - { type: pusher, port: 00 } - - { type: appservice, port: 00 } - - { type: federation_sender, port: 0 } - - { type: media_repository, port: 18221 } + - { type: generic_worker, port: 18111, metrics_port: 19111 } + - { type: generic_worker, port: 18112, metrics_port: 19112 } + - { type: generic_worker, port: 18113, metrics_port: 19113 } + - { type: generic_worker, port: 18114, metrics_port: 19114 } + - { type: generic_worker, port: 18115, metrics_port: 19115 } + - { type: generic_worker, port: 18116, metrics_port: 19116 } + - { type: pusher, port: 00, metrics_port: 19200 } + - { type: appservice, port: 00, metrics_port: 19300 } + - { type: federation_sender, port: 0, metrics_port: 19400 } + - { type: media_repository, port: 18551, metrics_port: 19551 } # disable until https://github.com/matrix-org/synapse/issues/8787 resolved -# - { type: user_dir, port: 18331 } - - { type: frontend_proxy, port: 18441 } +# - { type: user_dir, port: 18661, metrics_port: 19661 } + - { type: frontend_proxy, port: 18771, metrics_port: 19771 } # Redis information matrix_synapse_redis_enabled: false diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 index b65d9645..216d1aef 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 @@ -43,14 +43,18 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \ {% if matrix_synapse_manhole_enabled and matrix_synapse_container_manhole_api_host_bind_port %} -p {{ matrix_synapse_container_manhole_api_host_bind_port }}:9000 \ {% endif %} - {% if matrix_synapse_workers_enabled and not matrix_nginx_proxy_enabled|default(False) %} - {# Expose worker (by default 18xxx range) ports on host if not using internal nginx proxy #} {% for worker in matrix_synapse_workers_enabled_list %} + {% if matrix_synapse_workers_enabled and not matrix_nginx_proxy_enabled|default(False) %} + {# Expose worker ports (by default 18xxx range) on host if not using internal nginx proxy #} {% if worker.port != 0 %} -p {{ worker.port }}:{{ worker.port }} \ {% endif %} - {% endfor %} {% endif %} + {# Expose worker metrics ports on host if defined #} + {% if worker.metrics_port != 0 %} + -p {{ worker.metrics_port }}:{{ worker.metrics_port }} \ + {% endif %} + {% endfor %} --mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/data,ro \ --mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \ {% for volume in matrix_synapse_container_additional_volumes %} diff --git a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 index 2ee606dc..c8876545 100644 --- a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 @@ -21,6 +21,10 @@ worker_listeners: {% endif %} {% endif %} + - type: metrics + bind_address: '' + port: {{ item.metrics_port }} + {% if item.type == 'frontend_proxy' %} worker_main_http_uri: http://127.0.0.1:8008 {% endif %} From e892ac464f1708e912cafdd19654b6f7dadc0cda Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Tue, 1 Dec 2020 23:49:23 +0100 Subject: [PATCH 39/70] synapse workers: untangle config template and specify bind address .. to mitigate log noise - WARNING: Failed to listen on 0.0.0.0, continuing because listening on [::] --- roles/matrix-synapse/templates/synapse/worker.yaml.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 index c8876545..f77ff4ad 100644 --- a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 @@ -5,8 +5,8 @@ worker_name: {{ item.type ~ ':' ~ item.port }} worker_replication_host: 127.0.0.1 worker_replication_http_port: {{ matrix_synapse_replication_http_port }} -{% if item.type not in [ 'appservice', 'federation_sender', 'pusher' ] %} worker_listeners: +{% if item.type not in [ 'appservice', 'federation_sender', 'pusher' ] %} - type: http port: {{ item.port }} resources: @@ -18,11 +18,11 @@ worker_listeners: - federation {% elif item.type in [ 'media_repository' ] %} - media -{% endif %} {% endif %} +{% endif %} - type: metrics - bind_address: '' + bind_address: ['127.0.0.1'] port: {{ item.metrics_port }} {% if item.type == 'frontend_proxy' %} From 3156d966193b9ff49866925189b2aa2cf6530421 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Wed, 2 Dec 2020 00:29:20 +0100 Subject: [PATCH 40/70] synapse workers-doc-to-yaml.awk: escape slash for non-gnu awk versions --- roles/matrix-synapse/files/workers-doc-to-yaml.awk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse/files/workers-doc-to-yaml.awk b/roles/matrix-synapse/files/workers-doc-to-yaml.awk index fe018b6f..0f304417 100755 --- a/roles/matrix-synapse/files/workers-doc-to-yaml.awk +++ b/roles/matrix-synapse/files/workers-doc-to-yaml.awk @@ -13,7 +13,7 @@ function worker_stanza_append(string) { function line_is_endpoint_url(line) { # probably API endpoint if it starts with white-space and ^ or / - return (line ~ /^ +[\^/].*\//) + return (line ~ /^ +[\^\/].*\//) } # Put YAML marker at beginning of file. From b6b95fe7424721c55493f2d16a2874eb95aac78a Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Wed, 2 Dec 2020 23:22:02 +0100 Subject: [PATCH 41/70] synapse workers-doc-to-yaml script: compatibility++ with non-gnu awk --- roles/matrix-synapse/files/workers-doc-to-yaml.awk | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/matrix-synapse/files/workers-doc-to-yaml.awk b/roles/matrix-synapse/files/workers-doc-to-yaml.awk index 0f304417..d9295e32 100755 --- a/roles/matrix-synapse/files/workers-doc-to-yaml.awk +++ b/roles/matrix-synapse/files/workers-doc-to-yaml.awk @@ -55,7 +55,7 @@ enable_parsing { workers = (workers ? workers "\n" : "") " - " worker_type # loop through the lines (2 - number of fields in record) - for (i = 1; i < NF + 1; i++) { + for (i = 2; i < NF + 1; i++) { # copy line for gsub replacements line = $i @@ -106,7 +106,7 @@ enable_parsing { } # white-space only line? - } else if (line ~ /^\w*$/) { + } else if (line ~ /^ *$/) { if (i > 3 && i < NF) { # print white-space lines unless 1st or last line in section @@ -120,7 +120,7 @@ enable_parsing { worker_stanza_append(" # " line linefeed) # and take note of words hinting at additional conditions to be met - if (line ~ /\<[Ii]f\>|\<[Ff]or\>/) { + if (line ~ /(^| )[Ii]f |(^| )[Ff]or /) { endpoints_seem_conditional = 1 } } From f2c7d79238cd0f79544b9bc37442a9f5e889524c Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Sat, 23 Jan 2021 14:06:25 +0100 Subject: [PATCH 42/70] Drop probably incorrect comment from synapse homeserver.yaml.j2 --- roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index 8c6f5cb4..b6a7a5e2 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -265,7 +265,6 @@ start_pushers: false update_user_directory: false {% endif %} -# rather let systemd handle the forking daemonize: false {% endif %} From c8f051a42dabb12a404e47054a7ba0076e50cb4a Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Sat, 23 Jan 2021 14:34:41 +0100 Subject: [PATCH 43/70] Track workers endpoint list in repo instead of regenerating on user side --- .gitignore | 2 - .../files/workers-doc-to-yaml.sh | 6 + .../tasks/workers/setup_install.yml | 16 - roles/matrix-synapse/vars/workers.yml | 308 ++++++++++++++++++ 4 files changed, 314 insertions(+), 18 deletions(-) create mode 100755 roles/matrix-synapse/files/workers-doc-to-yaml.sh create mode 100644 roles/matrix-synapse/vars/workers.yml diff --git a/.gitignore b/.gitignore index c5279a46..36c65bda 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,5 @@ !/inventory/host_vars/.gitkeep !/inventory/scripts /roles/*/files/scratchpad -/roles/matrix-synapse/files/workers.upstream-documentation.md -/roles/matrix-synapse/vars/workers.yml .DS_Store .python-version diff --git a/roles/matrix-synapse/files/workers-doc-to-yaml.sh b/roles/matrix-synapse/files/workers-doc-to-yaml.sh new file mode 100755 index 00000000..5981523b --- /dev/null +++ b/roles/matrix-synapse/files/workers-doc-to-yaml.sh @@ -0,0 +1,6 @@ +#!/bin/sh +# Fetch the synapse worker documentation and extract endpoint URLs +# matrix-org/synapse master branch points to current stable release + +URL=https://github.com/matrix-org/synapse/raw/master/docs/workers.md +curl -L ${URL} | awk -f workers-doc-to-yaml.awk > ../vars/workers.yml diff --git a/roles/matrix-synapse/tasks/workers/setup_install.yml b/roles/matrix-synapse/tasks/workers/setup_install.yml index 6cd7ae42..947dcd55 100644 --- a/roles/matrix-synapse/tasks/workers/setup_install.yml +++ b/roles/matrix-synapse/tasks/workers/setup_install.yml @@ -1,21 +1,5 @@ --- -- name: Download synapse workers doc - local_action: - module: get_url - url: https://github.com/matrix-org/synapse/raw/master/docs/workers.md - dest: "{{ role_path }}/files/workers.upstream-documentation.md" - vars: - ansible_become: no - -- name: Download synapse workers doc and convert into YAML - local_action: - module: shell - cmd: "awk -f '{{ role_path }}/files/workers-doc-to-yaml.awk' -- '{{ role_path }}/files/workers.upstream-documentation.md' > '{{ role_path }}/vars/workers.yml'" - creates: "{{ role_path }}/vars/workers.yml" - vars: - ansible_become: no - - name: Load list of available worker apps and endpoints include_vars: "{{ role_path }}/vars/workers.yml" diff --git a/roles/matrix-synapse/vars/workers.yml b/roles/matrix-synapse/vars/workers.yml new file mode 100644 index 00000000..9dc79360 --- /dev/null +++ b/roles/matrix-synapse/vars/workers.yml @@ -0,0 +1,308 @@ +--- + +matrix_synapse_workers_generic_worker_endpoints: + # This worker can handle API requests matching the following regular + # expressions: + + # Sync requests + - ^/_matrix/client/(v2_alpha|r0)/sync$ + - ^/_matrix/client/(api/v1|v2_alpha|r0)/events$ + - ^/_matrix/client/(api/v1|r0)/initialSync$ + - ^/_matrix/client/(api/v1|r0)/rooms/[^/]+/initialSync$ + + # Federation requests + - ^/_matrix/federation/v1/event/ + - ^/_matrix/federation/v1/state/ + - ^/_matrix/federation/v1/state_ids/ + - ^/_matrix/federation/v1/backfill/ + - ^/_matrix/federation/v1/get_missing_events/ + - ^/_matrix/federation/v1/publicRooms + - ^/_matrix/federation/v1/query/ + - ^/_matrix/federation/v1/make_join/ + - ^/_matrix/federation/v1/make_leave/ + - ^/_matrix/federation/v1/send_join/ + - ^/_matrix/federation/v2/send_join/ + - ^/_matrix/federation/v1/send_leave/ + - ^/_matrix/federation/v2/send_leave/ + - ^/_matrix/federation/v1/invite/ + - ^/_matrix/federation/v2/invite/ + - ^/_matrix/federation/v1/query_auth/ + - ^/_matrix/federation/v1/event_auth/ + - ^/_matrix/federation/v1/exchange_third_party_invite/ + - ^/_matrix/federation/v1/user/devices/ + - ^/_matrix/federation/v1/get_groups_publicised$ + - ^/_matrix/key/v2/query + + # Inbound federation transaction request + - ^/_matrix/federation/v1/send/ + + # Client API requests + - ^/_matrix/client/(api/v1|r0|unstable)/publicRooms$ + - ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/joined_members$ + - ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/context/.*$ + - ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/members$ + - ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state$ + - ^/_matrix/client/(api/v1|r0|unstable)/account/3pid$ + - ^/_matrix/client/(api/v1|r0|unstable)/keys/query$ + - ^/_matrix/client/(api/v1|r0|unstable)/keys/changes$ + - ^/_matrix/client/versions$ + - ^/_matrix/client/(api/v1|r0|unstable)/voip/turnServer$ + - ^/_matrix/client/(api/v1|r0|unstable)/joined_groups$ + - ^/_matrix/client/(api/v1|r0|unstable)/publicised_groups$ + - ^/_matrix/client/(api/v1|r0|unstable)/publicised_groups/ + - ^/_synapse/client/password_reset/email/submit_token$ + + # Registration/login requests + - ^/_matrix/client/(api/v1|r0|unstable)/login$ + - ^/_matrix/client/(r0|unstable)/register$ + # FIXME: possible bug with SSO and multiple generic workers + # see https://github.com/matrix-org/synapse/issues/7530 + # ^/_matrix/client/(r0|unstable)/auth/.*/fallback/web$ + + # Event sending requests + - ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/redact + - ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send + - ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state/ + - ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$ + - ^/_matrix/client/(api/v1|r0|unstable)/join/ + - ^/_matrix/client/(api/v1|r0|unstable)/profile/ + + + # Additionally, the following REST endpoints can be handled for GET requests: + + # FIXME: ADDITIONAL CONDITIONS REQUIRED: to be enabled manually + # ^/_matrix/federation/v1/groups/ + + # Pagination requests can also be handled, but all requests for a given + # room must be routed to the same instance. Additionally, care must be taken to + # ensure that the purge history admin API is not used while pagination requests + # for the room are in flight: + + # FIXME: ADDITIONAL CONDITIONS REQUIRED: to be enabled manually + # ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/messages$ + + # Additionally, the following endpoints should be included if Synapse is configured + # to use SSO (you only need to include the ones for whichever SSO provider you're + # using): + + # OpenID Connect requests. + # FIXME: ADDITIONAL CONDITIONS REQUIRED: to be enabled manually + # ^/_matrix/client/(api/v1|r0|unstable)/login/sso/redirect$ + # ^/_synapse/oidc/callback$ + + # SAML requests. + # FIXME: ADDITIONAL CONDITIONS REQUIRED: to be enabled manually + # ^/_matrix/client/(api/v1|r0|unstable)/login/sso/redirect$ + # ^/_matrix/saml2/authn_response$ + + # CAS requests. + # FIXME: ADDITIONAL CONDITIONS REQUIRED: to be enabled manually + # ^/_matrix/client/(api/v1|r0|unstable)/login/(cas|sso)/redirect$ + # ^/_matrix/client/(api/v1|r0|unstable)/login/cas/ticket$ + + # Note that a HTTP listener with `client` and `federation` resources must be + # configured in the `worker_listeners` option in the worker config. + + # Ensure that all SSO logins go to a single process (usually the main process). + # For multiple workers not handling the SSO endpoints properly, see + # [#7530](https://github.com/matrix-org/synapse/issues/7530). + + # #### Load balancing + + # It is possible to run multiple instances of this worker app, with incoming requests + # being load-balanced between them by the reverse-proxy. However, different endpoints + # have different characteristics and so admins + # may wish to run multiple groups of workers handling different endpoints so that + # load balancing can be done in different ways. + + # For `/sync` and `/initialSync` requests it will be more efficient if all + # requests from a particular user are routed to a single instance. Extracting a + # user ID from the access token or `Authorization` header is currently left as an + # exercise for the reader. Admins may additionally wish to separate out `/sync` + # requests that have a `since` query parameter from those that don't (and + # `/initialSync`), as requests that don't are known as "initial sync" that happens + # when a user logs in on a new device and can be *very* resource intensive, so + # isolating these requests will stop them from interfering with other users ongoing + # syncs. + + # Federation and client requests can be balanced via simple round robin. + + # The inbound federation transaction request `^/_matrix/federation/v1/send/` + # should be balanced by source IP so that transactions from the same remote server + # go to the same process. + + # Registration/login requests can be handled separately purely to help ensure that + # unexpected load doesn't affect new logins and sign ups. + + # Finally, event sending requests can be balanced by the room ID in the URI (or + # the full URI, or even just round robin), the room ID is the path component after + # `/rooms/`. If there is a large bridge connected that is sending or may send lots + # of events, then a dedicated set of workers can be provisioned to limit the + # effects of bursts of events from that bridge on events sent by normal users. + + # #### Stream writers + + # Additionally, there is *experimental* support for moving writing of specific + # streams (such as events) off of the main process to a particular worker. (This + # is only supported with Redis-based replication.) + + # Currently supported streams are `events` and `typing`. + + # To enable this, the worker must have a HTTP replication listener configured, + # have a `worker_name` and be listed in the `instance_map` config. For example to + # move event persistence off to a dedicated worker, the shared configuration would + # include: + + # ```yaml + # instance_map: + # event_persister1: + # host: localhost + # port: 8034 + + # stream_writers: + # events: event_persister1 + # ``` + + # The `events` stream also experimentally supports having multiple writers, where + # work is sharded between them by room ID. Note that you *must* restart all worker + # instances when adding or removing event persisters. An example `stream_writers` + # configuration with multiple writers: + + # ```yaml + # stream_writers: + # events: + # - event_persister1 + # - event_persister2 + # ``` + + # #### Background tasks + + # There is also *experimental* support for moving background tasks to a separate + # worker. Background tasks are run periodically or started via replication. Exactly + # which tasks are configured to run depends on your Synapse configuration (e.g. if + # stats is enabled). + + # To enable this, the worker must have a `worker_name` and can be configured to run + # background tasks. For example, to move background tasks to a dedicated worker, + # the shared configuration would include: + + # ```yaml + # run_background_tasks_on: background_worker + # ``` + + # You might also wish to investigate the `update_user_directory` and + # `media_instance_running_background_jobs` settings. + +# pusher worker (no API endpoints) [ + # Handles sending push notifications to sygnal and email. Doesn't handle any + # REST endpoints itself, but you should set `start_pushers: False` in the + # shared configuration file to stop the main synapse sending push notifications. + + # Note this worker cannot be load-balanced: only one instance should be active. +# ] + +# appservice worker (no API endpoints) [ + # Handles sending output traffic to Application Services. Doesn't handle any + # REST endpoints itself, but you should set `notify_appservices: False` in the + # shared configuration file to stop the main synapse sending appservice notifications. + + # Note this worker cannot be load-balanced: only one instance should be active. + +# ] + +# federation_sender worker (no API endpoints) [ + # Handles sending federation traffic to other servers. Doesn't handle any + # REST endpoints itself, but you should set `send_federation: False` in the + # shared configuration file to stop the main synapse sending this traffic. + + # If running multiple federation senders then you must list each + # instance in the `federation_sender_instances` option by their `worker_name`. + # All instances must be stopped and started when adding or removing instances. + # For example: + + # ```yaml + # federation_sender_instances: + # - federation_sender1 + # - federation_sender2 + # ``` +# ] + +matrix_synapse_workers_media_repository_endpoints: + # Handles the media repository. It can handle all endpoints starting with: + + - ^/_matrix/media/ + + # ... and the following regular expressions matching media-specific administration APIs: + + - ^/_synapse/admin/v1/purge_media_cache$ + - ^/_synapse/admin/v1/room/.*/media.*$ + - ^/_synapse/admin/v1/user/.*/media.*$ + - ^/_synapse/admin/v1/media/.*$ + - ^/_synapse/admin/v1/quarantine_media/.*$ + + # You should also set `enable_media_repo: False` in the shared configuration + # file to stop the main synapse running background jobs related to managing the + # media repository. + + # In the `media_repository` worker configuration file, configure the http listener to + # expose the `media` resource. For example: + + # ```yaml + # worker_listeners: + # - type: http + # port: 8085 + # resources: + # - names: + # - media + # ``` + + # Note that if running multiple media repositories they must be on the same server + # and you must configure a single instance to run the background tasks, e.g.: + + # ```yaml + # media_instance_running_background_jobs: "media-repository-1" + # ``` + + # Note that if a reverse proxy is used , then `/_matrix/media/` must be routed for both inbound client and federation requests (if they are handled separately). + +matrix_synapse_workers_user_dir_endpoints: + # Handles searches in the user directory. It can handle REST endpoints matching + # the following regular expressions: + + - ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$ + + # When using this worker you must also set `update_user_directory: False` in the + # shared configuration file to stop the main synapse running background + # jobs related to updating the user directory. + +matrix_synapse_workers_frontend_proxy_endpoints: + # Proxies some frequently-requested client endpoints to add caching and remove + # load from the main synapse. It can handle REST endpoints matching the following + # regular expressions: + + - ^/_matrix/client/(api/v1|r0|unstable)/keys/upload + + # If `use_presence` is False in the homeserver config, it can also handle REST + # endpoints matching the following regular expressions: + + # FIXME: ADDITIONAL CONDITIONS REQUIRED: to be enabled manually + # ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status + + # This "stub" presence handler will pass through `GET` request but make the + # `PUT` effectively a no-op. + + # It will proxy any requests it cannot handle to the main synapse instance. It + # must therefore be configured with the location of the main instance, via + # the `worker_main_http_uri` setting in the `frontend_proxy` worker configuration + # file. For example: + + # worker_main_http_uri: http://127.0.0.1:8008 + +matrix_synapse_workers_avail_list: + - appservice + - federation_sender + - frontend_proxy + - generic_worker + - media_repository + - pusher + - user_dir From edc21f15e575e9d40c96e67757e8f8a18722bc0e Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Sun, 24 Jan 2021 08:53:09 +0100 Subject: [PATCH 44/70] Restrict publishing worker (metrics) ports to localhost --- .../templates/synapse/systemd/matrix-synapse.service.j2 | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 index a88bb366..3bf51b6f 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 @@ -47,14 +47,15 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \ {% endif %} {% for worker in matrix_synapse_workers_enabled_list %} {% if matrix_synapse_workers_enabled and not matrix_nginx_proxy_enabled|default(False) %} - {# Expose worker ports (by default 18xxx range) on host if not using internal nginx proxy #} + {# Expose worker ports (by default in 18xxx range) on localhost, f.e. when using + an external reverse proxy outside the matrix docker network #} {% if worker.port != 0 %} - -p {{ worker.port }}:{{ worker.port }} \ + -p 127.0.0.1:{{ worker.port }}:{{ worker.port }} \ {% endif %} {% endif %} - {# Expose worker metrics ports on host if defined #} + {# Expose worker metrics ports on localhost #} {% if worker.metrics_port != 0 %} - -p {{ worker.metrics_port }}:{{ worker.metrics_port }} \ + -p 127.0.0.1:{{ worker.metrics_port }}:{{ worker.metrics_port }} \ {% endif %} {% endfor %} --mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/data,ro \ From 92ee3d78a05dd52eacffae287c3ab3145ef780ae Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 24 Jan 2021 19:42:32 +0200 Subject: [PATCH 45/70] Fix matrix-remove-all for when Synapse workers are enabled --- .../usr-local-bin/matrix-remove-all.j2 | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/roles/matrix-base/templates/usr-local-bin/matrix-remove-all.j2 b/roles/matrix-base/templates/usr-local-bin/matrix-remove-all.j2 index 2a647aba..699d26ad 100644 --- a/roles/matrix-base/templates/usr-local-bin/matrix-remove-all.j2 +++ b/roles/matrix-base/templates/usr-local-bin/matrix-remove-all.j2 @@ -15,11 +15,26 @@ if [ "$sure" != "Yes, I really want to remove everything!" ]; then exit 0 else echo "Stop and remove matrix services" - for s in $(find {{ matrix_systemd_path }}/ -name "matrix-*" -printf "%f\n"); do + + # Look for and stop services, avoiding things like + # 'matrix-synapse-worker@.service' (just a template for instantiated services; can't stop it directly). + # We use '-xtype f' and not '-type f', because we wish to match symlinks like this: + # '/etc/systemd/system/matrix-synapse.service.wants/matrix-synapse-worker@generic_worker:18111.service' + # and stop these instantiated services as well. + for s in $(find {{ matrix_systemd_path }}/ -xtype f -name "matrix-*" -printf "%f\n" | grep -v '@.service' | uniq); do systemctl stop $s + done + + # Get rid of regular service files, as well as symlinks like + # '/etc/systemd/system/matrix-synapse.service.wants/matrix-synapse-worker@generic_worker:18111.service' + # and even + # '/etc/systemd/system/multi-user.target.wants/matrix-synapse.service'. + for s in $(find {{ matrix_systemd_path }}/ -xtype f -name "matrix-*" -printf "%p\n"); do rm -f {{ matrix_systemd_path }}/$s done + systemctl daemon-reload + echo "Remove matrix scripts" find {{ matrix_local_bin_path }}/ -name "matrix-*" -delete echo "Remove unused Docker images and resources" From cc5cf0d7257ed093abad322fe842d235eaf8b4c5 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 24 Jan 2021 20:17:10 +0200 Subject: [PATCH 46/70] Load roles/matrix-synapse/vars/workers.yml earlier to not break --tags=setup-nginx-proxy If we load it at runtime, during matrix-synapse role execution, it's good enough for matrix-synapse and all roles after that, but.. it breaks when someone uses `--tags=setup-nginx-proxy` alone. The downside of including this vars file like this in `setup.yml` is that the variables contained in it cannot be overriden by the user (in their inventory's `vars.yml`). ... but it's not like overriding these variables was possible anyway when including them at runtime. --- roles/matrix-synapse/tasks/workers/setup_install.yml | 3 --- setup.yml | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/matrix-synapse/tasks/workers/setup_install.yml b/roles/matrix-synapse/tasks/workers/setup_install.yml index 947dcd55..33ddb0b9 100644 --- a/roles/matrix-synapse/tasks/workers/setup_install.yml +++ b/roles/matrix-synapse/tasks/workers/setup_install.yml @@ -1,8 +1,5 @@ --- -- name: Load list of available worker apps and endpoints - include_vars: "{{ role_path }}/vars/workers.yml" - - name: Ensure synapse worker base service file installed template: src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker@.service.j2" diff --git a/setup.yml b/setup.yml index 1f883243..d277b912 100755 --- a/setup.yml +++ b/setup.yml @@ -3,6 +3,9 @@ hosts: "{{ target if target is defined else 'matrix_servers' }}" become: true + vars_files: + - roles/matrix-synapse/vars/workers.yml + roles: - matrix-base - matrix-dynamic-dns From 5ca68210cdda7c7302da52fa1ae7c457cbbb857b Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 24 Jan 2021 22:18:31 +0200 Subject: [PATCH 47/70] Do not handle /_matrix/federation on client-server port, nor /_matrix/client stuff on federation port I guess it didn't hurt to do it until now, but it's not great serving federation APIs on the client-server API port, etc. matrix-corporal doesn't work yet (still something to be solved in the future), but its firewalling operations will also be sabotaged by Client-Server APIs being served on the federation port (it's a way to get around its firewalling). --- group_vars/matrix_servers | 5 ++-- roles/matrix-nginx-proxy/defaults/main.yml | 3 ++- .../nginx/conf.d/matrix-synapse.conf.j2 | 5 ++-- roles/matrix-synapse/vars/main.yml | 25 +++++++++++++++++++ 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index aa0c1fbe..56c9ebf6 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -975,8 +975,9 @@ matrix_nginx_proxy_synapse_presence_disabled: "{{ not matrix_synapse_use_presenc matrix_nginx_proxy_synapse_workers_enabled: "{{ matrix_synapse_workers_enabled }}" matrix_nginx_proxy_synapse_workers_list: "{{ matrix_synapse_workers_enabled_list }}" -matrix_nginx_proxy_synapse_generic_worker_locations: "{{ matrix_synapse_workers_generic_worker_endpoints|default([]) }}" -matrix_nginx_proxy_synapse_media_repository_locations: "{{ matrix_synapse_workers_media_repository_endpoints|default([]) }}" +matrix_nginx_proxy_synapse_generic_worker_client_server_locations: "{{ matrix_synapse_workers_generic_worker_client_server_endpoints }}" +matrix_nginx_proxy_synapse_generic_worker_federation_locations: "{{ matrix_synapse_workers_generic_worker_federation_endpoints }}" +matrix_nginx_proxy_synapse_media_repository_locations: "{{matrix_synapse_workers_media_repository_endpoints|default([]) }}" matrix_nginx_proxy_synapse_user_dir_locations: "{{ matrix_synapse_workers_user_dir_endpoints|default([]) }}" matrix_nginx_proxy_synapse_frontend_proxy_locations: "{{ matrix_synapse_workers_frontend_proxy_endpoints|default([]) }}" diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 148116e2..44ed8acf 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -324,7 +324,8 @@ matrix_nginx_proxy_proxy_matrix_nginx_status_allowed_addresses: ['{{ ansible_def # synapse worker activation and endpoint mappings matrix_nginx_proxy_synapse_workers_enabled: false matrix_nginx_proxy_synapse_workers_list: [] -matrix_nginx_proxy_synapse_generic_worker_locations: [] +matrix_nginx_proxy_synapse_generic_worker_client_server_locations: [] +matrix_nginx_proxy_synapse_generic_worker_federation_locations: [] matrix_nginx_proxy_synapse_media_repository_locations: [] matrix_nginx_proxy_synapse_user_dir_locations: [] matrix_nginx_proxy_synapse_frontend_proxy_locations: [] diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 632241e7..5d204343 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -109,14 +109,13 @@ {% if generic_workers %} # https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappgeneric_worker - {% for location in matrix_nginx_proxy_synapse_generic_worker_locations %} + {% for location in matrix_nginx_proxy_synapse_generic_worker_client_server_locations %} location ~ {{ location }} { proxy_pass http://generic_worker_upstream$request_uri; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; } {% endfor %} - # FIXME: add GET ^/_matrix/federation/v1/groups/ {% endif %} {% if media_repository_workers %} @@ -361,7 +360,7 @@ server { {% if matrix_nginx_proxy_synapse_workers_enabled %} {% if generic_workers %} # https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappgeneric_worker - {% for location in matrix_nginx_proxy_synapse_generic_worker_locations %} + {% for location in matrix_nginx_proxy_synapse_generic_worker_federation_locations %} location ~ {{ location }} { proxy_pass http://generic_worker_upstream$request_uri; proxy_set_header Host $host; diff --git a/roles/matrix-synapse/vars/main.yml b/roles/matrix-synapse/vars/main.yml index 7c07145b..83325975 100644 --- a/roles/matrix-synapse/vars/main.yml +++ b/roles/matrix-synapse/vars/main.yml @@ -8,3 +8,28 @@ matrix_synapse_role_executed: false matrix_synapse_media_store_parent_path: "{{ matrix_synapse_media_store_path|dirname }}" matrix_synapse_media_store_directory_name: "{{ matrix_synapse_media_store_path|basename }}" + +# A Synapse generic worker can handle both federation and client-server API endpoints. +# We wish to split these, as we normally serve federation separately and don't want them mixed up. +# +# This is some ugly Ansible/Jinja2 hack (seen here: https://stackoverflow.com/a/47831492), +# which takes a list of various strings and removes the ones NOT containing `/_matrix/client` anywhere in them. +# +# We intentionally don't do a diff between everything possible (`matrix_synapse_workers_generic_worker_endpoints`) and `matrix_synapse_workers_generic_worker_federation_endpoints`, +# because `matrix_synapse_workers_generic_worker_endpoints` also contains things like `/_synapse/client/`, etc. +# While /_synapse/client/ endpoints are somewhat client-server API-related, they're: +# - neither part of the client-server API spec (and are thus, different) +# - nor always OK to forward to a worker (we're supposed to obey `matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_client_api_enabled`) +# +# It's also not too many of these APIs (only `^/_synapse/client/password_reset/email/submit_token$` at the time of this writing / 2021-01-24), +# so it's not that important whether we forward them or not. +# +# Basically, we aim to cover most things. Skipping `/_synapse/client` or a few other minor things doesn't matter too much. +matrix_synapse_workers_generic_worker_client_server_endpoints: "{{ matrix_synapse_workers_generic_worker_endpoints|default([]) | map('regex_search', '.*/_matrix/client.*')| list | difference([none]) }}" + +# A Synapse generic worker can handle both federation and client-server API endpoints. +# We wish to split these, as we normally serve federation separately and don't want them mixed up. +# +# This is some ugly Ansible/Jinja2 hack (seen here: https://stackoverflow.com/a/47831492), +# which takes a list of various strings and removes the ones NOT containing `/_matrix/federation` anywhere in them. +matrix_synapse_workers_generic_worker_federation_endpoints: "{{ matrix_synapse_workers_generic_worker_endpoints|default([]) | map('regex_search', '.*/_matrix/federation.*')| list | difference([none]) }}" From f66a6b066b18b4ebd8288d40349d42164b56db84 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 25 Jan 2021 01:34:58 +0200 Subject: [PATCH 48/70] Be more specific with the Redis version being used --- roles/matrix-redis/defaults/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/matrix-redis/defaults/main.yml b/roles/matrix-redis/defaults/main.yml index f48ea542..74728d87 100644 --- a/roles/matrix-redis/defaults/main.yml +++ b/roles/matrix-redis/defaults/main.yml @@ -5,8 +5,7 @@ matrix_redis_connection_password: "" matrix_redis_base_path: "{{ matrix_base_data_path }}/redis" matrix_redis_data_path: "{{ matrix_redis_base_path }}/data" -matrix_redis_docker_image_v5: "redis:5.0-alpine" -matrix_redis_docker_image_v6: "redis:6.0-alpine" +matrix_redis_docker_image_v6: "docker.io/redis:6.0.10-alpine" matrix_redis_docker_image_latest: "{{ matrix_redis_docker_image_v6 }}" matrix_redis_docker_image_to_use: '{{ matrix_redis_docker_image_latest }}' From 63301b0ef1e3c1b85c7e0822abb03ea4be079aef Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 25 Jan 2021 08:25:43 +0200 Subject: [PATCH 49/70] Improvements around Synapse worker/metrics ports exposure There was a `matrix_nginx_proxy_enabled|default(False)` check, but: - it didn't seem to work reliably for some reason (hmm) - referring to a `matrix_nginx_proxy_*` variable from within the `matrix-synapse` role is not ideal - exposing always happened on `127.0.0.1`, which may not be good enough for some rarer setups (where the own webserver is external to the host) --- group_vars/matrix_servers | 3 +++ roles/matrix-synapse/defaults/main.yml | 8 ++++++++ .../synapse/systemd/matrix-synapse.service.j2 | 11 ++++------- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 56c9ebf6..f64b02a0 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1276,6 +1276,9 @@ matrix_synapse_container_metrics_api_host_bind_port: "{{ '127.0.0.1:9100' if (ma # # For exposing the Synapse Manhole port (plain HTTP) to the local host. matrix_synapse_container_manhole_api_host_bind_port: "{{ '127.0.0.1:9000' if matrix_synapse_manhole_enabled else '' }}" +# +# For exposing the Synapse worker (and metrics) ports to the local host. +matrix_synapse_workers_container_host_bind_address: "{{ '127.0.0.1' if (matrix_synapse_workers_enabled and not matrix_nginx_proxy_enabled) else '' }}" matrix_synapse_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'synapse.db') | to_uuid }}" diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index d159f8a3..7c00c3e5 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -301,6 +301,14 @@ matrix_synapse_manhole_enabled: false # Enable support for Synapse workers matrix_synapse_workers_enabled: false + +# Controls whether the matrix-synapse container exposes the various worker ports +# (see `port` and `metrics_port` in `matrix_synapse_workers_enabled_list`) outside of the container. +# +# Takes an "" value (e.g. "127.0.0.1", "0.0.0.0", etc), or empty string to not expose. +# It takes "*" to signify "bind on all interfaces" ("0.0.0.0" is IPv4-only). +matrix_synapse_workers_container_host_bind_address: '' + # Default list of workers to spawn (order in accord to docs) # - no endpoints / doesn't need port mapping if port ends on 0 # - single-instance-only if 2nd last digit of port number is 0 diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 index 3bf51b6f..3b9ccdf8 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 @@ -45,19 +45,16 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \ {% if matrix_synapse_manhole_enabled and matrix_synapse_container_manhole_api_host_bind_port %} -p {{ matrix_synapse_container_manhole_api_host_bind_port }}:9000 \ {% endif %} + {% if matrix_synapse_workers_enabled and matrix_synapse_workers_container_host_bind_address %} {% for worker in matrix_synapse_workers_enabled_list %} - {% if matrix_synapse_workers_enabled and not matrix_nginx_proxy_enabled|default(False) %} - {# Expose worker ports (by default in 18xxx range) on localhost, f.e. when using - an external reverse proxy outside the matrix docker network #} {% if worker.port != 0 %} - -p 127.0.0.1:{{ worker.port }}:{{ worker.port }} \ + -p {{ '' if matrix_synapse_workers_container_host_bind_address == '*' else (matrix_synapse_workers_container_host_bind_address + ':') }}{{ worker.port }}:{{ worker.port }} \ {% endif %} - {% endif %} - {# Expose worker metrics ports on localhost #} {% if worker.metrics_port != 0 %} - -p 127.0.0.1:{{ worker.metrics_port }}:{{ worker.metrics_port }} \ + -p {{ '' if matrix_synapse_workers_container_host_bind_address == '*' else (matrix_synapse_workers_container_host_bind_address + ':') }}{{ worker.metrics_port }}:{{ worker.metrics_port }} \ {% endif %} {% endfor %} + {% endif %} --mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/data,ro \ --mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \ {% for volume in matrix_synapse_container_additional_volumes %} From c05d3d09bd223a2337e1323d2648ce38b390e1d0 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 25 Jan 2021 08:58:23 +0200 Subject: [PATCH 50/70] Disable systemd services while stopping them This removes some `multi-target.wants` symlinks as well, etc. But despite systemd saying: > Removed symlink /etc/systemd/system/matrix-synapse.service.wants/matrix-synapse-worker@appservice:0.service .. I still see such symlinks tehre for me for some reason, so keeping the code (below) to find & delete them still seems like a good idea. --- roles/matrix-base/templates/usr-local-bin/matrix-remove-all.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-base/templates/usr-local-bin/matrix-remove-all.j2 b/roles/matrix-base/templates/usr-local-bin/matrix-remove-all.j2 index 699d26ad..01e0ac70 100644 --- a/roles/matrix-base/templates/usr-local-bin/matrix-remove-all.j2 +++ b/roles/matrix-base/templates/usr-local-bin/matrix-remove-all.j2 @@ -22,7 +22,7 @@ else # '/etc/systemd/system/matrix-synapse.service.wants/matrix-synapse-worker@generic_worker:18111.service' # and stop these instantiated services as well. for s in $(find {{ matrix_systemd_path }}/ -xtype f -name "matrix-*" -printf "%f\n" | grep -v '@.service' | uniq); do - systemctl stop $s + systemctl disable --now $s done # Get rid of regular service files, as well as symlinks like From 4d62a75f6f46becb0ece21bd4c063830db229ece Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 25 Jan 2021 09:21:17 +0200 Subject: [PATCH 51/70] Get matrix-corporal to play nicely with a Synapse worker setup We do this by creating one more layer of indirection. First we reach some generic vhost handling matrix.DOMAIN. A bunch of override rules are added there (capturing traffic to send to ma1sd, etc). nginx-status and similar generic things also live there. We then proxy to the homeserver on some other vhost (only Synapse being available right now, but repointing this to Dendrite or other will be possible in the future). Then that homeserver-specific vhost does its thing to proxy to the homeserver. It may or may not use workers, etc. Without matrix-corporal, the flow is now: 1. matrix.DOMAIN (matrix-nginx-proxy/matrix-domain.conf) 2. matrix-nginx-proxy/matrix-synapse.conf 3. matrix-synapse With matrix-corporal enabled, it becomes: 1. matrix.DOMAIN (matrix-nginx-proxy/matrix-domain.conf) 2. matrix-corporal 3. matrix-nginx-proxy/matrix-synapse.conf 4. matrix-synapse (matrix-corporal gets injected at step 2). --- group_vars/matrix_servers | 17 +- roles/matrix-nginx-proxy/defaults/main.yml | 38 ++- .../tasks/setup_nginx_proxy.yml | 24 +- .../nginx/conf.d/matrix-base-domain.conf.j2 | 70 +++++ .../nginx/conf.d/matrix-domain.conf.j2 | 196 +++++++++++- .../nginx/conf.d/matrix-synapse.conf.j2 | 278 +++--------------- 6 files changed, 360 insertions(+), 263 deletions(-) create mode 100644 roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-base-domain.conf.j2 diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index f64b02a0..88afb186 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -673,7 +673,8 @@ matrix_corporal_systemd_required_services_list: | (['matrix-synapse.service']) }} -matrix_corporal_matrix_homeserver_api_endpoint: "http://matrix-synapse:8008" +# This goes to Synapse's vhost +matrix_corporal_matrix_homeserver_api_endpoint: "http://matrix-nginx-proxy:12080" matrix_corporal_matrix_auth_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret }}" @@ -885,7 +886,7 @@ matrix_ma1sd_synapsesql_connection: //{{ matrix_synapse_database_host }}/{{ matr matrix_ma1sd_dns_overwrite_enabled: true matrix_ma1sd_dns_overwrite_homeserver_client_name: "{{ matrix_server_fqn_matrix }}" -matrix_ma1sd_dns_overwrite_homeserver_client_value: "http://{{ 'matrix-corporal:41080' if matrix_corporal_enabled else 'matrix-synapse:8008' }}" +matrix_ma1sd_dns_overwrite_homeserver_client_value: "http://{{ matrix_nginx_proxy_proxy_matrix_client_api_addr_with_container }}" # By default, we send mail through the `matrix-mailer` service. matrix_ma1sd_threepid_medium_email_identity_from: "{{ matrix_mailer_sender_address }}" @@ -932,8 +933,8 @@ matrix_ma1sd_database_password: "{{ matrix_synapse_macaroon_secret_key | passwor # If that's not the case, you may wish to disable this and take care of proxying yourself. matrix_nginx_proxy_enabled: true -matrix_nginx_proxy_proxy_matrix_client_api_addr_with_container: "{{ 'matrix-corporal:41080' if matrix_corporal_enabled else 'matrix-synapse:8008' }}" -matrix_nginx_proxy_proxy_matrix_client_api_addr_sans_container: "{{ '127.0.0.1:41080' if matrix_corporal_enabled else '127.0.0.1:8008' }}" +matrix_nginx_proxy_proxy_matrix_client_api_addr_with_container: "{{ 'matrix-corporal:41080' if matrix_corporal_enabled else 'matrix-nginx-proxy:12080' }}" +matrix_nginx_proxy_proxy_matrix_client_api_addr_sans_container: "{{ '127.0.0.1:41080' if matrix_corporal_enabled else '127.0.0.1:12080' }}" matrix_nginx_proxy_proxy_matrix_client_api_client_max_body_size_mb: "{{ matrix_synapse_max_upload_size_mb }}" matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_admin_api_enabled: "{{ matrix_synapse_admin_enabled }}" @@ -956,8 +957,12 @@ matrix_nginx_proxy_proxy_matrix_identity_api_addr_sans_container: "127.0.0.1:809 # By default, we do TLS termination for the Matrix Federation API (port 8448) at matrix-nginx-proxy. # Unless this is handled there OR Synapse's federation listener port is disabled, we'll reverse-proxy. matrix_nginx_proxy_proxy_matrix_federation_api_enabled: "{{ matrix_synapse_federation_port_enabled and not matrix_synapse_tls_federation_listener_enabled }}" -matrix_nginx_proxy_proxy_matrix_federation_api_addr_with_container: "matrix-synapse:8048" -matrix_nginx_proxy_proxy_matrix_federation_api_addr_sans_container: "127.0.0.1:8048" +matrix_nginx_proxy_proxy_matrix_federation_api_addr_with_container: "matrix-nginx-proxy:12088" +matrix_nginx_proxy_proxy_matrix_federation_api_addr_sans_container: "127.0.0.1:12088" + +# Settings controlling matrix-synapse-proxy.conf +matrix_nginx_proxy_proxy_synapse_enabled: "{{ matrix_synapse_enabled }}" +matrix_nginx_proxy_proxy_synapse_federation_api_enabled: "{{ matrix_nginx_proxy_proxy_matrix_federation_api_enabled }}" matrix_nginx_proxy_container_federation_host_bind_port: "{{ matrix_federation_public_port }}" diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 44ed8acf..6ab7e624 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -99,6 +99,10 @@ matrix_nginx_proxy_access_log_enabled: true matrix_nginx_proxy_proxy_riot_compat_redirect_enabled: false matrix_nginx_proxy_proxy_riot_compat_redirect_hostname: "riot.{{ matrix_domain }}" +# Controls whether proxying the Synapse domain should be done. +matrix_nginx_proxy_proxy_synapse_enabled: false +matrix_nginx_proxy_proxy_synapse_hostname: "matrix-nginx-proxy" + # Controls whether proxying the Element domain should be done. matrix_nginx_proxy_proxy_element_enabled: false matrix_nginx_proxy_proxy_element_hostname: "{{ matrix_server_fqn_element }}" @@ -146,8 +150,13 @@ matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_key: "" # The addresses where the Matrix Client API is. # Certain extensions (like matrix-corporal) may override this in order to capture all traffic. -matrix_nginx_proxy_proxy_matrix_client_api_addr_with_container: "matrix-synapse:8008" -matrix_nginx_proxy_proxy_matrix_client_api_addr_sans_container: "127.0.0.1:8008" +matrix_nginx_proxy_proxy_matrix_client_api_addr_with_container: "matrix-nginx-proxy:12080" +matrix_nginx_proxy_proxy_matrix_client_api_addr_sans_container: "127.0.0.1:12080" + +# The addresses where the Matrix Client API is, when using Synapse. +matrix_nginx_proxy_proxy_synapse_client_api_addr_with_container: "matrix-synapse:8008" +matrix_nginx_proxy_proxy_synapse_client_api_addr_sans_container: "127.0.0.1:8008" + # This needs to be equal or higher than the maximum upload size accepted by Synapse. matrix_nginx_proxy_proxy_matrix_client_api_client_max_body_size_mb: 50 @@ -185,34 +194,41 @@ matrix_nginx_proxy_proxy_matrix_client_redirect_root_uri_to_domain: "" # Controls whether proxying for the Matrix Federation API should be done. matrix_nginx_proxy_proxy_matrix_federation_api_enabled: false -matrix_nginx_proxy_proxy_matrix_federation_api_addr_with_container: "matrix-synapse:8048" -matrix_nginx_proxy_proxy_matrix_federation_api_addr_sans_container: "localhost:8048" +matrix_nginx_proxy_proxy_matrix_federation_api_addr_with_container: "matrix-nginx-proxy:12088" +matrix_nginx_proxy_proxy_matrix_federation_api_addr_sans_container: "localhost:12088" matrix_nginx_proxy_proxy_matrix_federation_api_client_max_body_size_mb: "{{ (matrix_nginx_proxy_proxy_matrix_client_api_client_max_body_size_mb | int) * 3 }}" matrix_nginx_proxy_proxy_matrix_federation_api_ssl_certificate: "{{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_proxy_matrix_hostname }}/fullchain.pem" matrix_nginx_proxy_proxy_matrix_federation_api_ssl_certificate_key: "{{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_proxy_matrix_hostname }}/privkey.pem" +# The addresses where the Federation API is, when using Synapse. +matrix_nginx_proxy_proxy_synapse_federation_api_addr_with_container: "matrix-synapse:8048" +matrix_nginx_proxy_proxy_synapse_federation_api_addr_sans_container: "localhost:8048" + # The tmpfs at /tmp needs to be large enough to handle multiple concurrent file uploads. matrix_nginx_proxy_tmp_directory_size_mb: "{{ (matrix_nginx_proxy_proxy_matrix_federation_api_client_max_body_size_mb | int) * 50 }}" -# A list of strings containing additional configuration blocks to add to the nginx http's server configuration. +# A list of strings containing additional configuration blocks to add to the nginx http's server configuration (nginx-http.conf). matrix_nginx_proxy_proxy_http_additional_server_configuration_blocks: [] -# A list of strings containing additional configuration blocks to add to the matrix synapse's server configuration. +# A list of strings containing additional configuration blocks to add to the base matrix server configuration (matrix-domain.conf). matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks: [] -# A list of strings containing additional configuration blocks to add to Riot's server configuration. +# A list of strings containing additional configuration blocks to add to the synapse's server configuration (matrix-synapse.conf). +matrix_nginx_proxy_proxy_synapse_additional_server_configuration_blocks: [] + +# A list of strings containing additional configuration blocks to add to Riot's server configuration (matrix-riot-web.conf). matrix_nginx_proxy_proxy_riot_additional_server_configuration_blocks: [] -# A list of strings containing additional configuration blocks to add to Element's server configuration. +# A list of strings containing additional configuration blocks to add to Element's server configuration (matrix-client-element.conf). matrix_nginx_proxy_proxy_element_additional_server_configuration_blocks: [] -# A list of strings containing additional configuration blocks to add to Dimension's server configuration. +# A list of strings containing additional configuration blocks to add to Dimension's server configuration (matrix-dimension.conf). matrix_nginx_proxy_proxy_dimension_additional_server_configuration_blocks: [] -# A list of strings containing additional configuration blocks to add to Jitsi's server configuration. +# A list of strings containing additional configuration blocks to add to Jitsi's server configuration (matrix-jitsi.conf). matrix_nginx_proxy_proxy_jitsi_additional_server_configuration_blocks: [] -# A list of strings containing additional configuration blocks to add to the base domain server configuration. +# A list of strings containing additional configuration blocks to add to the base domain server configuration (matrix-base-domain.conf). matrix_nginx_proxy_proxy_domain_additional_server_configuration_blocks: [] # Specifies the SSL configuration that should be used for the SSL protocols and ciphers diff --git a/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml b/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml index 90f0da73..9a9bef2d 100644 --- a/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml +++ b/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml @@ -45,12 +45,18 @@ mode: 0644 when: matrix_nginx_proxy_enabled|bool -- name: Ensure Matrix nginx-proxy configuration for matrix domain exists +- name: Ensure Matrix nginx-proxy configuration for matrix-synapse exists template: src: "{{ role_path }}/templates/nginx/conf.d/matrix-synapse.conf.j2" dest: "{{ matrix_nginx_proxy_confd_path }}/matrix-synapse.conf" mode: 0644 - when: matrix_nginx_proxy_proxy_matrix_enabled|bool + when: matrix_nginx_proxy_proxy_synapse_enabled|bool + +- name: Ensure Matrix nginx-proxy configuration for matrix-synapse deleted + file: + path: "{{ matrix_nginx_proxy_confd_path }}/matrix-synapse.conf" + state: absent + when: "not matrix_nginx_proxy_proxy_synapse_enabled|bool" - name: Ensure Matrix nginx-proxy configuration for Element domain exists template: @@ -80,6 +86,12 @@ mode: 0644 when: matrix_nginx_proxy_proxy_jitsi_enabled|bool +- name: Ensure Matrix nginx-proxy configuration for Matrix domain exists + template: + src: "{{ role_path }}/templates/nginx/conf.d/matrix-domain.conf.j2" + dest: "{{ matrix_nginx_proxy_confd_path }}/matrix-domain.conf" + mode: 0644 + - name: Ensure Matrix nginx-proxy data directory for base domain exists file: path: "{{ matrix_nginx_proxy_data_path }}/matrix-domain" @@ -100,8 +112,8 @@ - name: Ensure Matrix nginx-proxy configuration for base domain exists template: - src: "{{ role_path }}/templates/nginx/conf.d/matrix-domain.conf.j2" - dest: "{{ matrix_nginx_proxy_confd_path }}/matrix-domain.conf" + src: "{{ role_path }}/templates/nginx/conf.d/matrix-base-domain.conf.j2" + dest: "{{ matrix_nginx_proxy_confd_path }}/matrix-base-domain.conf" mode: 0644 when: matrix_nginx_proxy_base_domain_serving_enabled|bool @@ -161,7 +173,7 @@ - name: Ensure Matrix nginx-proxy configuration for matrix domain deleted file: - path: "{{ matrix_nginx_proxy_confd_path }}/matrix-synapse.conf" + path: "{{ matrix_nginx_proxy_confd_path }}/matrix-domain.conf" state: absent when: "not matrix_nginx_proxy_proxy_matrix_enabled|bool" @@ -191,7 +203,7 @@ - name: Ensure Matrix nginx-proxy configuration for base domain deleted file: - path: "{{ matrix_nginx_proxy_confd_path }}/matrix-domain.conf" + path: "{{ matrix_nginx_proxy_confd_path }}/matrix-base-domain.conf" state: absent when: "not matrix_nginx_proxy_base_domain_serving_enabled|bool" diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-base-domain.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-base-domain.conf.j2 new file mode 100644 index 00000000..227747a5 --- /dev/null +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-base-domain.conf.j2 @@ -0,0 +1,70 @@ +#jinja2: lstrip_blocks: "True" + +{% macro render_vhost_directives() %} + root /nginx-data/matrix-domain; + + gzip on; + gzip_types text/plain application/json; + {% for configuration_block in matrix_nginx_proxy_proxy_domain_additional_server_configuration_blocks %} + {{- configuration_block }} + {% endfor %} + + location /.well-known/matrix { + root {{ matrix_static_files_base_path }}; + {# + A somewhat long expires value is used to prevent outages + in case this is unreachable due to network failure. + #} + expires 4h; + default_type application/json; + add_header Access-Control-Allow-Origin *; + } +{% endmacro %} + +server { + listen {{ 8080 if matrix_nginx_proxy_enabled else 80 }}; + + server_name {{ matrix_nginx_proxy_base_domain_hostname }}; + server_tokens off; + + {% if matrix_nginx_proxy_https_enabled %} + location /.well-known/acme-challenge { + {% if matrix_nginx_proxy_enabled %} + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-certbot:8080"; + proxy_pass http://$backend; + {% else %} + {# Generic configuration for use outside of our container setup #} + proxy_pass http://127.0.0.1:{{ matrix_ssl_lets_encrypt_certbot_standalone_http_port }}; + {% endif %} + } + + location / { + return 301 https://$http_host$request_uri; + } + {% else %} + {{ render_vhost_directives() }} + {% endif %} +} + +{% if matrix_nginx_proxy_https_enabled %} +server { + listen {{ 8443 if matrix_nginx_proxy_enabled else 443 }} ssl http2; + listen [::]:{{ 8443 if matrix_nginx_proxy_enabled else 443 }} ssl http2; + + server_name {{ matrix_nginx_proxy_base_domain_hostname }}; + server_tokens off; + + ssl_certificate {{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_base_domain_hostname }}/fullchain.pem; + ssl_certificate_key {{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_base_domain_hostname }}/privkey.pem; + + ssl_protocols {{ matrix_nginx_proxy_ssl_protocols }}; + {% if matrix_nginx_proxy_ssl_ciphers != '' %} + ssl_ciphers {{ matrix_nginx_proxy_ssl_ciphers }}; + {% endif %} + ssl_prefer_server_ciphers {{ matrix_nginx_proxy_ssl_prefer_server_ciphers }}; + + {{ render_vhost_directives() }} +} +{% endif %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-domain.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-domain.conf.j2 index 227747a5..2ab78a1b 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-domain.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-domain.conf.j2 @@ -1,31 +1,148 @@ #jinja2: lstrip_blocks: "True" +{% macro render_nginx_status_location_block(addresses) %} + {# Empty first line to make indentation prettier. #} + + location /nginx_status { + stub_status on; + access_log off; + {% for address in addresses %} + allow {{ address }}; + {% endfor %} + deny all; + } +{% endmacro %} -{% macro render_vhost_directives() %} - root /nginx-data/matrix-domain; +{% macro render_vhost_directives() %} gzip on; gzip_types text/plain application/json; - {% for configuration_block in matrix_nginx_proxy_proxy_domain_additional_server_configuration_blocks %} - {{- configuration_block }} - {% endfor %} location /.well-known/matrix { root {{ matrix_static_files_base_path }}; {# A somewhat long expires value is used to prevent outages - in case this is unreachable due to network failure. + in case this is unreachable due to network failure or + due to the base domain's server completely dying. #} expires 4h; default_type application/json; add_header Access-Control-Allow-Origin *; } + + {% if matrix_nginx_proxy_proxy_matrix_nginx_status_enabled %} + {{ render_nginx_status_location_block(matrix_nginx_proxy_proxy_matrix_nginx_status_allowed_addresses) }} + {% endif %} + + {% if matrix_nginx_proxy_proxy_matrix_corporal_api_enabled %} + location ^~ /_matrix/corporal { + {% if matrix_nginx_proxy_enabled %} + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "{{ matrix_nginx_proxy_proxy_matrix_corporal_api_addr_with_container }}"; + proxy_pass http://$backend; + {% else %} + {# Generic configuration for use outside of our container setup #} + proxy_pass http://{{ matrix_nginx_proxy_proxy_matrix_corporal_api_addr_sans_container }}; + {% endif %} + + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} + + {% if matrix_nginx_proxy_proxy_matrix_identity_api_enabled %} + location ^~ /_matrix/identity { + {% if matrix_nginx_proxy_enabled %} + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "{{ matrix_nginx_proxy_proxy_matrix_identity_api_addr_with_container }}"; + proxy_pass http://$backend; + {% else %} + {# Generic configuration for use outside of our container setup #} + proxy_pass http://{{ matrix_nginx_proxy_proxy_matrix_identity_api_addr_sans_container }}; + {% endif %} + + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} + + {% if matrix_nginx_proxy_proxy_matrix_user_directory_search_enabled %} + location ^~ /_matrix/client/r0/user_directory/search { + {% if matrix_nginx_proxy_enabled %} + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "{{ matrix_nginx_proxy_proxy_matrix_user_directory_search_addr_with_container }}"; + proxy_pass http://$backend; + {% else %} + {# Generic configuration for use outside of our container setup #} + proxy_pass http://{{ matrix_nginx_proxy_proxy_matrix_user_directory_search_addr_sans_container }}; + {% endif %} + + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} + + {% if matrix_nginx_proxy_proxy_matrix_3pid_registration_enabled %} + location ~ ^/_matrix/client/r0/register/(email|msisdn)/requestToken$ { + {% if matrix_nginx_proxy_enabled %} + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "{{ matrix_nginx_proxy_proxy_matrix_3pid_registration_addr_with_container }}"; + proxy_pass http://$backend; + {% else %} + {# Generic configuration for use outside of our container setup #} + proxy_pass http://{{ matrix_nginx_proxy_proxy_matrix_3pid_registration_addr_sans_container }}; + {% endif %} + + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} + + {% for configuration_block in matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks %} + {{- configuration_block }} + {% endfor %} + + {# + This handles the Matrix Client API only. + The Matrix Federation API is handled by a separate vhost. + #} + location ~* ^({{ matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_prefix_regexes|join('|') }}) { + {% if matrix_nginx_proxy_enabled %} + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "{{ matrix_nginx_proxy_proxy_matrix_client_api_addr_with_container }}"; + proxy_pass http://$backend; + {% else %} + {# Generic configuration for use outside of our container setup #} + proxy_pass http://{{ matrix_nginx_proxy_proxy_matrix_client_api_addr_sans_container }}; + {% endif %} + + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + + client_body_buffer_size 25M; + client_max_body_size {{ matrix_nginx_proxy_proxy_matrix_client_api_client_max_body_size_mb }}M; + proxy_max_temp_file_size 0; + } + + location / { + {% if matrix_nginx_proxy_proxy_matrix_client_redirect_root_uri_to_domain %} + return 302 $scheme://{{ matrix_nginx_proxy_proxy_matrix_client_redirect_root_uri_to_domain }}$request_uri; + {% else %} + rewrite ^/$ /_matrix/static/ last; + {% endif %} + } {% endmacro %} server { listen {{ 8080 if matrix_nginx_proxy_enabled else 80 }}; + server_name {{ matrix_nginx_proxy_proxy_matrix_hostname }}; - server_name {{ matrix_nginx_proxy_base_domain_hostname }}; server_tokens off; + root /dev/null; {% if matrix_nginx_proxy_https_enabled %} location /.well-known/acme-challenge { @@ -40,6 +157,10 @@ server { {% endif %} } + {% if matrix_nginx_proxy_proxy_matrix_nginx_status_enabled %} + {{ render_nginx_status_location_block(matrix_nginx_proxy_proxy_matrix_nginx_status_allowed_addresses) }} + {% endif %} + location / { return 301 https://$http_host$request_uri; } @@ -53,11 +174,13 @@ server { listen {{ 8443 if matrix_nginx_proxy_enabled else 443 }} ssl http2; listen [::]:{{ 8443 if matrix_nginx_proxy_enabled else 443 }} ssl http2; - server_name {{ matrix_nginx_proxy_base_domain_hostname }}; + server_name {{ matrix_nginx_proxy_proxy_matrix_hostname }}; + server_tokens off; + root /dev/null; - ssl_certificate {{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_base_domain_hostname }}/fullchain.pem; - ssl_certificate_key {{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_base_domain_hostname }}/privkey.pem; + ssl_certificate {{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_proxy_matrix_hostname }}/fullchain.pem; + ssl_certificate_key {{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_proxy_matrix_hostname }}/privkey.pem; ssl_protocols {{ matrix_nginx_proxy_ssl_protocols }}; {% if matrix_nginx_proxy_ssl_ciphers != '' %} @@ -68,3 +191,56 @@ server { {{ render_vhost_directives() }} } {% endif %} + +{% if matrix_nginx_proxy_proxy_matrix_federation_api_enabled %} +{# + This federation vhost is a little special. + It serves federation over HTTP or HTTPS, depending on `matrix_nginx_proxy_https_enabled`. +#} +server { + {% if matrix_nginx_proxy_https_enabled %} + listen 8448 ssl http2; + listen [::]:8448 ssl http2; + {% else %} + listen 8448; + {% endif %} + + server_name {{ matrix_nginx_proxy_proxy_matrix_hostname }}; + server_tokens off; + + root /dev/null; + + gzip on; + gzip_types text/plain application/json; + + {% if matrix_nginx_proxy_https_enabled %} + ssl_certificate {{ matrix_nginx_proxy_proxy_matrix_federation_api_ssl_certificate }}; + ssl_certificate_key {{ matrix_nginx_proxy_proxy_matrix_federation_api_ssl_certificate_key }}; + + ssl_protocols {{ matrix_nginx_proxy_ssl_protocols }}; + {% if matrix_nginx_proxy_ssl_ciphers != '' %} + ssl_ciphers {{ matrix_nginx_proxy_ssl_ciphers }}; + {% endif %} + ssl_prefer_server_ciphers {{ matrix_nginx_proxy_ssl_prefer_server_ciphers }}; + {% endif %} + + location / { + {% if matrix_nginx_proxy_enabled %} + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "{{ matrix_nginx_proxy_proxy_matrix_federation_api_addr_with_container }}"; + proxy_pass http://$backend; + {% else %} + {# Generic configuration for use outside of our container setup #} + proxy_pass http://{{ matrix_nginx_proxy_proxy_matrix_federation_api_addr_sans_container }}; + {% endif %} + + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + + client_body_buffer_size 25M; + client_max_body_size {{ matrix_nginx_proxy_proxy_matrix_federation_api_client_max_body_size_mb }}M; + proxy_max_temp_file_size 0; + } +} +{% endif %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 5d204343..0dcaf9a6 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -1,108 +1,58 @@ #jinja2: lstrip_blocks: "True" -{% macro render_nginx_status_location_block(addresses) %} - {# Empty first line to make indentation prettier. #} - - location /nginx_status { - stub_status on; - access_log off; - {% for address in addresses %} - allow {{ address }}; - {% endfor %} - deny all; - } -{% endmacro %} +{% set generic_workers = matrix_nginx_proxy_synapse_workers_list|selectattr('type', 'equalto', 'generic_worker')|list %} +{% set media_repository_workers = matrix_nginx_proxy_synapse_workers_list|selectattr('type', 'equalto', 'media_repository')|list %} +{% set user_dir_workers = matrix_nginx_proxy_synapse_workers_list|selectattr('type', 'equalto', 'user_dir')|list %} +{% set frontend_proxy_workers = matrix_nginx_proxy_synapse_workers_list|selectattr('type', 'equalto', 'frontend_proxy')|list %} +{% if matrix_nginx_proxy_synapse_workers_enabled %} + # Round Robin "upstream" pools for workers -{% macro render_vhost_directives() %} - gzip on; - gzip_types text/plain application/json; + {% if generic_workers %} + upstream generic_worker_upstream { + # ensures that requests from the same client will always be passed + # to the same server (except when this server is unavailable) + ip_hash; - location /.well-known/matrix { - root {{ matrix_static_files_base_path }}; - {# - A somewhat long expires value is used to prevent outages - in case this is unreachable due to network failure or - due to the base domain's server completely dying. - #} - expires 4h; - default_type application/json; - add_header Access-Control-Allow-Origin *; + {% for worker in generic_workers %} + server "matrix-synapse:{{ worker.port }}"; + {% endfor %} } - - {% if matrix_nginx_proxy_proxy_matrix_nginx_status_enabled %} - {{ render_nginx_status_location_block(matrix_nginx_proxy_proxy_matrix_nginx_status_allowed_addresses) }} {% endif %} - {% if matrix_nginx_proxy_proxy_matrix_corporal_api_enabled %} - location ^~ /_matrix/corporal { - {% if matrix_nginx_proxy_enabled %} - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "{{ matrix_nginx_proxy_proxy_matrix_corporal_api_addr_with_container }}"; - proxy_pass http://$backend; - {% else %} - {# Generic configuration for use outside of our container setup #} - proxy_pass http://{{ matrix_nginx_proxy_proxy_matrix_corporal_api_addr_sans_container }}; - {% endif %} - - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; + {% if frontend_proxy_workers %} + upstream frontend_proxy_upstream { + {% for worker in frontend_proxy_workers %} + server "matrix-synapse:{{ worker.port }}"; + {% endfor %} } {% endif %} - {% if matrix_nginx_proxy_proxy_matrix_identity_api_enabled %} - location ^~ /_matrix/identity { - {% if matrix_nginx_proxy_enabled %} - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "{{ matrix_nginx_proxy_proxy_matrix_identity_api_addr_with_container }}"; - proxy_pass http://$backend; - {% else %} - {# Generic configuration for use outside of our container setup #} - proxy_pass http://{{ matrix_nginx_proxy_proxy_matrix_identity_api_addr_sans_container }}; - {% endif %} - - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; + {% if media_repository_workers %} + upstream media_repository_upstream { + {% for worker in media_repository_workers %} + server "matrix-synapse:{{ worker.port }}"; + {% endfor %} } {% endif %} - {% if matrix_nginx_proxy_proxy_matrix_user_directory_search_enabled %} - # NOTE: This redirects user lookup requests to the identity server instead of - # synapse, so user_dir_workers endpoints listed further down in this file will - # not be reached and workers of this kind should be disabled for consistency. - location ^~ /_matrix/client/r0/user_directory/search { - {% if matrix_nginx_proxy_enabled %} - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "{{ matrix_nginx_proxy_proxy_matrix_user_directory_search_addr_with_container }}"; - proxy_pass http://$backend; - {% else %} - {# Generic configuration for use outside of our container setup #} - proxy_pass http://{{ matrix_nginx_proxy_proxy_matrix_user_directory_search_addr_sans_container }}; - {% endif %} - - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; + {% if user_dir_workers %} + upstream user_dir_upstream { + {% for worker in user_dir_workers %} + server "matrix-synapse:{{ worker.port }}"; + {% endfor %} } {% endif %} +{% endif %} - {% if matrix_nginx_proxy_proxy_matrix_3pid_registration_enabled %} - location ~ ^/_matrix/client/r0/register/(email|msisdn)/requestToken$ { - {% if matrix_nginx_proxy_enabled %} - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "{{ matrix_nginx_proxy_proxy_matrix_3pid_registration_addr_with_container }}"; - proxy_pass http://$backend; - {% else %} - {# Generic configuration for use outside of our container setup #} - proxy_pass http://{{ matrix_nginx_proxy_proxy_matrix_3pid_registration_addr_sans_container }}; - {% endif %} +server { + listen 12080; + server_name {{ matrix_nginx_proxy_proxy_synapse_hostname }}; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {% endif %} + server_tokens off; + root /dev/null; + + gzip on; + gzip_types text/plain application/json; {% if matrix_nginx_proxy_synapse_workers_enabled %} {# Workers redirects BEGIN #} @@ -167,7 +117,7 @@ {% endif %} - {% for configuration_block in matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks %} + {% for configuration_block in matrix_nginx_proxy_proxy_synapse_additional_server_configuration_blocks %} {{- configuration_block }} {% endfor %} @@ -193,19 +143,16 @@ } {% endif %} - {# - This handles the Matrix Client API only. - The Matrix Federation API is handled by a separate vhost. - #} - location ~* ^({{ matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_prefix_regexes|join('|') }}) { + {# Everything else just goes to the API server ##} + location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; - set $backend "{{ matrix_nginx_proxy_proxy_matrix_client_api_addr_with_container }}"; + set $backend "{{ matrix_nginx_proxy_proxy_synapse_client_api_addr_with_container }}"; proxy_pass http://$backend; {% else %} {# Generic configuration for use outside of our container setup #} - proxy_pass http://{{ matrix_nginx_proxy_proxy_matrix_client_api_addr_sans_container }}; + proxy_pass http://{{ matrix_nginx_proxy_proxy_synapse_client_api_addr_sans_container }}; {% endif %} proxy_set_header Host $host; @@ -215,129 +162,13 @@ client_max_body_size {{ matrix_nginx_proxy_proxy_matrix_client_api_client_max_body_size_mb }}M; proxy_max_temp_file_size 0; } - - location / { - {% if matrix_nginx_proxy_proxy_matrix_client_redirect_root_uri_to_domain %} - return 302 $scheme://{{ matrix_nginx_proxy_proxy_matrix_client_redirect_root_uri_to_domain }}$request_uri; - {% else %} - rewrite ^/$ /_matrix/static/ last; - {% endif %} - } -{% endmacro %} - -{% set generic_workers = matrix_nginx_proxy_synapse_workers_list|selectattr('type', 'equalto', 'generic_worker')|list %} -{% set media_repository_workers = matrix_nginx_proxy_synapse_workers_list|selectattr('type', 'equalto', 'media_repository')|list %} -{% set user_dir_workers = matrix_nginx_proxy_synapse_workers_list|selectattr('type', 'equalto', 'user_dir')|list %} -{% set frontend_proxy_workers = matrix_nginx_proxy_synapse_workers_list|selectattr('type', 'equalto', 'frontend_proxy')|list %} -{% if matrix_nginx_proxy_synapse_workers_enabled %} - # Round Robin "upstream" pools for workers - - {% if generic_workers %} - upstream generic_worker_upstream { - # ensures that requests from the same client will always be passed - # to the same server (except when this server is unavailable) - ip_hash; - - {% for worker in generic_workers %} - server "matrix-synapse:{{ worker.port }}"; - {% endfor %} - } - {% endif %} - - {% if frontend_proxy_workers %} - upstream frontend_proxy_upstream { - {% for worker in frontend_proxy_workers %} - server "matrix-synapse:{{ worker.port }}"; - {% endfor %} - } - {% endif %} - - {% if media_repository_workers %} - upstream media_repository_upstream { - {% for worker in media_repository_workers %} - server "matrix-synapse:{{ worker.port }}"; - {% endfor %} - } - {% endif %} - - {% if user_dir_workers %} - upstream user_dir_upstream { - {% for worker in user_dir_workers %} - server "matrix-synapse:{{ worker.port }}"; - {% endfor %} - } - {% endif %} -{% endif %} - -server { - listen {{ 8080 if matrix_nginx_proxy_enabled else 80 }}; - server_name {{ matrix_nginx_proxy_proxy_matrix_hostname }}; - - server_tokens off; - root /dev/null; - - {% if matrix_nginx_proxy_https_enabled %} - location /.well-known/acme-challenge { - {% if matrix_nginx_proxy_enabled %} - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "matrix-certbot:8080"; - proxy_pass http://$backend; - {% else %} - {# Generic configuration for use outside of our container setup #} - proxy_pass http://127.0.0.1:{{ matrix_ssl_lets_encrypt_certbot_standalone_http_port }}; - {% endif %} - } - - {% if matrix_nginx_proxy_proxy_matrix_nginx_status_enabled %} - {{ render_nginx_status_location_block(matrix_nginx_proxy_proxy_matrix_nginx_status_allowed_addresses) }} - {% endif %} - - location / { - return 301 https://$http_host$request_uri; - } - {% else %} - {{ render_vhost_directives() }} - {% endif %} -} - -{% if matrix_nginx_proxy_https_enabled %} -server { - listen {{ 8443 if matrix_nginx_proxy_enabled else 443 }} ssl http2; - listen [::]:{{ 8443 if matrix_nginx_proxy_enabled else 443 }} ssl http2; - - server_name {{ matrix_nginx_proxy_proxy_matrix_hostname }}; - - server_tokens off; - root /dev/null; - - ssl_certificate {{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_proxy_matrix_hostname }}/fullchain.pem; - ssl_certificate_key {{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_proxy_matrix_hostname }}/privkey.pem; - - ssl_protocols {{ matrix_nginx_proxy_ssl_protocols }}; - {% if matrix_nginx_proxy_ssl_ciphers != '' %} - ssl_ciphers {{ matrix_nginx_proxy_ssl_ciphers }}; - {% endif %} - ssl_prefer_server_ciphers {{ matrix_nginx_proxy_ssl_prefer_server_ciphers }}; - - {{ render_vhost_directives() }} } -{% endif %} -{% if matrix_nginx_proxy_proxy_matrix_federation_api_enabled %} -{# - This federation vhost is a little special. - It serves federation over HTTP or HTTPS, depending on `matrix_nginx_proxy_https_enabled`. -#} +{% if matrix_nginx_proxy_proxy_synapse_federation_api_enabled %} server { - {% if matrix_nginx_proxy_https_enabled %} - listen 8448 ssl http2; - listen [::]:8448 ssl http2; - {% else %} - listen 8448; - {% endif %} + listen 12088; - server_name {{ matrix_nginx_proxy_proxy_matrix_hostname }}; + server_name {{ matrix_nginx_proxy_proxy_synapse_hostname }}; server_tokens off; root /dev/null; @@ -345,18 +176,6 @@ server { gzip on; gzip_types text/plain application/json; - {% if matrix_nginx_proxy_https_enabled %} - ssl_certificate {{ matrix_nginx_proxy_proxy_matrix_federation_api_ssl_certificate }}; - ssl_certificate_key {{ matrix_nginx_proxy_proxy_matrix_federation_api_ssl_certificate_key }}; - - ssl_protocols {{ matrix_nginx_proxy_ssl_protocols }}; - {% if matrix_nginx_proxy_ssl_ciphers != '' %} - ssl_ciphers {{ matrix_nginx_proxy_ssl_ciphers }}; - {% endif %} - ssl_prefer_server_ciphers {{ matrix_nginx_proxy_ssl_prefer_server_ciphers }}; - - {% endif %} - {% if matrix_nginx_proxy_synapse_workers_enabled %} {% if generic_workers %} # https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappgeneric_worker @@ -367,7 +186,6 @@ server { proxy_set_header X-Forwarded-For $remote_addr; } {% endfor %} - # FIXME: add GET ^/_matrix/federation/v1/groups/ {% endif %} {% if media_repository_workers %} # https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappmedia_repository @@ -389,11 +207,11 @@ server { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; - set $backend "{{ matrix_nginx_proxy_proxy_matrix_federation_api_addr_with_container }}"; + set $backend "{{ matrix_nginx_proxy_proxy_synapse_federation_api_addr_with_container }}"; proxy_pass http://$backend; {% else %} {# Generic configuration for use outside of our container setup #} - proxy_pass http://{{ matrix_nginx_proxy_proxy_matrix_federation_api_addr_sans_container }}; + proxy_pass http://{{ matrix_nginx_proxy_proxy_synapse_federation_api_addr_sans_container }}; {% endif %} proxy_set_header Host $host; From da50fb27a07e19bf5ee1433c21f20064296a9639 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 25 Jan 2021 09:31:52 +0200 Subject: [PATCH 52/70] Whitelist /_matrix/key requests for going to generic workers on the federation port --- roles/matrix-synapse/vars/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/matrix-synapse/vars/main.yml b/roles/matrix-synapse/vars/main.yml index 83325975..9c6d8ce4 100644 --- a/roles/matrix-synapse/vars/main.yml +++ b/roles/matrix-synapse/vars/main.yml @@ -31,5 +31,5 @@ matrix_synapse_workers_generic_worker_client_server_endpoints: "{{ matrix_synaps # We wish to split these, as we normally serve federation separately and don't want them mixed up. # # This is some ugly Ansible/Jinja2 hack (seen here: https://stackoverflow.com/a/47831492), -# which takes a list of various strings and removes the ones NOT containing `/_matrix/federation` anywhere in them. -matrix_synapse_workers_generic_worker_federation_endpoints: "{{ matrix_synapse_workers_generic_worker_endpoints|default([]) | map('regex_search', '.*/_matrix/federation.*')| list | difference([none]) }}" +# which takes a list of various strings and removes the ones NOT containing `/_matrix/federation` or `/_matrix/key` anywhere in them. +matrix_synapse_workers_generic_worker_federation_endpoints: "{{ matrix_synapse_workers_generic_worker_endpoints|default([]) | map('regex_search', '.*(/_matrix/federation|/_matrix/key).*')| list | difference([none]) }}" From 70796703d36efcbe0f67cbf18466335a9a78e849 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 25 Jan 2021 12:14:46 +0200 Subject: [PATCH 53/70] Run Synapse workers in their own containers This switches the `docker exec` method of spawning Synapse workers inside the `matrix-synapse` container with dedicated containers for each worker. We also have dedicated systemd services for each worker, so this are now: - more consistent with everything else (we don't use systemd instantiated services anywhere) - we don't need the "parse systemd instance name into worker name + port" part - we don't need to keep track of PIDs manually - we don't need jq (less depenendencies) - workers dying would be restarted by systemd correctly, like any other service - `docker ps` shows each worker separately and we can observe resource usage --- .../nginx/conf.d/matrix-synapse.conf.j2 | 8 +- .../matrix-synapse-worker-write-pid | 30 ------- roles/matrix-synapse/tasks/init.yml | 7 ++ roles/matrix-synapse/tasks/workers/setup.yml | 14 ++++ .../tasks/workers/setup_install.yml | 84 ++++--------------- .../tasks/workers/setup_uninstall.yml | 42 +++------- .../inject_systemd_services_for_worker.yml | 6 ++ .../workers/util/setup_files_for_worker.yml | 20 +++++ .../systemd/matrix-synapse-worker.service.j2 | 58 +++++++++++++ .../systemd/matrix-synapse-worker@.service.j2 | 39 --------- .../synapse/systemd/matrix-synapse.service.j2 | 20 ++--- .../templates/synapse/worker.yaml.j2 | 22 ++--- 12 files changed, 160 insertions(+), 190 deletions(-) delete mode 100644 roles/matrix-synapse/files/usr-local-bin/matrix-synapse-worker-write-pid create mode 100644 roles/matrix-synapse/tasks/workers/util/inject_systemd_services_for_worker.yml create mode 100644 roles/matrix-synapse/tasks/workers/util/setup_files_for_worker.yml create mode 100644 roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 delete mode 100644 roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 0dcaf9a6..6801f4f9 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -14,7 +14,7 @@ ip_hash; {% for worker in generic_workers %} - server "matrix-synapse:{{ worker.port }}"; + server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}"; {% endfor %} } {% endif %} @@ -22,7 +22,7 @@ {% if frontend_proxy_workers %} upstream frontend_proxy_upstream { {% for worker in frontend_proxy_workers %} - server "matrix-synapse:{{ worker.port }}"; + server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}"; {% endfor %} } {% endif %} @@ -30,7 +30,7 @@ {% if media_repository_workers %} upstream media_repository_upstream { {% for worker in media_repository_workers %} - server "matrix-synapse:{{ worker.port }}"; + server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}"; {% endfor %} } {% endif %} @@ -38,7 +38,7 @@ {% if user_dir_workers %} upstream user_dir_upstream { {% for worker in user_dir_workers %} - server "matrix-synapse:{{ worker.port }}"; + server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}"; {% endfor %} } {% endif %} diff --git a/roles/matrix-synapse/files/usr-local-bin/matrix-synapse-worker-write-pid b/roles/matrix-synapse/files/usr-local-bin/matrix-synapse-worker-write-pid deleted file mode 100644 index 02c5ba09..00000000 --- a/roles/matrix-synapse/files/usr-local-bin/matrix-synapse-worker-write-pid +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# Find a synapse worker's PID and write it to a file so systemd can manage it as a service - -# example invocation: -# matrix-synapse-worker-write-pid user_dir:18700 /run/matrix-synapse-worker.user_dir:18700.pid - -docker_api_call() { curl --silent --unix-socket /var/run/docker.sock ${@}; } - -TARGETCONTAINER=matrix-synapse -TARGETWORKER=${1} -PIDFILE=${2} - -# get ID list of subprocesses executed in $TARGETCONTAINER, and for each.. -for EXECID in $(docker_api_call http://localhost/containers/${TARGETCONTAINER}/json | jq --raw-output '.ExecIDs[]') -do - # fetch detailed process info - EXECINFO=$(docker_api_call http://localhost/exec/${EXECID}/json) - - # extract config file path from last command argument - WORKERCONFIGFILE=$(echo ${EXECINFO} | jq --raw-output .ProcessConfig.arguments[-1]) - - # reconstruct worker name - WORKERNAME=${WORKERCONFIGFILE#*/worker.} - WORKERNAME=${WORKERNAME%.yaml} - - # if name matches the target worker: write out most recent PID & quit - [ "${WORKERNAME}" = "${TARGETWORKER}" ] \ - && echo ${EXECINFO} | jq --raw-output .Pid > ${PIDFILE} \ - && exit 0 -done diff --git a/roles/matrix-synapse/tasks/init.yml b/roles/matrix-synapse/tasks/init.yml index bdb62476..46c7d22b 100644 --- a/roles/matrix-synapse/tasks/init.yml +++ b/roles/matrix-synapse/tasks/init.yml @@ -2,6 +2,13 @@ matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-synapse.service'] }}" when: matrix_synapse_enabled|bool +- name: Ensure systemd services for workers are injected + include_tasks: "{{ role_path }}/tasks/workers/util/inject_systemd_services_for_worker.yml" + with_items: "{{ matrix_synapse_workers_enabled_list }}" + loop_control: + loop_var: matrix_synapse_worker_details + when: matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool + - set_fact: matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-goofys.service'] }}" when: matrix_s3_media_store_enabled|bool diff --git a/roles/matrix-synapse/tasks/workers/setup.yml b/roles/matrix-synapse/tasks/workers/setup.yml index 083da807..3a7e6c98 100644 --- a/roles/matrix-synapse/tasks/workers/setup.yml +++ b/roles/matrix-synapse/tasks/workers/setup.yml @@ -1,5 +1,19 @@ --- +# A previous version of the worker setup used this. +# This is a temporary cleanup for people who ran that version. +- name: Ensure old matrix-synapse.service.wants directory is gone + file: + path: "{{ matrix_systemd_path }}/matrix-synapse.service.wants" + state: absent + +# Same. This was part of a previous version of the worker setup. +# No longer necessary. +- name: Ensure matrix-synapse-worker-write-pid script is removed + file: + path: "{{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid" + state: absent + - include_tasks: "{{ role_path }}/tasks/workers/setup_install.yml" when: "matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool" diff --git a/roles/matrix-synapse/tasks/workers/setup_install.yml b/roles/matrix-synapse/tasks/workers/setup_install.yml index 33ddb0b9..ff34210a 100644 --- a/roles/matrix-synapse/tasks/workers/setup_install.yml +++ b/roles/matrix-synapse/tasks/workers/setup_install.yml @@ -1,81 +1,33 @@ --- -- name: Ensure synapse worker base service file installed - template: - src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker@.service.j2" - dest: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service" - mode: 0644 - register: matrix_synapse_worker_systemd_service_result - -- name: Ensure previous worker service symlinks are cleaned - file: - path: "{{ item.root + '/' + item.path }}" - state: absent - when: - - item.state == 'link' - - item.path is match('matrix-synapse-worker@.*\\.service') - with_filetree: - - "{{ matrix_systemd_path }}/matrix-synapse.service.wants" - -- name: Ensure systemd reloaded the worker service unit - service: - daemon_reload: yes - -- name: Ensure individual worker service symlinks exist - service: - name: "matrix-synapse-worker@{{ item.type }}:{{ item.port }}.service" - enabled: true - with_items: "{{ matrix_synapse_workers_enabled_list }}" - - name: Find worker configs to be cleaned find: path: "{{ matrix_synapse_config_dir_path }}" patterns: "worker.*.yaml" use_regex: true - register: worker_config_files + register: matrix_synapse_workers_current_config_files - name: Ensure previous worker configs are cleaned file: path: "{{ item.path }}" state: absent - with_items: "{{ worker_config_files.files }}" + with_items: "{{ matrix_synapse_workers_current_config_files.files }}" -- name: Ensure creation of specific worker configs - template: - src: "{{ role_path }}/templates/synapse/worker.yaml.j2" - dest: "{{ matrix_synapse_config_dir_path }}/worker.{{ item.type }}:{{ item.port }}.yaml" - with_list: "{{ matrix_synapse_workers_enabled_list }}" - -- name: Add workers to synapse.wants list - set_fact: - matrix_synapse_systemd_wanted_services_list: > - {{ matrix_synapse_systemd_wanted_services_list + - ['matrix-synapse-worker@' + item.type + ':' + item.port|string + '.service'] }} - with_items: "{{ matrix_synapse_workers_enabled_list }}" - -- name: Ensure matrix-synapse-worker-write-pid script is created - copy: - src: "{{ role_path }}/files/usr-local-bin/matrix-synapse-worker-write-pid" - dest: "{{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid" - mode: 0750 - -- name: Ensure jq is installed (Archlinux) - pacman: - name: - - jq - state: present - when: (ansible_distribution == 'Archlinux') +- name: Find worker systemd services to be cleaned + find: + path: "{{ matrix_systemd_path }}" + patterns: "matrix-synapse-worker.*.service" + use_regex: true + register: matrix_synapse_workers_current_systemd_services -- name: Ensure jq is installed (CentOS) - yum: - name: - - jq - state: present - when: (ansible_distribution == 'CentOS') +- name: Ensure previous worker systemd services are cleaned + file: + path: "{{ item.path }}" + state: absent + with_items: "{{ matrix_synapse_workers_current_systemd_services.files }}" -- name: Ensure jq is installed (Debian) - apt: - name: - - jq - state: present - when: (ansible_os_family == 'Debian') +- name: Ensure creation of worker systemd service files and configuration files + include_tasks: "{{ role_path }}/tasks/workers/util/setup_files_for_worker.yml" + with_items: "{{ matrix_synapse_workers_enabled_list }}" + loop_control: + loop_var: matrix_synapse_worker_details diff --git a/roles/matrix-synapse/tasks/workers/setup_uninstall.yml b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml index a9884fca..4a90bfa6 100644 --- a/roles/matrix-synapse/tasks/workers/setup_uninstall.yml +++ b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml @@ -7,46 +7,30 @@ service: name: "{{ item.key }}" state: stopped - with_dict: "{{ ansible_facts.services|default({})|dict2items|selectattr('key', 'match', 'matrix-synapse-worker@.+\\.service')|list|items2dict }}" - -- name: Ensure worker service symlinks are cleaned - file: - path: "{{ item.root + '/' + item.path }}" - state: absent - when: - - item.state == 'link' - - item.path is match('matrix-synapse-worker@.*\\.service') - with_filetree: - - "{{ matrix_systemd_path }}/matrix-synapse.service.wants" - -- name: Ensure synapse worker base service file gets removed - file: - path: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service" - state: absent - register: matrix_synapse_worker_systemd_service_result + with_dict: "{{ ansible_facts.services|default({})|dict2items|selectattr('key', 'match', 'matrix-synapse-worker-.+\\.service')|list|items2dict }}" - name: Find worker configs to be cleaned find: path: "{{ matrix_synapse_config_dir_path }}" patterns: "worker.*.yaml" use_regex: true - register: worker_config_files + register: matrix_synapse_workers_current_config_files -- name: Ensure worker configs are cleaned +- name: Ensure previous worker configs are cleaned file: path: "{{ item.path }}" state: absent - with_items: "{{ worker_config_files.files }}" + with_items: "{{ matrix_synapse_workers_current_config_files.files }}" -- name: Remove workers from synapse.wants list - set_fact: - matrix_synapse_systemd_wanted_services_list: "{{ matrix_synapse_systemd_wanted_services_list | reject('search', '^matrix-synapse-worker@') | list }}" - -- name: Ensure systemd noticed removal of worker service units - service: - daemon_reload: yes +- name: Find worker systemd services to be cleaned + find: + path: "{{ matrix_systemd_path }}" + patterns: "matrix-synapse-worker.*.service" + use_regex: true + register: matrix_synapse_workers_current_systemd_services -- name: Ensure matrix-synapse-worker-write-pid script is removed +- name: Ensure previous worker systemd services are cleaned file: - path: "{{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid" + path: "{{ item.path }}" state: absent + with_items: "{{ matrix_synapse_workers_current_systemd_services.files }}" diff --git a/roles/matrix-synapse/tasks/workers/util/inject_systemd_services_for_worker.yml b/roles/matrix-synapse/tasks/workers/util/inject_systemd_services_for_worker.yml new file mode 100644 index 00000000..c95f881a --- /dev/null +++ b/roles/matrix-synapse/tasks/workers/util/inject_systemd_services_for_worker.yml @@ -0,0 +1,6 @@ + +- set_fact: + matrix_synapse_worker_systemd_service_name: "matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.port }}.service" + +- set_fact: + matrix_systemd_services_list: "{{ matrix_systemd_services_list + [matrix_synapse_worker_systemd_service_name] }}" diff --git a/roles/matrix-synapse/tasks/workers/util/setup_files_for_worker.yml b/roles/matrix-synapse/tasks/workers/util/setup_files_for_worker.yml new file mode 100644 index 00000000..6a15e048 --- /dev/null +++ b/roles/matrix-synapse/tasks/workers/util/setup_files_for_worker.yml @@ -0,0 +1,20 @@ + +- set_fact: + matrix_synapse_worker_systemd_service_name: "matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.port }}" + +- set_fact: + matrix_synapse_worker_container_name: "{{ matrix_synapse_worker_systemd_service_name }}" + +- set_fact: + matrix_synapse_worker_config_file_name: "worker.{{ matrix_synapse_worker_details.type }}_{{ matrix_synapse_worker_details.port }}.yaml" + +- name: Ensure configuration exists for {{ matrix_synapse_worker_systemd_service_name }} + template: + src: "{{ role_path }}/templates/synapse/worker.yaml.j2" + dest: "{{ matrix_synapse_config_dir_path }}/{{ matrix_synapse_worker_config_file_name }}" + +- name: Ensure systemd service exists for {{ matrix_synapse_worker_systemd_service_name }} + template: + src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker.service.j2" + dest: "{{ matrix_systemd_path }}/{{ matrix_synapse_worker_systemd_service_name }}.service" + mode: 0644 diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 new file mode 100644 index 00000000..0f5e7be2 --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 @@ -0,0 +1,58 @@ +#jinja2: lstrip_blocks: "True" +[Unit] +Description=Synapse worker ({{ matrix_synapse_worker_container_name }}) +AssertPathExists={{ matrix_synapse_config_dir_path }}/{{ matrix_synapse_worker_config_file_name }} +After=matrix-synapse.service + +[Service] +Type=simple +Environment="HOME={{ matrix_systemd_unit_home_path }}" + +ExecStartPre=-{{ matrix_host_command_docker }} kill {{ matrix_synapse_worker_container_name }} +ExecStartPre=-{{ matrix_host_command_docker }} rm {{ matrix_synapse_worker_container_name }} + +# Intentional delay, so that the homeserver can manage to start. +ExecStartPre={{ matrix_host_command_sleep }} 5 + +ExecStart={{ matrix_host_command_docker }} run --rm --name {{ matrix_synapse_worker_container_name }} \ + --log-driver=none \ + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + --cap-drop=ALL \ + --entrypoint=python \ + --read-only \ + --tmpfs=/tmp:rw,noexec,nosuid,size={{ matrix_synapse_tmp_directory_size_mb }}m \ + --network={{ matrix_docker_network }} \ + {% if matrix_synapse_workers_enabled and matrix_synapse_workers_container_host_bind_address %} + {% if matrix_synapse_worker_details.port != 0 %} + -p {{ '' if matrix_synapse_workers_container_host_bind_address == '*' else (matrix_synapse_workers_container_host_bind_address + ':') }}{{ matrix_synapse_worker_details.port }}:{{ matrix_synapse_worker_details.port }} \ + {% endif %} + {% if matrix_synapse_worker_details.metrics_port != 0 %} + -p {{ '' if matrix_synapse_workers_container_host_bind_address == '*' else (matrix_synapse_workers_container_host_bind_address + ':') }}{{ matrix_synapse_worker_details.metrics_port }}:{{ matrix_synapse_worker_details.metrics_port }} \ + {% endif %} + {% endif %} + --mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/data,ro \ + --mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \ + {% for volume in matrix_synapse_container_additional_volumes %} + -v {{ volume.src }}:{{ volume.dst }}:{{ volume.options }} \ + {% endfor %} + {% for arg in matrix_synapse_container_extra_arguments %} + {{ arg }} \ + {% endfor %} + {{ matrix_synapse_docker_image }} \ + -m synapse.app.{{ matrix_synapse_worker_details.type }} -c /data/homeserver.yaml -c /data/{{ matrix_synapse_worker_config_file_name }} + + +ExecStop=-{{ matrix_host_command_docker }} kill {{ matrix_synapse_worker_container_name }} +ExecStop=-{{ matrix_host_command_docker }} rm {{ matrix_synapse_worker_container_name }} + +ExecReload={{ matrix_host_command_docker }} exec {{ matrix_synapse_worker_container_name }} /bin/sh -c 'kill -HUP 1' +Restart=always +RestartSec=30 +SyslogIdentifier={{ matrix_synapse_worker_container_name }} + +# Intentionally not making this WantedBy=matrix-synapse.service, +# as matrix.synapse.service already has `Wants=` lines. +# Also, WantedBy will trigger the creation of some `matrix-synapse.service.wants/` directory, +# which we'd have to clean, etc. Better not. +[Install] +WantedBy=multi-user.target diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 deleted file mode 100644 index 983426ba..00000000 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 +++ /dev/null @@ -1,39 +0,0 @@ -#jinja2: lstrip_blocks: "True" -# Instantiable worker service, running inside the synapse container -# alongside the homeserver main process. -# c.f. https://github.com/matrix-org/synapse/pull/4662 -[Unit] -Description=Matrix worker synapse.app.%i -AssertPathExists={{ matrix_synapse_config_dir_path }}/worker.%i.yaml -After=matrix-synapse.service -BindsTo=matrix-synapse.service - -[Service] -Type=simple - -# Intentional delay, so that the homeserver can manage to start. -ExecStartPre={{ matrix_host_command_sleep }} 5 - -# no sane way of instancing more than one variable (systemd "cant-fix" 🤦) -# c.f. https://github.com/systemd/systemd/issues/14895#issuecomment-594123923 -# So use good ol' shell parameter expansion to get the worker type.. -ExecStart=/bin/sh -c "WORKER=%i; WORKER=$${WORKER%%:*}; \ - exec {{ matrix_host_command_docker }} exec \ - --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ - matrix-synapse \ - python -m synapse.app.$${WORKER} -c /data/homeserver.yaml -c /data/worker.%i.yaml" - -# wait for worker startup & write out PID of actual worker process so systemd can handle it -ExecStartPost={{ matrix_host_command_sleep }} 5 -ExecStartPost={{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid %i /run/matrix-synapse-worker.%i.pid - -ExecReload=/bin/kill -HUP $MAINPID -ExecStop=/bin/kill $MAINPID -PIDFile=/run/matrix-synapse-worker.%i.pid -KillMode=process -Restart=always -RestartSec=10 -SyslogIdentifier=matrix-synapse-%i - -[Install] -WantedBy=matrix-synapse.service diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 index 3b9ccdf8..3028cfc1 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 @@ -4,10 +4,18 @@ Description=Synapse server {% for service in matrix_synapse_systemd_required_services_list %} Requires={{ service }} After={{ service }} + {% endfor %} {% for service in matrix_synapse_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} + +{% if matrix_synapse_workers_enabled %} +{% for matrix_synapse_worker_details in matrix_synapse_workers_enabled_list %} +Wants=matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.port }}.service +{% endfor %} +{% endif %} + DefaultDependencies=no [Service] @@ -45,16 +53,6 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \ {% if matrix_synapse_manhole_enabled and matrix_synapse_container_manhole_api_host_bind_port %} -p {{ matrix_synapse_container_manhole_api_host_bind_port }}:9000 \ {% endif %} - {% if matrix_synapse_workers_enabled and matrix_synapse_workers_container_host_bind_address %} - {% for worker in matrix_synapse_workers_enabled_list %} - {% if worker.port != 0 %} - -p {{ '' if matrix_synapse_workers_container_host_bind_address == '*' else (matrix_synapse_workers_container_host_bind_address + ':') }}{{ worker.port }}:{{ worker.port }} \ - {% endif %} - {% if worker.metrics_port != 0 %} - -p {{ '' if matrix_synapse_workers_container_host_bind_address == '*' else (matrix_synapse_workers_container_host_bind_address + ':') }}{{ worker.metrics_port }}:{{ worker.metrics_port }} \ - {% endif %} - {% endfor %} - {% endif %} --mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/data,ro \ --mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \ {% for volume in matrix_synapse_container_additional_volumes %} @@ -68,7 +66,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \ ExecStop=-{{ matrix_host_command_docker }} kill matrix-synapse ExecStop=-{{ matrix_host_command_docker }} rm matrix-synapse -ExecReload={{ matrix_host_command_docker }} exec matrix-synapse kill -HUP 1 +ExecReload={{ matrix_host_command_docker }} exec matrix-synapse /bin/sh -c 'kill -HUP 1' Restart=always RestartSec=30 SyslogIdentifier=matrix-synapse diff --git a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 index f77ff4ad..bac2f7ee 100644 --- a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 @@ -1,32 +1,32 @@ #jinja2: lstrip_blocks: "True" -worker_app: synapse.app.{{ item.type }} -worker_name: {{ item.type ~ ':' ~ item.port }} +worker_app: synapse.app.{{ matrix_synapse_worker_details.type }} +worker_name: {{ matrix_synapse_worker_details.type ~ ':' ~ matrix_synapse_worker_details.port }} -worker_replication_host: 127.0.0.1 +worker_replication_host: matrix-synapse worker_replication_http_port: {{ matrix_synapse_replication_http_port }} worker_listeners: -{% if item.type not in [ 'appservice', 'federation_sender', 'pusher' ] %} +{% if matrix_synapse_worker_details.type not in [ 'appservice', 'federation_sender', 'pusher' ] %} - type: http - port: {{ item.port }} + port: {{ matrix_synapse_worker_details.port }} resources: - names: -{% if item.type in [ 'generic_worker', 'frontend_proxy', 'user_dir' ] %} +{% if matrix_synapse_worker_details.type in [ 'generic_worker', 'frontend_proxy', 'user_dir' ] %} - client {% endif %} -{% if item.type in [ 'generic_worker' ] %} +{% if matrix_synapse_worker_details.type in [ 'generic_worker' ] %} - federation -{% elif item.type in [ 'media_repository' ] %} +{% elif matrix_synapse_worker_details.type in [ 'media_repository' ] %} - media {% endif %} {% endif %} - type: metrics bind_address: ['127.0.0.1'] - port: {{ item.metrics_port }} + port: {{ matrix_synapse_worker_details.metrics_port }} -{% if item.type == 'frontend_proxy' %} -worker_main_http_uri: http://127.0.0.1:8008 +{% if matrix_synapse_worker_details.type == 'frontend_proxy' %} +worker_main_http_uri: http://matrix-synapse:8008 {% endif %} worker_daemonize: false From 01747c8cc45ffd809ed558b3e9ec3082c6e33557 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 25 Jan 2021 12:24:12 +0200 Subject: [PATCH 54/70] Prevent Synapse warning about enabling metric listeners with enable_metrics: false > synapse.app.generic_worker - 606 - WARNING - None - Metrics listener configured, but enable_metrics is not True! --- roles/matrix-synapse/templates/synapse/worker.yaml.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 index bac2f7ee..7bfc94cb 100644 --- a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 @@ -19,11 +19,12 @@ worker_listeners: {% elif matrix_synapse_worker_details.type in [ 'media_repository' ] %} - media {% endif %} - {% endif %} +{% if matrix_synapse_metrics_enabled %} - type: metrics bind_address: ['127.0.0.1'] port: {{ matrix_synapse_worker_details.metrics_port }} +{% endif %} {% if matrix_synapse_worker_details.type == 'frontend_proxy' %} worker_main_http_uri: http://matrix-synapse:8008 From 1462409b3481d3f54cd527acebb86b4bb27873e7 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 25 Jan 2021 12:29:47 +0200 Subject: [PATCH 55/70] Fix worker listening addresses Not specifying bind addresses for the worker resulted in this warning: > synapse.app - 47 - WARNING - None - Failed to listen on 0.0.0.0, continuing because listening on [::] Additionally, metrics listening only on 127.0.0.1 seems like a no-op. Only having it accessible from within the container is likely not what we intend. Changed that to all interfaces as well. Whether it actually gets exposed or not depends on the systemd service and `matrix_synapse_workers_container_host_bind_address`. --- roles/matrix-synapse/templates/synapse/worker.yaml.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 index 7bfc94cb..1acc4df2 100644 --- a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 @@ -8,6 +8,7 @@ worker_replication_http_port: {{ matrix_synapse_replication_http_port }} worker_listeners: {% if matrix_synapse_worker_details.type not in [ 'appservice', 'federation_sender', 'pusher' ] %} - type: http + bind_addresses: ['::'] port: {{ matrix_synapse_worker_details.port }} resources: - names: @@ -22,7 +23,7 @@ worker_listeners: {% endif %} {% if matrix_synapse_metrics_enabled %} - type: metrics - bind_address: ['127.0.0.1'] + bind_addresses: ['::'] port: {{ matrix_synapse_worker_details.metrics_port }} {% endif %} From 66cdc7bf5aa6081caa6545aa670a113b9f22b18b Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 25 Jan 2021 13:02:01 +0200 Subject: [PATCH 56/70] Clean up worker.yaml generation a bit and make it more flexible --- roles/matrix-synapse/defaults/main.yml | 11 ++++++- .../templates/synapse/homeserver.yaml.j2 | 3 ++ .../templates/synapse/worker.yaml.j2 | 30 ++++++++++++------- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 7c00c3e5..1f1057dd 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -333,7 +333,16 @@ matrix_synapse_redis_host: "" matrix_synapse_redis_port: 6379 matrix_synapse_redis_password: "" -# Port used for communication between main synapse process and workers +# Controls whether Synapse starts a replication listener necessary for workers. +# +# If Redis is available, we prefer to use that, instead of talking over Synapse's custom replication protocol. +# +# matrix_synapse_replication_listener_enabled: "{{ matrix_synapse_workers_enabled and not matrix_redis_enabled }}" +# We force-enable this listener for now until we debug why communication via Redis fails. +matrix_synapse_replication_listener_enabled: true + +# Port used for communication between main synapse process and workers. +# Only gets used if `matrix_synapse_replication_listener_enabled: true` matrix_synapse_replication_http_port: 9093 # Send ERROR logs to sentry.io for easier tracking diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index 9738ea4b..3bfada51 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -277,6 +277,8 @@ listeners: {% endif %} {% if matrix_synapse_workers_enabled %} + +{% if matrix_synapse_replication_listener_enabled %} # c.f. https://github.com/matrix-org/synapse/tree/master/docs/workers.md # HTTP replication: for the workers to send data to the main synapse process - port: {{ matrix_synapse_replication_http_port }} @@ -284,6 +286,7 @@ listeners: type: http resources: - names: [replication] +{% endif %} # c.f. https://github.com/matrix-org/synapse/tree/master/contrib/systemd-with-workers/README.md worker_app: synapse.app.homeserver diff --git a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 index 1acc4df2..330086ad 100644 --- a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 @@ -2,30 +2,40 @@ worker_app: synapse.app.{{ matrix_synapse_worker_details.type }} worker_name: {{ matrix_synapse_worker_details.type ~ ':' ~ matrix_synapse_worker_details.port }} +{% if matrix_synapse_replication_listener_enabled %} worker_replication_host: matrix-synapse worker_replication_http_port: {{ matrix_synapse_replication_http_port }} +{% endif %} + +{% set has_listeners = (matrix_synapse_worker_details.type not in [ 'appservice', 'federation_sender', 'pusher' ] or matrix_synapse_metrics_enabled) %} + +{% set http_resources = [] %} + +{% if matrix_synapse_worker_details.type in ['generic_worker', 'frontend_proxy', 'user_dir'] %} + {% set http_resources = http_resources + ['client'] %} +{% endif %} +{% if matrix_synapse_worker_details.type in ['generic_worker'] %} + {% set http_resources = http_resources+ ['federation'] %} +{% endif %} +{% if matrix_synapse_worker_details.type in ['media_repository'] %} + {% set http_resources = http_resources + ['media'] %} +{% endif %} +{% if http_resources|length > 0 or matrix_synapse_metrics_enabled %} worker_listeners: -{% if matrix_synapse_worker_details.type not in [ 'appservice', 'federation_sender', 'pusher' ] %} +{% if http_resources|length > 0 %} - type: http bind_addresses: ['::'] port: {{ matrix_synapse_worker_details.port }} resources: - - names: -{% if matrix_synapse_worker_details.type in [ 'generic_worker', 'frontend_proxy', 'user_dir' ] %} - - client -{% endif %} -{% if matrix_synapse_worker_details.type in [ 'generic_worker' ] %} - - federation -{% elif matrix_synapse_worker_details.type in [ 'media_repository' ] %} - - media -{% endif %} + - names: {{ http_resources|to_json }} {% endif %} {% if matrix_synapse_metrics_enabled %} - type: metrics bind_addresses: ['::'] port: {{ matrix_synapse_worker_details.metrics_port }} {% endif %} +{% endif %} {% if matrix_synapse_worker_details.type == 'frontend_proxy' %} worker_main_http_uri: http://matrix-synapse:8008 From d3ecc6f017c801fcf5de10392265faf3cab28420 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 25 Jan 2021 13:55:08 +0200 Subject: [PATCH 57/70] Fix bridges failing to upload media when Synapse workers are enabled --- group_vars/matrix_servers | 10 +++++++--- roles/matrix-base/defaults/main.yml | 5 +++++ roles/matrix-bot-matrix-reminder-bot/defaults/main.yml | 2 +- roles/matrix-bridge-appservice-irc/defaults/main.yml | 2 +- roles/matrix-bridge-mautrix-facebook/defaults/main.yml | 2 +- roles/matrix-bridge-mautrix-hangouts/defaults/main.yml | 2 +- roles/matrix-bridge-mautrix-telegram/defaults/main.yml | 2 +- .../matrix-bridge-mx-puppet-discord/defaults/main.yml | 2 +- .../defaults/main.yml | 2 +- roles/matrix-bridge-mx-puppet-skype/defaults/main.yml | 2 +- roles/matrix-bridge-mx-puppet-slack/defaults/main.yml | 2 +- roles/matrix-bridge-mx-puppet-steam/defaults/main.yml | 2 +- .../matrix-bridge-mx-puppet-twitter/defaults/main.yml | 2 +- roles/matrix-dimension/templates/config.yaml.j2 | 2 +- 14 files changed, 24 insertions(+), 15 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 88afb186..11783fc6 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -18,6 +18,10 @@ matrix_identity_server_url: "{{ ('https://' + matrix_server_fqn_matrix) if matrix_ma1sd_enabled else None }}" +# If Synapse workers are enabled and matrix-nginx-proxy is disabled, certain APIs may not work over 'http://matrix-synapse:8008'. +# This is because we explicitly disable them for the main Synapse process. +matrix_homeserver_container_url: "{{ 'http://matrix-nginx-proxy:12080' if matrix_nginx_proxy_enabled else 'http://matrix-synapse:8008' }}" + ###################################################################### # # /matrix-base @@ -283,7 +287,7 @@ matrix_mautrix_signal_systemd_required_services_list: | matrix_mautrix_signal_homeserver_domain: '{{ matrix_domain }}' -matrix_mautrix_signal_homeserver_address: "{{ 'http://matrix-synapse:8008' if matrix_synapse_enabled else '' }}" +matrix_mautrix_signal_homeserver_address: "{{ matrix_homeserver_container_url }}" matrix_mautrix_signal_homeserver_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'si.hs.token') | to_uuid }}" @@ -674,7 +678,7 @@ matrix_corporal_systemd_required_services_list: | }} # This goes to Synapse's vhost -matrix_corporal_matrix_homeserver_api_endpoint: "http://matrix-nginx-proxy:12080" +matrix_corporal_matrix_homeserver_api_endpoint: "{{ matrix_homeserver_container_url }}" matrix_corporal_matrix_auth_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret }}" @@ -1388,7 +1392,7 @@ matrix_registration_riot_instance: "{{ ('https://' + matrix_server_fqn_element) matrix_registration_shared_secret: "{{ matrix_synapse_registration_shared_secret if matrix_synapse_enabled else '' }}" -matrix_registration_server_location: "{{ 'http://matrix-synapse:8008' if matrix_synapse_enabled else '' }}" +matrix_registration_server_location: "{{ matrix_homeserver_container_url }}" matrix_registration_api_validate_certs: "{{ false if matrix_ssl_retrieval_method == 'self-signed' else true }}" diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index d8285e1c..02b5906d 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -72,6 +72,11 @@ matrix_ntpd_service: "{{ 'ntpd' if ansible_os_family == 'RedHat' or ansible_dist matrix_homeserver_url: "https://{{ matrix_server_fqn_matrix }}" +# Specifies where the homeserver is on the container network. +# Where this is depends on whether there's a reverse-proxy in front of it, etc. +# This likely gets overriden elsewhere. +matrix_homeserver_container_url: "http://matrix-synapse:8008" + matrix_identity_server_url: ~ matrix_integration_manager_rest_url: ~ diff --git a/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml b/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml index 29bc8307..c3deb2f2 100644 --- a/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml +++ b/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml @@ -58,7 +58,7 @@ matrix_bot_matrix_reminder_bot_matrix_user_id: '@{{ matrix_bot_matrix_reminder_b # The password that the bot uses to authenticate. matrix_bot_matrix_reminder_bot_matrix_user_password: '' -matrix_bot_matrix_reminder_bot_matrix_homeserver_url: 'http://matrix-synapse:8008' +matrix_bot_matrix_reminder_bot_matrix_homeserver_url: "{{ matrix_homeserver_container_url }}" # The timezone to use when creating reminders. # Examples: 'Europe/London', 'Etc/UTC' diff --git a/roles/matrix-bridge-appservice-irc/defaults/main.yml b/roles/matrix-bridge-appservice-irc/defaults/main.yml index ba4e1e1b..e2c91926 100644 --- a/roles/matrix-bridge-appservice-irc/defaults/main.yml +++ b/roles/matrix-bridge-appservice-irc/defaults/main.yml @@ -14,7 +14,7 @@ matrix_appservice_irc_base_path: "{{ matrix_base_data_path }}/appservice-irc" matrix_appservice_irc_config_path: "{{ matrix_appservice_irc_base_path }}/config" matrix_appservice_irc_data_path: "{{ matrix_appservice_irc_base_path }}/data" -matrix_appservice_irc_homeserver_url: 'http://matrix-synapse:8008' +matrix_appservice_irc_homeserver_url: "{{ matrix_homeserver_container_url }}" matrix_appservice_irc_homeserver_media_url: 'https://{{ matrix_server_fqn_matrix }}' matrix_appservice_irc_homeserver_domain: '{{ matrix_domain }}' matrix_appservice_irc_homeserver_enablePresence: true diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml index e99514e0..120c2960 100644 --- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml @@ -16,7 +16,7 @@ matrix_mautrix_facebook_config_path: "{{ matrix_mautrix_facebook_base_path }}/co matrix_mautrix_facebook_data_path: "{{ matrix_mautrix_facebook_base_path }}/data" matrix_mautrix_facebook_docker_src_files_path: "{{ matrix_mautrix_facebook_base_path }}/docker-src" -matrix_mautrix_facebook_homeserver_address: 'http://matrix-synapse:8008' +matrix_mautrix_facebook_homeserver_address: "{{ matrix_homeserver_container_url }}" matrix_mautrix_facebook_homeserver_domain: '{{ matrix_domain }}' matrix_mautrix_facebook_appservice_address: 'http://matrix-mautrix-facebook:29319' diff --git a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml index 8dfee030..183c3ac7 100644 --- a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml @@ -18,7 +18,7 @@ matrix_mautrix_hangouts_docker_src_files_path: "{{ matrix_mautrix_hangouts_base_ matrix_mautrix_hangouts_public_endpoint: '/mautrix-hangouts' -matrix_mautrix_hangouts_homeserver_address: 'http://matrix-synapse:8008' +matrix_mautrix_hangouts_homeserver_address: "{{ matrix_homeserver_container_url }}" matrix_mautrix_hangouts_homeserver_domain: '{{ matrix_domain }}' matrix_mautrix_hangouts_appservice_address: 'http://matrix-mautrix-hangouts:8080' diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml index 7e072b5a..5eab0c0b 100644 --- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml @@ -25,7 +25,7 @@ matrix_mautrix_telegram_bot_token: disabled # Example: /741a0483-ba17-4682-9900-30bd7269f1cc matrix_mautrix_telegram_public_endpoint: '' -matrix_mautrix_telegram_homeserver_address: 'http://matrix-synapse:8008' +matrix_mautrix_telegram_homeserver_address: "{{ matrix_homeserver_container_url }}" matrix_mautrix_telegram_homeserver_domain: '{{ matrix_domain }}' matrix_mautrix_telegram_appservice_address: 'http://matrix-mautrix-telegram:8080' matrix_mautrix_telegram_appservice_public_external: 'https://{{ matrix_server_fqn_matrix }}{{ matrix_mautrix_telegram_public_endpoint }}' diff --git a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml index 97b20313..25f6ae06 100644 --- a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml @@ -22,7 +22,7 @@ matrix_mx_puppet_discord_docker_src_files_path: "{{ matrix_mx_puppet_discord_bas matrix_mx_puppet_discord_appservice_port: "8432" -matrix_mx_puppet_discord_homeserver_address: 'http://matrix-synapse:8008' +matrix_mx_puppet_discord_homeserver_address: "{{ matrix_homeserver_container_url }}" matrix_mx_puppet_discord_homeserver_domain: '{{ matrix_domain }}' matrix_mx_puppet_discord_appservice_address: 'http://matrix-mx-puppet-discord:{{ matrix_mx_puppet_discord_appservice_port }}' diff --git a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml index cd08c010..4c9fbd98 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml @@ -16,7 +16,7 @@ matrix_mx_puppet_instagram_data_path: "{{ matrix_mx_puppet_instagram_base_path } matrix_mx_puppet_instagram_docker_src_files_path: "{{ matrix_mx_puppet_instagram_base_path }}/docker-src" matrix_mx_puppet_instagram_appservice_port: "8440" -matrix_mx_puppet_instagram_homeserver_address: 'http://matrix-synapse:8008' +matrix_mx_puppet_instagram_homeserver_address: "{{ matrix_homeserver_container_url }}" matrix_mx_puppet_instagram_homeserver_domain: '{{ matrix_domain }}' matrix_mx_puppet_instagram_appservice_address: 'http://matrix-mx-puppet-instagram:{{ matrix_mx_puppet_instagram_appservice_port }}' diff --git a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml index 83cd3dc5..53c8e379 100644 --- a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml @@ -17,7 +17,7 @@ matrix_mx_puppet_skype_docker_src_files_path: "{{ matrix_mx_puppet_skype_base_pa matrix_mx_puppet_skype_appservice_port: "8438" -matrix_mx_puppet_skype_homeserver_address: 'http://matrix-synapse:8008' +matrix_mx_puppet_skype_homeserver_address: "{{ matrix_homeserver_container_url }}" matrix_mx_puppet_skype_appservice_address: 'http://matrix-mx-puppet-skype:{{ matrix_mx_puppet_skype_appservice_port }}' # "@user:server.com" to allow specific user diff --git a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml index 70b98ece..b1fb7487 100644 --- a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml @@ -22,7 +22,7 @@ matrix_mx_puppet_slack_docker_src_files_path: "{{ matrix_mx_puppet_slack_base_pa matrix_mx_puppet_slack_appservice_port: "8432" -matrix_mx_puppet_slack_homeserver_address: 'http://matrix-synapse:8008' +matrix_mx_puppet_slack_homeserver_address: "{{ matrix_homeserver_container_url }}" matrix_mx_puppet_slack_homeserver_domain: '{{ matrix_domain }}' matrix_mx_puppet_slack_appservice_address: 'http://matrix-mx-puppet-slack:{{ matrix_mx_puppet_slack_appservice_port }}' diff --git a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml index 15fa889f..c3ac977e 100644 --- a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml @@ -22,7 +22,7 @@ matrix_mx_puppet_steam_docker_src_files_path: "{{ matrix_mx_puppet_steam_base_pa matrix_mx_puppet_steam_appservice_port: "8432" -matrix_mx_puppet_steam_homeserver_address: 'http://matrix-synapse:8008' +matrix_mx_puppet_steam_homeserver_address: "{{ matrix_homeserver_container_url }}" matrix_mx_puppet_steam_homeserver_domain: '{{ matrix_domain }}' matrix_mx_puppet_steam_appservice_address: 'http://matrix-mx-puppet-steam:{{ matrix_mx_puppet_steam_appservice_port }}' diff --git a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml index 28639fda..d8582e53 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml @@ -22,7 +22,7 @@ matrix_mx_puppet_twitter_docker_src_files_path: "{{ matrix_mx_puppet_twitter_bas matrix_mx_puppet_twitter_appservice_port: "8432" -matrix_mx_puppet_twitter_homeserver_address: 'http://matrix-synapse:8008' +matrix_mx_puppet_twitter_homeserver_address: "{{ matrix_homeserver_container_url }}" matrix_mx_puppet_twitter_homeserver_domain: '{{ matrix_domain }}' matrix_mx_puppet_twitter_appservice_address: 'http://matrix-mx-puppet-twitter:{{ matrix_mx_puppet_twitter_appservice_port }}' diff --git a/roles/matrix-dimension/templates/config.yaml.j2 b/roles/matrix-dimension/templates/config.yaml.j2 index 200871e7..39721d71 100644 --- a/roles/matrix-dimension/templates/config.yaml.j2 +++ b/roles/matrix-dimension/templates/config.yaml.j2 @@ -13,7 +13,7 @@ homeserver: # The URL that Dimension, go-neb, and other services provisioned by Dimension should # use to access the homeserver with. - clientServerUrl: "http://matrix-synapse:8008" + clientServerUrl: "{{ matrix_homeserver_container_url }}" # The URL that Dimension should use when trying to communicate with federated APIs on # the homeserver. If not supplied or left empty Dimension will try to resolve the address From 70dcdd41a738f2450060b89f9c6f0d59b02727a7 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 25 Jan 2021 14:02:06 +0200 Subject: [PATCH 58/70] Simplify matrix-remove-all We don't have instantiated services anymore, nor /etc/systemd/system/matrix-synapse.service.wants/ stuff. --- .../templates/usr-local-bin/matrix-remove-all.j2 | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/roles/matrix-base/templates/usr-local-bin/matrix-remove-all.j2 b/roles/matrix-base/templates/usr-local-bin/matrix-remove-all.j2 index 01e0ac70..f4b23b44 100644 --- a/roles/matrix-base/templates/usr-local-bin/matrix-remove-all.j2 +++ b/roles/matrix-base/templates/usr-local-bin/matrix-remove-all.j2 @@ -16,20 +16,8 @@ if [ "$sure" != "Yes, I really want to remove everything!" ]; then else echo "Stop and remove matrix services" - # Look for and stop services, avoiding things like - # 'matrix-synapse-worker@.service' (just a template for instantiated services; can't stop it directly). - # We use '-xtype f' and not '-type f', because we wish to match symlinks like this: - # '/etc/systemd/system/matrix-synapse.service.wants/matrix-synapse-worker@generic_worker:18111.service' - # and stop these instantiated services as well. - for s in $(find {{ matrix_systemd_path }}/ -xtype f -name "matrix-*" -printf "%f\n" | grep -v '@.service' | uniq); do + for s in $(find {{ matrix_systemd_path }}/ -type f -name "matrix-*" -printf "%f\n"); do systemctl disable --now $s - done - - # Get rid of regular service files, as well as symlinks like - # '/etc/systemd/system/matrix-synapse.service.wants/matrix-synapse-worker@generic_worker:18111.service' - # and even - # '/etc/systemd/system/multi-user.target.wants/matrix-synapse.service'. - for s in $(find {{ matrix_systemd_path }}/ -xtype f -name "matrix-*" -printf "%p\n"); do rm -f {{ matrix_systemd_path }}/$s done From a535226210a0cbe100219cf69bb97db7c9941039 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 25 Jan 2021 15:20:37 +0200 Subject: [PATCH 59/70] Stop/disable unnecessary worker services before deleting them --- .../tasks/workers/setup_install.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/roles/matrix-synapse/tasks/workers/setup_install.yml b/roles/matrix-synapse/tasks/workers/setup_install.yml index ff34210a..c77bd737 100644 --- a/roles/matrix-synapse/tasks/workers/setup_install.yml +++ b/roles/matrix-synapse/tasks/workers/setup_install.yml @@ -1,26 +1,35 @@ --- -- name: Find worker configs to be cleaned +- name: Determine current worker configs find: path: "{{ matrix_synapse_config_dir_path }}" patterns: "worker.*.yaml" use_regex: true register: matrix_synapse_workers_current_config_files +# This also deletes some things which we need. They will be recreated below. - name: Ensure previous worker configs are cleaned file: path: "{{ item.path }}" state: absent with_items: "{{ matrix_synapse_workers_current_config_files.files }}" -- name: Find worker systemd services to be cleaned +- name: Determine current worker systemd services find: path: "{{ matrix_systemd_path }}" patterns: "matrix-synapse-worker.*.service" use_regex: true register: matrix_synapse_workers_current_systemd_services -- name: Ensure previous worker systemd services are cleaned +- name: Ensure unnecessary worker systemd services are stopped and disabled + service: + name: "{{ item.path|basename }}" + state: stopped + enabled: false + with_items: "{{ matrix_synapse_workers_current_systemd_services.files }}" + when: "not ansible_check_mode and item.path|basename not in matrix_systemd_services_list" + +- name: Ensure unnecessary worker systemd services are cleaned file: path: "{{ item.path }}" state: absent From e7f3f7c4318ccc6d2d2c5c4ad726b99eb3c63bf3 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 27 Jan 2021 22:18:47 +0200 Subject: [PATCH 60/70] Enable /devices endpoint for generic workers --- roles/matrix-synapse/vars/workers.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/matrix-synapse/vars/workers.yml b/roles/matrix-synapse/vars/workers.yml index 9dc79360..a3b50dc4 100644 --- a/roles/matrix-synapse/vars/workers.yml +++ b/roles/matrix-synapse/vars/workers.yml @@ -43,6 +43,7 @@ matrix_synapse_workers_generic_worker_endpoints: - ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/members$ - ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state$ - ^/_matrix/client/(api/v1|r0|unstable)/account/3pid$ + - ^/_matrix/client/(api/v1|r0|unstable)/devices$ - ^/_matrix/client/(api/v1|r0|unstable)/keys/query$ - ^/_matrix/client/(api/v1|r0|unstable)/keys/changes$ - ^/_matrix/client/versions$ From 453a4ec2d859dcb8563f1e637ee167a2387d0158 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 15 Feb 2021 10:42:20 +0200 Subject: [PATCH 61/70] Relocate tasks related to Synapse workers --- roles/matrix-synapse/tasks/init.yml | 2 +- roles/matrix-synapse/tasks/setup_synapse.yml | 2 +- roles/matrix-synapse/tasks/{ => synapse}/workers/setup.yml | 4 ++-- .../tasks/{ => synapse}/workers/setup_install.yml | 2 +- .../tasks/{ => synapse}/workers/setup_uninstall.yml | 0 .../workers/util/inject_systemd_services_for_worker.yml | 0 .../{ => synapse}/workers/util/setup_files_for_worker.yml | 0 7 files changed, 5 insertions(+), 5 deletions(-) rename roles/matrix-synapse/tasks/{ => synapse}/workers/setup.yml (81%) rename roles/matrix-synapse/tasks/{ => synapse}/workers/setup_install.yml (94%) rename roles/matrix-synapse/tasks/{ => synapse}/workers/setup_uninstall.yml (100%) rename roles/matrix-synapse/tasks/{ => synapse}/workers/util/inject_systemd_services_for_worker.yml (100%) rename roles/matrix-synapse/tasks/{ => synapse}/workers/util/setup_files_for_worker.yml (100%) diff --git a/roles/matrix-synapse/tasks/init.yml b/roles/matrix-synapse/tasks/init.yml index 46c7d22b..60eb3f17 100644 --- a/roles/matrix-synapse/tasks/init.yml +++ b/roles/matrix-synapse/tasks/init.yml @@ -3,7 +3,7 @@ when: matrix_synapse_enabled|bool - name: Ensure systemd services for workers are injected - include_tasks: "{{ role_path }}/tasks/workers/util/inject_systemd_services_for_worker.yml" + include_tasks: "{{ role_path }}/tasks/synapse/workers/util/inject_systemd_services_for_worker.yml" with_items: "{{ matrix_synapse_workers_enabled_list }}" loop_control: loop_var: matrix_synapse_worker_details diff --git a/roles/matrix-synapse/tasks/setup_synapse.yml b/roles/matrix-synapse/tasks/setup_synapse.yml index 68d9f5f6..f8bc05a1 100644 --- a/roles/matrix-synapse/tasks/setup_synapse.yml +++ b/roles/matrix-synapse/tasks/setup_synapse.yml @@ -18,7 +18,7 @@ - import_tasks: "{{ role_path }}/tasks/ext/setup.yml" -- import_tasks: "{{ role_path }}/tasks/workers/setup.yml" +- import_tasks: "{{ role_path }}/tasks/synapse/workers/setup.yml" - import_tasks: "{{ role_path }}/tasks/synapse/setup.yml" diff --git a/roles/matrix-synapse/tasks/workers/setup.yml b/roles/matrix-synapse/tasks/synapse/workers/setup.yml similarity index 81% rename from roles/matrix-synapse/tasks/workers/setup.yml rename to roles/matrix-synapse/tasks/synapse/workers/setup.yml index 3a7e6c98..ce66a2e4 100644 --- a/roles/matrix-synapse/tasks/workers/setup.yml +++ b/roles/matrix-synapse/tasks/synapse/workers/setup.yml @@ -14,8 +14,8 @@ path: "{{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid" state: absent -- include_tasks: "{{ role_path }}/tasks/workers/setup_install.yml" +- include_tasks: "{{ role_path }}/tasks/synapse/workers/setup_install.yml" when: "matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool" -- include_tasks: "{{ role_path }}/tasks/workers/setup_uninstall.yml" +- include_tasks: "{{ role_path }}/tasks/synapse/workers/setup_uninstall.yml" when: "not matrix_synapse_workers_enabled|bool" diff --git a/roles/matrix-synapse/tasks/workers/setup_install.yml b/roles/matrix-synapse/tasks/synapse/workers/setup_install.yml similarity index 94% rename from roles/matrix-synapse/tasks/workers/setup_install.yml rename to roles/matrix-synapse/tasks/synapse/workers/setup_install.yml index c77bd737..983f1876 100644 --- a/roles/matrix-synapse/tasks/workers/setup_install.yml +++ b/roles/matrix-synapse/tasks/synapse/workers/setup_install.yml @@ -36,7 +36,7 @@ with_items: "{{ matrix_synapse_workers_current_systemd_services.files }}" - name: Ensure creation of worker systemd service files and configuration files - include_tasks: "{{ role_path }}/tasks/workers/util/setup_files_for_worker.yml" + include_tasks: "{{ role_path }}/tasks/synapse/workers/util/setup_files_for_worker.yml" with_items: "{{ matrix_synapse_workers_enabled_list }}" loop_control: loop_var: matrix_synapse_worker_details diff --git a/roles/matrix-synapse/tasks/workers/setup_uninstall.yml b/roles/matrix-synapse/tasks/synapse/workers/setup_uninstall.yml similarity index 100% rename from roles/matrix-synapse/tasks/workers/setup_uninstall.yml rename to roles/matrix-synapse/tasks/synapse/workers/setup_uninstall.yml diff --git a/roles/matrix-synapse/tasks/workers/util/inject_systemd_services_for_worker.yml b/roles/matrix-synapse/tasks/synapse/workers/util/inject_systemd_services_for_worker.yml similarity index 100% rename from roles/matrix-synapse/tasks/workers/util/inject_systemd_services_for_worker.yml rename to roles/matrix-synapse/tasks/synapse/workers/util/inject_systemd_services_for_worker.yml diff --git a/roles/matrix-synapse/tasks/workers/util/setup_files_for_worker.yml b/roles/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml similarity index 100% rename from roles/matrix-synapse/tasks/workers/util/setup_files_for_worker.yml rename to roles/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml From 43059bb040aab5a8ed426689b86c84e0030d95d0 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 15 Feb 2021 11:19:07 +0200 Subject: [PATCH 62/70] Fix metrics listeners for Synapse workers `::` leads to errors like: > socket.gaierror: [Errno -9] Address family for hostname not supported --- roles/matrix-synapse/templates/synapse/worker.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 index 330086ad..36ae5a7e 100644 --- a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 @@ -32,7 +32,7 @@ worker_listeners: {% endif %} {% if matrix_synapse_metrics_enabled %} - type: metrics - bind_addresses: ['::'] + bind_addresses: ['0.0.0.0'] port: {{ matrix_synapse_worker_details.metrics_port }} {% endif %} {% endif %} From 85a05f38e873d71baf175effc5be1b1972ab1707 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 15 Feb 2021 11:25:35 +0200 Subject: [PATCH 63/70] Allow Synapse worker list to be generated dynamically This leads to much easier management and potential safety features (validation). In the future, we could try to avoid port conflicts as well, but it didn't seem worth the effort to do it now. Our port ranges seem large enough. This can also pave the way for a "presets" feature (similar to `matrix_nginx_proxy_ssl_presets`) which makes it even easier for people to configure worker counts. --- roles/matrix-synapse/defaults/main.yml | 71 +++++++++++----- roles/matrix-synapse/tasks/init.yml | 5 ++ .../tasks/synapse/workers/init.yml | 80 +++++++++++++++++++ .../matrix-synapse/tasks/validate_config.yml | 9 +++ 4 files changed, 147 insertions(+), 18 deletions(-) create mode 100644 roles/matrix-synapse/tasks/synapse/workers/init.yml diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index c25f79b0..058d4a3c 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -301,7 +301,6 @@ matrix_synapse_manhole_enabled: false # Enable support for Synapse workers matrix_synapse_workers_enabled: false - # Controls whether the matrix-synapse container exposes the various worker ports # (see `port` and `metrics_port` in `matrix_synapse_workers_enabled_list`) outside of the container. # @@ -309,23 +308,59 @@ matrix_synapse_workers_enabled: false # It takes "*" to signify "bind on all interfaces" ("0.0.0.0" is IPv4-only). matrix_synapse_workers_container_host_bind_address: '' -# Default list of workers to spawn (order in accord to docs) -# - no endpoints / doesn't need port mapping if port ends on 0 -# - single-instance-only if 2nd last digit of port number is 0 -matrix_synapse_workers_enabled_list: - - { type: generic_worker, port: 18111, metrics_port: 19111 } - - { type: generic_worker, port: 18112, metrics_port: 19112 } - - { type: generic_worker, port: 18113, metrics_port: 19113 } - - { type: generic_worker, port: 18114, metrics_port: 19114 } - - { type: generic_worker, port: 18115, metrics_port: 19115 } - - { type: generic_worker, port: 18116, metrics_port: 19116 } - - { type: pusher, port: 00, metrics_port: 19200 } - - { type: appservice, port: 00, metrics_port: 19300 } - - { type: federation_sender, port: 0, metrics_port: 19400 } - - { type: media_repository, port: 18551, metrics_port: 19551 } -# disable until https://github.com/matrix-org/synapse/issues/8787 resolved -# - { type: user_dir, port: 18661, metrics_port: 19661 } - - { type: frontend_proxy, port: 18771, metrics_port: 19771 } +matrix_synapse_workers_generic_workers_count: 3 +matrix_synapse_workers_generic_workers_port_range_start: 18111 +matrix_synapse_workers_generic_workers_metrics_range_start: 19111 + +# matrix_synapse_workers_pusher_workers_count can only be 0 or 1. More instances are not supported. +matrix_synapse_workers_pusher_workers_count: 1 +matrix_synapse_workers_pusher_workers_metrics_range_start: 19200 + +# matrix_synapse_workers_appservice_workers_count can only be 0 or 1. More instances are not supported. +matrix_synapse_workers_appservice_workers_count: 1 +matrix_synapse_workers_appservice_workers_metrics_range_start: 19300 + +matrix_synapse_workers_federation_sender_workers_count: 1 +matrix_synapse_workers_federation_sender_workers_metrics_range_start: 19400 + +matrix_synapse_workers_media_repository_workers_count: 1 +matrix_synapse_workers_media_repository_workers_port_range_start: 18551 +matrix_synapse_workers_media_repository_workers_metrics_range_start: 19551 + +# Disabled until https://github.com/matrix-org/synapse/issues/8787 is resolved. +matrix_synapse_workers_user_dir_workers_count: 0 +matrix_synapse_workers_user_dir_workers_port_range_start: 18661 +matrix_synapse_workers_user_dir_workers_metrics_range_start: 19661 + +matrix_synapse_workers_frontend_proxy_workers_count: 1 +matrix_synapse_workers_frontend_proxy_workers_port_range_start: 18771 +matrix_synapse_workers_frontend_proxy_workers_metrics_range_start: 19771 + +# Default list of workers to spawn. +# +# Unless you populate this manually, this list is dynamically generated +# based on other variables above: +# - `matrix_synapse_workers_*_workers_count` +# - `matrix_synapse_workers_*_workers_port_range_start` +# - `matrix_synapse_workers_*_workers_port_metrics_range_start` +# +# We advise that you use those variables and let this list be populated dynamically. +# Doing that is simpler and also protects you from shooting yourself in the foot, +# as certain workers can only be spawned just once. +# +# Example of what this needs to look like: +# matrix_synapse_workers_enabled_list: +# - { type: generic_worker, port: 18111, metrics_port: 19111 } +# - { type: generic_worker, port: 18112, metrics_port: 19112 } +# - { type: generic_worker, port: 18113, metrics_port: 19113 } +# - { type: generic_worker, port: 18114, metrics_port: 19114 } +# - { type: generic_worker, port: 18115, metrics_port: 19115 } +# - { type: generic_worker, port: 18116, metrics_port: 19116 } +# - { type: pusher, port: 0, metrics_port: 19200 } +# - { type: appservice, port: 0, metrics_port: 19300 } +# - { type: federation_sender, port: 0, metrics_port: 19400 } +# - { type: media_repository, port: 18551, metrics_port: 19551 } +matrix_synapse_workers_enabled_list: [] # Redis information matrix_synapse_redis_enabled: false diff --git a/roles/matrix-synapse/tasks/init.yml b/roles/matrix-synapse/tasks/init.yml index 60eb3f17..04b8d2b8 100644 --- a/roles/matrix-synapse/tasks/init.yml +++ b/roles/matrix-synapse/tasks/init.yml @@ -1,3 +1,8 @@ +# Unless `matrix_synapse_workers_enabled_list` is explicitly defined, +# we'll generate it dynamically. +- import_tasks: "{{ role_path }}/tasks/synapse/workers/init.yml" + when: "matrix_synapse_enabled and matrix_synapse_workers_enabled and matrix_synapse_workers_enabled_list|length == 0" + - set_fact: matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-synapse.service'] }}" when: matrix_synapse_enabled|bool diff --git a/roles/matrix-synapse/tasks/synapse/workers/init.yml b/roles/matrix-synapse/tasks/synapse/workers/init.yml new file mode 100644 index 00000000..a4c4f36f --- /dev/null +++ b/roles/matrix-synapse/tasks/synapse/workers/init.yml @@ -0,0 +1,80 @@ +# Below is a huge hack for dynamically building a list of workers and finally assigning it to `matrix_synapse_workers_enabled_list`. +# +# set_fact within a loop does not work reliably in Ansible (it only executes on the first iteration for some reason), +# so we're forced to do something much uglier. + +- name: Build generic workers + set_fact: + worker: + type: 'generic_worker' + port: "{{ matrix_synapse_workers_generic_workers_port_range_start + item }}" + metrics_port: "{{ matrix_synapse_workers_generic_workers_metrics_range_start + item }}" + register: "matrix_synapse_workers_list_results_generic_workers" + loop: "{{ range(0, matrix_synapse_workers_generic_workers_count)|list }}" + +- name: Build federation sender workers + set_fact: + worker: + type: 'federation_sender' + port: 0 + metrics_port: "{{ matrix_synapse_workers_federation_sender_workers_metrics_range_start + item }}" + register: "matrix_synapse_workers_list_results_federation_sender_workers" + loop: "{{ range(0, matrix_synapse_workers_federation_sender_workers_count)|list }}" + +# This type of worker can only have a count of 1, at most +- name: Build pusher workers + set_fact: + worker: + type: 'pusher' + port: 0 + metrics_port: "{{ matrix_synapse_workers_pusher_workers_metrics_range_start + item }}" + register: "matrix_synapse_workers_list_results_pusher_workers" + loop: "{{ range(0, matrix_synapse_workers_pusher_workers_count)|list }}" + +# This type of worker can only have a count of 1, at most +- name: Build appservice workers + set_fact: + worker: + type: 'appservice' + port: 0 + metrics_port: "{{ matrix_synapse_workers_appservice_workers_metrics_range_start + item }}" + register: "matrix_synapse_workers_list_results_appservice_workers" + loop: "{{ range(0, matrix_synapse_workers_appservice_workers_count)|list }}" + +- name: Build media_repository workers + set_fact: + worker: + type: 'media_repository' + port: "{{ matrix_synapse_workers_media_repository_workers_port_range_start + item }}" + metrics_port: "{{ matrix_synapse_workers_media_repository_workers_metrics_range_start + item }}" + register: "matrix_synapse_workers_list_results_media_repository_workers" + loop: "{{ range(0, matrix_synapse_workers_media_repository_workers_count)|list }}" + +- name: Build frontend_proxy workers + set_fact: + worker: + type: 'frontend_proxy' + port: "{{ matrix_synapse_workers_frontend_proxy_workers_port_range_start + item }}" + metrics_port: "{{ matrix_synapse_workers_frontend_proxy_workers_metrics_range_start + item }}" + register: "matrix_synapse_workers_list_results_frontend_proxy_workers" + loop: "{{ range(0, matrix_synapse_workers_frontend_proxy_workers_count)|list }}" + +- set_fact: + matrix_synapse_dynamic_workers_list: "{{ matrix_synapse_dynamic_workers_list|default([]) + [item.ansible_facts.worker] }}" + with_items: | + {{ + matrix_synapse_workers_list_results_generic_workers.results + + + matrix_synapse_workers_list_results_federation_sender_workers.results + + + matrix_synapse_workers_list_results_pusher_workers.results + + + matrix_synapse_workers_list_results_appservice_workers.results + + + matrix_synapse_workers_list_results_media_repository_workers.results + + + matrix_synapse_workers_list_results_frontend_proxy_workers.results + }} + +- set_fact: + matrix_synapse_workers_enabled_list: "{{ matrix_synapse_dynamic_workers_list }}" diff --git a/roles/matrix-synapse/tasks/validate_config.yml b/roles/matrix-synapse/tasks/validate_config.yml index b2c1f3a7..d05718d9 100644 --- a/roles/matrix-synapse/tasks/validate_config.yml +++ b/roles/matrix-synapse/tasks/validate_config.yml @@ -12,6 +12,15 @@ - "matrix_synapse_database_password" - "matrix_synapse_database_database" +- name: Fail if asking for more than 1 instance of single-instance workers + fail: + msg: >- + `{{ item }}` cannot be more than 1. This is a single-instance worker. + when: "vars[item] > 1" + with_items: + - "matrix_synapse_workers_appservice_workers_count" + - "matrix_synapse_workers_pusher_workers_count" + - name: (Deprecation) Catch and report renamed settings fail: msg: >- From 61e427d690c2ed9ebb28d6b2671e0415c3e2898f Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 15 Feb 2021 11:37:03 +0200 Subject: [PATCH 64/70] Do not let people enable more than 1 federation_sender worker --- roles/matrix-synapse/defaults/main.yml | 3 +++ roles/matrix-synapse/tasks/validate_config.yml | 1 + 2 files changed, 4 insertions(+) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 058d4a3c..31af47bc 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -320,6 +320,9 @@ matrix_synapse_workers_pusher_workers_metrics_range_start: 19200 matrix_synapse_workers_appservice_workers_count: 1 matrix_synapse_workers_appservice_workers_metrics_range_start: 19300 +# matrix_synapse_workers_federation_sender_workers_count can only be 0 or 1 for now. +# More instances are not supported due to a playbook limitation having to do with keeping `federation_sender_instances` in `homeserver.yaml` updated. +# See https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappfederation_sender matrix_synapse_workers_federation_sender_workers_count: 1 matrix_synapse_workers_federation_sender_workers_metrics_range_start: 19400 diff --git a/roles/matrix-synapse/tasks/validate_config.yml b/roles/matrix-synapse/tasks/validate_config.yml index d05718d9..283483b4 100644 --- a/roles/matrix-synapse/tasks/validate_config.yml +++ b/roles/matrix-synapse/tasks/validate_config.yml @@ -20,6 +20,7 @@ with_items: - "matrix_synapse_workers_appservice_workers_count" - "matrix_synapse_workers_pusher_workers_count" + - "matrix_synapse_workers_federation_sender_workers_count" - name: (Deprecation) Catch and report renamed settings fail: From d33483b8ce1221d2b457626d2fce8e8afcd93a2c Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 16 Feb 2021 17:45:41 +0200 Subject: [PATCH 65/70] Document that Synapse pusher worker instances are shardable Related to: - https://github.com/matrix-org/synapse/pull/9407 - https://github.com/matrix-org/synapse/pull/7855 --- roles/matrix-synapse/defaults/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index a60530c3..17f2556a 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -312,7 +312,9 @@ matrix_synapse_workers_generic_workers_count: 3 matrix_synapse_workers_generic_workers_port_range_start: 18111 matrix_synapse_workers_generic_workers_metrics_range_start: 19111 -# matrix_synapse_workers_pusher_workers_count can only be 0 or 1. More instances are not supported. +# matrix_synapse_workers_pusher_workers_count can only be 0 or 1 for now. +# More instances are not supported due to a playbook limitation having to do with keeping `pusher_instances` in `homeserver.yaml` updated. +# See https://github.com/matrix-org/synapse/commit/ddfdf945064925eba761ae3748e38f3a1c73c328 matrix_synapse_workers_pusher_workers_count: 1 matrix_synapse_workers_pusher_workers_metrics_range_start: 19200 From d6c4d41c2b9ff16967190e57cb1bad213711e6ab Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 18 Feb 2021 18:19:51 +0200 Subject: [PATCH 66/70] Define instanceId property on workers This give us the possibility to run multiple instances of workers that that don't expose a port. Right now, we don't support that, but in the future we could run multiple `federation_sender` or `pusher` workers, without them fighting over naming (previously, they'd all be named something like `matrix-synapse-worker-pusher-0`, because they'd all define `port` as `0`). --- .../nginx/conf.d/matrix-synapse.conf.j2 | 8 +++--- roles/matrix-synapse/defaults/main.yml | 28 +++++++++++-------- .../tasks/synapse/workers/init.yml | 6 ++++ .../inject_systemd_services_for_worker.yml | 14 +++++++++- .../workers/util/setup_files_for_worker.yml | 5 ++-- 5 files changed, 42 insertions(+), 19 deletions(-) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 6801f4f9..0b44678c 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -14,7 +14,7 @@ ip_hash; {% for worker in generic_workers %} - server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}"; + server "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.port }}"; {% endfor %} } {% endif %} @@ -22,7 +22,7 @@ {% if frontend_proxy_workers %} upstream frontend_proxy_upstream { {% for worker in frontend_proxy_workers %} - server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}"; + server "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.port }}"; {% endfor %} } {% endif %} @@ -30,7 +30,7 @@ {% if media_repository_workers %} upstream media_repository_upstream { {% for worker in media_repository_workers %} - server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}"; + server "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.port }}"; {% endfor %} } {% endif %} @@ -38,7 +38,7 @@ {% if user_dir_workers %} upstream user_dir_upstream { {% for worker in user_dir_workers %} - server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}"; + server "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.port }}"; {% endfor %} } {% endif %} diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 17f2556a..dc7f4347 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -353,18 +353,24 @@ matrix_synapse_workers_frontend_proxy_workers_metrics_range_start: 19771 # Doing that is simpler and also protects you from shooting yourself in the foot, # as certain workers can only be spawned just once. # -# Example of what this needs to look like: +# Each worker instance in the list defines the following fields: +# - `type` - the type of worker (`generic_worker`, etc.) +# - `instanceId` - a string that identifies the worker. The combination of (`type` + `instanceId`) represents the name of the worker and must be unique. +# - `port` - an HTTP port where the worker listens for requests (can be `0` for workers that don't do HTTP request processing) +# - `metrics_port` - an HTTP port where the worker exports Prometheus metrics +# +# Example of what this needs to look like, if you're defining it manually: # matrix_synapse_workers_enabled_list: -# - { type: generic_worker, port: 18111, metrics_port: 19111 } -# - { type: generic_worker, port: 18112, metrics_port: 19112 } -# - { type: generic_worker, port: 18113, metrics_port: 19113 } -# - { type: generic_worker, port: 18114, metrics_port: 19114 } -# - { type: generic_worker, port: 18115, metrics_port: 19115 } -# - { type: generic_worker, port: 18116, metrics_port: 19116 } -# - { type: pusher, port: 0, metrics_port: 19200 } -# - { type: appservice, port: 0, metrics_port: 19300 } -# - { type: federation_sender, port: 0, metrics_port: 19400 } -# - { type: media_repository, port: 18551, metrics_port: 19551 } +# - { type: generic_worker, instanceId: '18111', port: 18111, metrics_port: 19111 } +# - { type: generic_worker, instanceId: '18112', port: 18112, metrics_port: 19112 } +# - { type: generic_worker, instanceId: '18113', port: 18113, metrics_port: 19113 } +# - { type: generic_worker, instanceId: '18114', port: 18114, metrics_port: 19114 } +# - { type: generic_worker, instanceId: '18115', port: 18115, metrics_port: 19115 } +# - { type: generic_worker, instanceId: '18116', port: 18116, metrics_port: 19116 } +# - { type: pusher, instanceId: '0', port: 0, metrics_port: 19200 } +# - { type: appservice, instanceId: '0', port: 0, metrics_port: 19300 } +# - { type: federation_sender, instanceId: '0', port: 0, metrics_port: 19400 } +# - { type: media_repository, instanceId: '18551', port: 18551, metrics_port: 19551 } matrix_synapse_workers_enabled_list: [] # Redis information diff --git a/roles/matrix-synapse/tasks/synapse/workers/init.yml b/roles/matrix-synapse/tasks/synapse/workers/init.yml index a4c4f36f..204a0841 100644 --- a/roles/matrix-synapse/tasks/synapse/workers/init.yml +++ b/roles/matrix-synapse/tasks/synapse/workers/init.yml @@ -7,6 +7,7 @@ set_fact: worker: type: 'generic_worker' + instanceId: "{{ matrix_synapse_workers_generic_workers_port_range_start + item }}" port: "{{ matrix_synapse_workers_generic_workers_port_range_start + item }}" metrics_port: "{{ matrix_synapse_workers_generic_workers_metrics_range_start + item }}" register: "matrix_synapse_workers_list_results_generic_workers" @@ -16,6 +17,7 @@ set_fact: worker: type: 'federation_sender' + instanceId: "{{ item }}" port: 0 metrics_port: "{{ matrix_synapse_workers_federation_sender_workers_metrics_range_start + item }}" register: "matrix_synapse_workers_list_results_federation_sender_workers" @@ -26,6 +28,7 @@ set_fact: worker: type: 'pusher' + instanceId: "{{ item }}" port: 0 metrics_port: "{{ matrix_synapse_workers_pusher_workers_metrics_range_start + item }}" register: "matrix_synapse_workers_list_results_pusher_workers" @@ -36,6 +39,7 @@ set_fact: worker: type: 'appservice' + instanceId: "{{ item }}" port: 0 metrics_port: "{{ matrix_synapse_workers_appservice_workers_metrics_range_start + item }}" register: "matrix_synapse_workers_list_results_appservice_workers" @@ -45,6 +49,7 @@ set_fact: worker: type: 'media_repository' + instanceId: "{{ matrix_synapse_workers_media_repository_workers_port_range_start + item }}" port: "{{ matrix_synapse_workers_media_repository_workers_port_range_start + item }}" metrics_port: "{{ matrix_synapse_workers_media_repository_workers_metrics_range_start + item }}" register: "matrix_synapse_workers_list_results_media_repository_workers" @@ -54,6 +59,7 @@ set_fact: worker: type: 'frontend_proxy' + instanceId: "{{ matrix_synapse_workers_frontend_proxy_workers_port_range_start + item }}" port: "{{ matrix_synapse_workers_frontend_proxy_workers_port_range_start + item }}" metrics_port: "{{ matrix_synapse_workers_frontend_proxy_workers_metrics_range_start + item }}" register: "matrix_synapse_workers_list_results_frontend_proxy_workers" diff --git a/roles/matrix-synapse/tasks/synapse/workers/util/inject_systemd_services_for_worker.yml b/roles/matrix-synapse/tasks/synapse/workers/util/inject_systemd_services_for_worker.yml index c95f881a..62b42625 100644 --- a/roles/matrix-synapse/tasks/synapse/workers/util/inject_systemd_services_for_worker.yml +++ b/roles/matrix-synapse/tasks/synapse/workers/util/inject_systemd_services_for_worker.yml @@ -1,6 +1,18 @@ +# The tasks below run before `validate_config.yml`. +# To avoid failing with a cryptic error message, we'll do validation here. +# +# This check is mostly relevant to people who explicitly define `matrix_synapse_workers_enabled_list` +# (Synapse Workers users from the earlier days of this PR - https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/456). +# +# In the future, it should be possible to remove this check. +# Our own code which dynamically builds `matrix_synapse_workers_enabled_list` does things right. +- name: Fail if instanceId not defined for worker + fail: + msg: "Synapse workers (like {{ matrix_synapse_worker_details|to_json }}) need to define an instanceId property (type + instanceId must be unique)" + when: "'instanceId' not in matrix_synapse_worker_details" - set_fact: - matrix_synapse_worker_systemd_service_name: "matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.port }}.service" + matrix_synapse_worker_systemd_service_name: "matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.instanceId }}.service" - set_fact: matrix_systemd_services_list: "{{ matrix_systemd_services_list + [matrix_synapse_worker_systemd_service_name] }}" diff --git a/roles/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml b/roles/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml index 6a15e048..93ed6575 100644 --- a/roles/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml +++ b/roles/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml @@ -1,12 +1,11 @@ - - set_fact: - matrix_synapse_worker_systemd_service_name: "matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.port }}" + matrix_synapse_worker_systemd_service_name: "matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.instanceId }}" - set_fact: matrix_synapse_worker_container_name: "{{ matrix_synapse_worker_systemd_service_name }}" - set_fact: - matrix_synapse_worker_config_file_name: "worker.{{ matrix_synapse_worker_details.type }}_{{ matrix_synapse_worker_details.port }}.yaml" + matrix_synapse_worker_config_file_name: "worker.{{ matrix_synapse_worker_details.type }}_{{ matrix_synapse_worker_details.instanceId }}.yaml" - name: Ensure configuration exists for {{ matrix_synapse_worker_systemd_service_name }} template: From 2f732e4234cba5db64e590edf08c9162822ef87a Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 19 Feb 2021 11:36:14 +0200 Subject: [PATCH 67/70] Update Synapse worker endpoints --- roles/matrix-synapse/vars/workers.yml | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/roles/matrix-synapse/vars/workers.yml b/roles/matrix-synapse/vars/workers.yml index a3b50dc4..3adfd9c3 100644 --- a/roles/matrix-synapse/vars/workers.yml +++ b/roles/matrix-synapse/vars/workers.yml @@ -51,7 +51,6 @@ matrix_synapse_workers_generic_worker_endpoints: - ^/_matrix/client/(api/v1|r0|unstable)/joined_groups$ - ^/_matrix/client/(api/v1|r0|unstable)/publicised_groups$ - ^/_matrix/client/(api/v1|r0|unstable)/publicised_groups/ - - ^/_synapse/client/password_reset/email/submit_token$ # Registration/login requests - ^/_matrix/client/(api/v1|r0|unstable)/login$ @@ -86,28 +85,33 @@ matrix_synapse_workers_generic_worker_endpoints: # to use SSO (you only need to include the ones for whichever SSO provider you're # using): + # for all SSO providers + # FIXME: ADDITIONAL CONDITIONS REQUIRED: to be enabled manually + # ^/_matrix/client/(api/v1|r0|unstable)/login/sso/redirect + # ^/_synapse/client/pick_idp$ + # ^/_synapse/client/pick_username + # ^/_synapse/client/new_user_consent$ + # ^/_synapse/client/sso_register$ + # OpenID Connect requests. # FIXME: ADDITIONAL CONDITIONS REQUIRED: to be enabled manually - # ^/_matrix/client/(api/v1|r0|unstable)/login/sso/redirect$ - # ^/_synapse/oidc/callback$ + # ^/_synapse/client/oidc/callback$ # SAML requests. # FIXME: ADDITIONAL CONDITIONS REQUIRED: to be enabled manually - # ^/_matrix/client/(api/v1|r0|unstable)/login/sso/redirect$ - # ^/_matrix/saml2/authn_response$ + # ^/_synapse/client/saml2/authn_response$ # CAS requests. # FIXME: ADDITIONAL CONDITIONS REQUIRED: to be enabled manually - # ^/_matrix/client/(api/v1|r0|unstable)/login/(cas|sso)/redirect$ # ^/_matrix/client/(api/v1|r0|unstable)/login/cas/ticket$ - # Note that a HTTP listener with `client` and `federation` resources must be - # configured in the `worker_listeners` option in the worker config. - - # Ensure that all SSO logins go to a single process (usually the main process). + # Ensure that all SSO logins go to a single process. # For multiple workers not handling the SSO endpoints properly, see # [#7530](https://github.com/matrix-org/synapse/issues/7530). + # Note that a HTTP listener with `client` and `federation` resources must be + # configured in the `worker_listeners` option in the worker config. + # #### Load balancing # It is possible to run multiple instances of this worker app, with incoming requests From eaea2152820badabe3c057db61eb0077c3c9e572 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 19 Feb 2021 11:36:48 +0200 Subject: [PATCH 68/70] Allow Synapse workers to be used with an external nginx webserver We're talking about a webserver running on the same machine, which imports the configuration files generated by the `matrix-nginx-proxy` in the `/matrix/nginx-proxy/conf.d` directory. Users who run an nginx webserver on some other machine will need to do something different. --- .../nginx/conf.d/matrix-synapse.conf.j2 | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 0b44678c..7041468e 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -14,7 +14,11 @@ ip_hash; {% for worker in generic_workers %} - server "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.port }}"; + {% if matrix_nginx_proxy_enabled %} + server "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.port }}"; + {% else %} + server "127.0.0.1:{{ worker.port }}"; + {% endif %} {% endfor %} } {% endif %} @@ -22,7 +26,11 @@ {% if frontend_proxy_workers %} upstream frontend_proxy_upstream { {% for worker in frontend_proxy_workers %} - server "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.port }}"; + {% if matrix_nginx_proxy_enabled %} + server "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.port }}"; + {% else %} + server "127.0.0.1:{{ worker.port }}"; + {% endif %} {% endfor %} } {% endif %} @@ -30,7 +38,11 @@ {% if media_repository_workers %} upstream media_repository_upstream { {% for worker in media_repository_workers %} - server "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.port }}"; + {% if matrix_nginx_proxy_enabled %} + server "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.port }}"; + {% else %} + server "127.0.0.1:{{ worker.port }}"; + {% endif %} {% endfor %} } {% endif %} @@ -38,7 +50,11 @@ {% if user_dir_workers %} upstream user_dir_upstream { {% for worker in user_dir_workers %} - server "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.port }}"; + {% if matrix_nginx_proxy_enabled %} + server "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.port }}"; + {% else %} + server "127.0.0.1:{{ worker.port }}"; + {% endif %} {% endfor %} } {% endif %} From 9dc87bb948d57196158956306d6b58f491b7d27d Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 19 Feb 2021 11:38:47 +0200 Subject: [PATCH 69/70] Add Synapse worker presets for easier configuration Adding more presets in the future would be nice. --- docs/configuring-playbook-synapse.md | 22 +++++++--- roles/matrix-synapse/defaults/main.yml | 43 ++++++++++++++++--- .../tasks/synapse/workers/init.yml | 12 +++--- .../matrix-synapse/tasks/validate_config.yml | 2 +- 4 files changed, 60 insertions(+), 19 deletions(-) diff --git a/docs/configuring-playbook-synapse.md b/docs/configuring-playbook-synapse.md index 29afa70e..bdfdfa63 100644 --- a/docs/configuring-playbook-synapse.md +++ b/docs/configuring-playbook-synapse.md @@ -19,18 +19,30 @@ Alternatively, **if there is no pre-defined variable** for a Synapse setting you ## Load balancing with workers -To have synapse gracefully handle thousands of users, worker support should be enabled. It factors out some homeserver tasks and spreads the load of incoming client and server-to-server traffic between multiple processes. More information can be found at https://github.com/matrix-org/synapse/blob/master/docs/workers.md (which, coincidentally, also is the file which an awk script extracts the endpoint URLs from when running with tag `setup-synapse`). -To enable synapse worker support, set +To have Synapse gracefully handle thousands of users, worker support should be enabled. It factors out some homeserver tasks and spreads the load of incoming client and server-to-server traffic between multiple processes. More information can be found in the [official Synapse workers documentation](https://github.com/matrix-org/synapse/blob/master/docs/workers.md). + +To enable Synapse worker support, update your `inventory/host_vars/matrix.DOMAIN/vars.yml` file: ```yaml matrix_synapse_workers_enabled: true ``` -in your `inventory/host_vars/matrix.DOMAIN/vars.yml` file. -There, you can also override the default `matrix_synapse_workers_enabled_list` from [`roles/matrix-synapse/defaults/main.yml`](../roles/matrix-synapse/defaults/main.yml). +We support a few configuration presets (`matrix_synapse_workers_preset: one-of-each` being the default configuration): +- `little-federation-helper` - a very minimal worker configuration to improve federation performance +- `one-of-each` - one worker of each supported type + +If you'd like more customization power, you can start with one of the presets and tweak various `matrix_synapse_workers_*_count` variables manually. + +If you increase worker counts too much, you may need to increase the maximum number of Postgres connections too (example): + +```yaml +matrix_postgres_process_extra_arguments: [ + "-c 'max_connections=200'" +] +``` -If you are not using the inbuilt nginx proxy container but an instance managed by yourself, you are currently on your own as the template needs yet to be adapted to better support this use case. +If you're using the default setup (the `matrix-nginx-proxy` webserver being enabled) or you're using your own `nginx` server (which imports the configuration files generated by the playbook), you're good to go. If you use some other webserver, you may need to tweak your reverse-proxy setup manually to forward traffic to the various workers. In case any problems occur, make sure to have a look at the [list of synapse issues about workers](https://github.com/matrix-org/synapse/issues?q=workers+in%3Atitle) and your `journalctl --unit 'matrix-*'`. diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index dc7f4347..04678136 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -301,6 +301,35 @@ matrix_synapse_manhole_enabled: false # Enable support for Synapse workers matrix_synapse_workers_enabled: false +# Specifies worker configuration that should be used when workers are enabled. +# +# The posible values (as seen in `matrix_synapse_workers_presets`) are: +# - "little-federation-helper" - a very minimal worker configuration to improve federation performance +# - "one-of-each" - one worker of each supported type +# +# You can override `matrix_synapse_workers_presets` to define your own presets, which is ill-advised, because it's fragile. +# To use a more custom configuration, start with one of these presets as a base and configure `matrix_synapse_workers_*_count` variables manually, to suit your liking. +matrix_synapse_workers_preset: one-of-each + +matrix_synapse_workers_presets: + little-federation-helper: + generic_workers_count: 0 + pusher_workers_count: 0 + appservice_workers_count: 0 + federation_sender_workers_count: 1 + media_repository_workers_count: 0 + user_dir_workers_count: 0 + frontend_proxy_workers_count: 0 + one-of-each: + generic_workers_count: 1 + pusher_workers_count: 1 + appservice_workers_count: 1 + federation_sender_workers_count: 1 + media_repository_workers_count: 1 + # Disabled until https://github.com/matrix-org/synapse/issues/8787 is resolved. + user_dir_workers_count: 0 + frontend_proxy_workers_count: 1 + # Controls whether the matrix-synapse container exposes the various worker ports # (see `port` and `metrics_port` in `matrix_synapse_workers_enabled_list`) outside of the container. # @@ -308,36 +337,36 @@ matrix_synapse_workers_enabled: false # It takes "*" to signify "bind on all interfaces" ("0.0.0.0" is IPv4-only). matrix_synapse_workers_container_host_bind_address: '' -matrix_synapse_workers_generic_workers_count: 3 +matrix_synapse_workers_generic_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['generic_workers_count'] }}" matrix_synapse_workers_generic_workers_port_range_start: 18111 matrix_synapse_workers_generic_workers_metrics_range_start: 19111 # matrix_synapse_workers_pusher_workers_count can only be 0 or 1 for now. # More instances are not supported due to a playbook limitation having to do with keeping `pusher_instances` in `homeserver.yaml` updated. # See https://github.com/matrix-org/synapse/commit/ddfdf945064925eba761ae3748e38f3a1c73c328 -matrix_synapse_workers_pusher_workers_count: 1 +matrix_synapse_workers_pusher_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['pusher_workers_count'] }}" matrix_synapse_workers_pusher_workers_metrics_range_start: 19200 # matrix_synapse_workers_appservice_workers_count can only be 0 or 1. More instances are not supported. -matrix_synapse_workers_appservice_workers_count: 1 +matrix_synapse_workers_appservice_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['appservice_workers_count'] }}" matrix_synapse_workers_appservice_workers_metrics_range_start: 19300 # matrix_synapse_workers_federation_sender_workers_count can only be 0 or 1 for now. # More instances are not supported due to a playbook limitation having to do with keeping `federation_sender_instances` in `homeserver.yaml` updated. # See https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappfederation_sender -matrix_synapse_workers_federation_sender_workers_count: 1 +matrix_synapse_workers_federation_sender_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['federation_sender_workers_count'] }}" matrix_synapse_workers_federation_sender_workers_metrics_range_start: 19400 -matrix_synapse_workers_media_repository_workers_count: 1 +matrix_synapse_workers_media_repository_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['media_repository_workers_count'] }}" matrix_synapse_workers_media_repository_workers_port_range_start: 18551 matrix_synapse_workers_media_repository_workers_metrics_range_start: 19551 # Disabled until https://github.com/matrix-org/synapse/issues/8787 is resolved. -matrix_synapse_workers_user_dir_workers_count: 0 +matrix_synapse_workers_user_dir_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['user_dir_workers_count'] }}" matrix_synapse_workers_user_dir_workers_port_range_start: 18661 matrix_synapse_workers_user_dir_workers_metrics_range_start: 19661 -matrix_synapse_workers_frontend_proxy_workers_count: 1 +matrix_synapse_workers_frontend_proxy_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['frontend_proxy_workers_count'] }}" matrix_synapse_workers_frontend_proxy_workers_port_range_start: 18771 matrix_synapse_workers_frontend_proxy_workers_metrics_range_start: 19771 diff --git a/roles/matrix-synapse/tasks/synapse/workers/init.yml b/roles/matrix-synapse/tasks/synapse/workers/init.yml index 204a0841..c6fc32c3 100644 --- a/roles/matrix-synapse/tasks/synapse/workers/init.yml +++ b/roles/matrix-synapse/tasks/synapse/workers/init.yml @@ -11,7 +11,7 @@ port: "{{ matrix_synapse_workers_generic_workers_port_range_start + item }}" metrics_port: "{{ matrix_synapse_workers_generic_workers_metrics_range_start + item }}" register: "matrix_synapse_workers_list_results_generic_workers" - loop: "{{ range(0, matrix_synapse_workers_generic_workers_count)|list }}" + loop: "{{ range(0, matrix_synapse_workers_generic_workers_count|int)|list }}" - name: Build federation sender workers set_fact: @@ -21,7 +21,7 @@ port: 0 metrics_port: "{{ matrix_synapse_workers_federation_sender_workers_metrics_range_start + item }}" register: "matrix_synapse_workers_list_results_federation_sender_workers" - loop: "{{ range(0, matrix_synapse_workers_federation_sender_workers_count)|list }}" + loop: "{{ range(0, matrix_synapse_workers_federation_sender_workers_count|int)|list }}" # This type of worker can only have a count of 1, at most - name: Build pusher workers @@ -32,7 +32,7 @@ port: 0 metrics_port: "{{ matrix_synapse_workers_pusher_workers_metrics_range_start + item }}" register: "matrix_synapse_workers_list_results_pusher_workers" - loop: "{{ range(0, matrix_synapse_workers_pusher_workers_count)|list }}" + loop: "{{ range(0, matrix_synapse_workers_pusher_workers_count|int)|list }}" # This type of worker can only have a count of 1, at most - name: Build appservice workers @@ -43,7 +43,7 @@ port: 0 metrics_port: "{{ matrix_synapse_workers_appservice_workers_metrics_range_start + item }}" register: "matrix_synapse_workers_list_results_appservice_workers" - loop: "{{ range(0, matrix_synapse_workers_appservice_workers_count)|list }}" + loop: "{{ range(0, matrix_synapse_workers_appservice_workers_count|int)|list }}" - name: Build media_repository workers set_fact: @@ -53,7 +53,7 @@ port: "{{ matrix_synapse_workers_media_repository_workers_port_range_start + item }}" metrics_port: "{{ matrix_synapse_workers_media_repository_workers_metrics_range_start + item }}" register: "matrix_synapse_workers_list_results_media_repository_workers" - loop: "{{ range(0, matrix_synapse_workers_media_repository_workers_count)|list }}" + loop: "{{ range(0, matrix_synapse_workers_media_repository_workers_count|int)|list }}" - name: Build frontend_proxy workers set_fact: @@ -63,7 +63,7 @@ port: "{{ matrix_synapse_workers_frontend_proxy_workers_port_range_start + item }}" metrics_port: "{{ matrix_synapse_workers_frontend_proxy_workers_metrics_range_start + item }}" register: "matrix_synapse_workers_list_results_frontend_proxy_workers" - loop: "{{ range(0, matrix_synapse_workers_frontend_proxy_workers_count)|list }}" + loop: "{{ range(0, matrix_synapse_workers_frontend_proxy_workers_count|int)|list }}" - set_fact: matrix_synapse_dynamic_workers_list: "{{ matrix_synapse_dynamic_workers_list|default([]) + [item.ansible_facts.worker] }}" diff --git a/roles/matrix-synapse/tasks/validate_config.yml b/roles/matrix-synapse/tasks/validate_config.yml index 283483b4..f7631111 100644 --- a/roles/matrix-synapse/tasks/validate_config.yml +++ b/roles/matrix-synapse/tasks/validate_config.yml @@ -16,7 +16,7 @@ fail: msg: >- `{{ item }}` cannot be more than 1. This is a single-instance worker. - when: "vars[item] > 1" + when: "vars[item]|int > 1" with_items: - "matrix_synapse_workers_appservice_workers_count" - "matrix_synapse_workers_pusher_workers_count" From b754c2778b38698002b575d0c6e68a0c17495dd3 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 19 Feb 2021 11:39:58 +0200 Subject: [PATCH 70/70] Announce Synapse workers support --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54031268..3138a96a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,18 @@ +# 2021-02-19 + +## Synapse workers support + +After [lots and lots of work](https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/456) (done over many months by [Marcel Partap](https://github.com/eMPee584), [Max Klenk](https://github.com/maxklenk), a few others from the [Technical University of Dresden, Germany](https://tu-dresden.de/) and various other contributors), support for Synapse workers has finally landed. + +Having support for workers makes the playbook suitable for larger homeserver deployments. + +Our setup is not yet perfect (we don't support all types of workers; scaling some of them (like `pusher`, `federation_sender`) beyond a single instance is not yet supported). Still, it's a great start and can already power homeservers with thousands of users, like the [Matrix deployment at TU Dresden](https://doc.matrix.tu-dresden.de/en/) discussed in [Matrix Live S06E09 - TU Dresden on their Matrix deployment](https://www.youtube.com/watch?v=UHJX2pmT2gk). + +By default, workers are disabled and Synapse runs as a single process (homeservers don't necessarily need the complexity and increased memory requirements of running a worker-based setup). + +To enable Synapse workers, follow our [Load balancing with workers](docs/configuring-playbook-synapse.md#load-balancing-with-workers) documentation. + + # 2021-02-12 ## (Potential Breaking Change) Monitoring/metrics support using Prometheus and Grafana