synapse workers: reworkings + get endpoints from upstream docs via awk

(yes, a bit awkward and brittle… xD)
development
Marcel Partap 4 years ago
parent 87bd64ce9e
commit 2d1b9f2dbf

@ -799,6 +799,15 @@ matrix_nginx_proxy_proxy_matrix_user_directory_search_addr_sans_container: "{{ m
matrix_nginx_proxy_self_check_validate_certificates: "{{ false if matrix_ssl_retrieval_method == 'self-signed' else true }}"
matrix_nginx_proxy_synapse_presence_disabled: "{{ not matrix_synapse_use_presence }}"
matrix_nginx_proxy_synapse_workers_enabled: "{{ matrix_synapse_workers_enabled }}"
matrix_nginx_proxy_synapse_workers_list: "{{ matrix_synapse_workers_enabled_list }}"
matrix_nginx_proxy_synapse_generic_worker_locations: "{{ matrix_synapse_workers_generic_worker_endpoints }}"
matrix_nginx_proxy_synapse_media_repository_locations: "{{ matrix_synapse_workers_media_repository_endpoints }}"
matrix_nginx_proxy_synapse_user_dir_locations: "{{ matrix_synapse_workers_user_dir_endpoints }}"
matrix_nginx_proxy_synapse_frontend_proxy_locations: "{{ matrix_synapse_workers_frontend_proxy_endpoints }}"
matrix_nginx_proxy_systemd_wanted_services_list: |
{{
(['matrix-synapse.service'])
@ -1017,7 +1026,7 @@ matrix_synapse_systemd_wanted_services_list: |
(['matrix-mailer.service'] if matrix_mailer_enabled else [])
}}
# Worker support with redis
# Synapse workers (used for parallel load-scaling) need Redis for IPC.
matrix_synapse_redis_enabled: "{{ matrix_redis_enabled }}"
matrix_synapse_redis_host: "{{ 'matrix-redis' if matrix_redis_enabled else '' }}"
matrix_synapse_redis_password: "{{ matrix_redis_connection_password if matrix_redis_enabled else '' }}"

@ -249,79 +249,10 @@ matrix_nginx_proxy_proxy_matrix_nginx_status_enabled: false
matrix_nginx_proxy_proxy_matrix_nginx_status_allowed_addresses: ['{{ ansible_default_ipv4.address }}']
# worker
matrix_nginx_proxy_synapse_workers_enabled: "{{ matrix_synapse_workers_enabled }}"
matrix_nginx_proxy_synapse_workers_enabled_list: "{{ matrix_synapse_workers_enabled_list }}"
matrix_nginx_proxy_synapse_generic_worker_locations: [
# Sync requests
'^/_matrix/client/(v2_alpha|r0)/sync$',
'^/_matrix/client/(api/v1|v2_alpha|r0)/events$',
'^/_matrix/client/(api/v1|r0)/initialSync$',
'^/_matrix/client/(api/v1|r0)/rooms/[^/]+/initialSync$',
# Federation requests
'^/_matrix/federation/v1/event/',
'^/_matrix/federation/v1/state/',
'^/_matrix/federation/v1/state_ids/',
'^/_matrix/federation/v1/backfill/',
'^/_matrix/federation/v1/get_missing_events/',
'^/_matrix/federation/v1/publicRooms',
'^/_matrix/federation/v1/query/',
'^/_matrix/federation/v1/make_join/',
'^/_matrix/federation/v1/make_leave/',
'^/_matrix/federation/v1/send_join/',
'^/_matrix/federation/v2/send_join/',
'^/_matrix/federation/v1/send_leave/',
'^/_matrix/federation/v2/send_leave/',
'^/_matrix/federation/v1/invite/',
'^/_matrix/federation/v2/invite/',
'^/_matrix/federation/v1/query_auth/',
'^/_matrix/federation/v1/event_auth/',
'^/_matrix/federation/v1/exchange_third_party_invite/',
'^/_matrix/federation/v1/user/devices/',
'^/_matrix/federation/v1/get_groups_publicised$',
'^/_matrix/key/v2/query',
# Inbound federation transaction request
'^/_matrix/federation/v1/send/',
# Client API requests
'^/_matrix/client/(api/v1|r0|unstable)/publicRooms$',
'^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/joined_members$',
'^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/context/.*$',
'^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/members$',
'^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state$',
'^/_matrix/client/(api/v1|r0|unstable)/account/3pid$',
'^/_matrix/client/(api/v1|r0|unstable)/keys/query$',
'^/_matrix/client/(api/v1|r0|unstable)/keys/changes$',
'^/_matrix/client/versions$',
'^/_matrix/client/(api/v1|r0|unstable)/voip/turnServer$',
'^/_matrix/client/(api/v1|r0|unstable)/joined_groups$',
'^/_matrix/client/(api/v1|r0|unstable)/publicised_groups$',
'^/_matrix/client/(api/v1|r0|unstable)/publicised_groups/',
# Registration/login requests
'^/_matrix/client/(api/v1|r0|unstable)/login$',
'^/_matrix/client/(r0|unstable)/register$',
'^/_matrix/client/(r0|unstable)/auth/.*/fallback/web$',
# Event sending requests
'^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send',
'^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state/',
'^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$',
'^/_matrix/client/(api/v1|r0|unstable)/join/',
'^/_matrix/client/(api/v1|r0|unstable)/profile/',
]
matrix_nginx_proxy_synapse_media_repository_locations: [
'^/_matrix/media/',
'^/_synapse/admin/v1/purge_media_cache$',
'^/_synapse/admin/v1/room/.*/media.*$',
'^/_synapse/admin/v1/user/.*/media.*$',
'^/_synapse/admin/v1/media/.*$',
'^/_synapse/admin/v1/quarantine_media/.*$',
]
matrix_nginx_proxy_synapse_user_dir_locations: [
'^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$',
]
# synapse worker activation and endpoint mappings
matrix_nginx_proxy_synapse_workers_enabled: false
matrix_nginx_proxy_synapse_workers_list: []
matrix_nginx_proxy_synapse_generic_worker_locations: []
matrix_nginx_proxy_synapse_media_repository_locations: []
matrix_nginx_proxy_synapse_user_dir_locations: []
matrix_nginx_proxy_synapse_frontend_proxy_locations: []

@ -68,6 +68,7 @@
{% endif %}
{% if matrix_nginx_proxy_proxy_matrix_user_directory_search_enabled %}
# FIXME: if this is enabled, user_dir_workers should be disabled
location /_matrix/client/r0/user_directory/search {
{% if matrix_nginx_proxy_enabled %}
{# Use the embedded DNS resolver in Docker containers to discover the service #}
@ -102,10 +103,10 @@
{% endif %}
{% if matrix_nginx_proxy_synapse_workers_enabled %}
{# Synapse Workers #}
# Workers redirects BEGIN
{% if generic_worker_workers %}
{# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappgeneric_worker #}
{% if generic_workers %}
# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappgeneric_worker
{% for location in matrix_nginx_proxy_synapse_generic_worker_locations %}
location ~ {{ location }} {
proxy_pass http://generic_worker_upstream$request_uri;
@ -113,11 +114,11 @@
proxy_set_header X-Forwarded-For $remote_addr;
}
{% endfor %}
{# ToDo: add GET ^/_matrix/federation/v1/groups/ #}
# FIXME: add GET ^/_matrix/federation/v1/groups/
{% endif %}
{% if media_repository_workers %}
{# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappmedia_repository #}
# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappmedia_repository
{% for location in matrix_nginx_proxy_synapse_media_repository_locations %}
location ~ {{ location }} {
proxy_pass http://media_repository_upstream$request_uri;
@ -128,7 +129,8 @@
{% endif %}
{% if user_dir_workers %}
{# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappuser_dir #}
# FIXME: obsolete if matrix_nginx_proxy_proxy_matrix_user_directory_search_enabled is set
# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappuser_dir
{% for location in matrix_nginx_proxy_synapse_user_dir_locations %}
location ~ {{ location }} {
proxy_pass http://user_dir_upstream$request_uri;
@ -139,13 +141,16 @@
{% endif %}
{% if frontend_proxy_workers %}
{# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappfrontend_proxy #}
location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload {
# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappfrontend_proxy
{% for location in matrix_nginx_proxy_synapse_frontend_proxy_locations %}
location ~ {{ location }} {
proxy_pass http://frontend_proxy_upstream$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
{% if not matrix_synapse_use_presence %}
{% endfor %}
{% if matrix_nginx_proxy_synapse_presence_disabled %}
# FIXME: keep in sync with synapse workers documentation manually
location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status {
proxy_pass http://frontend_proxy_upstream$request_uri;
proxy_set_header Host $host;
@ -153,6 +158,7 @@
}
{% endif %}
{% endif %}
# Workers redirects END
{% endif %}
@ -229,20 +235,20 @@
}
{% endmacro %}
{% set generic_worker_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'generic_worker')|list %}
{% set media_repository_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|list %}
{% set user_dir_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|list %}
{% set frontend_proxy_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'frontend_proxy')|list %}
{% set generic_workers = matrix_nginx_proxy_synapse_workers_list|selectattr('type', 'equalto', 'generic_worker')|list %}
{% set media_repository_workers = matrix_nginx_proxy_synapse_workers_list|selectattr('type', 'equalto', 'media_repository')|list %}
{% set user_dir_workers = matrix_nginx_proxy_synapse_workers_list|selectattr('type', 'equalto', 'user_dir')|list %}
{% set frontend_proxy_workers = matrix_nginx_proxy_synapse_workers_list|selectattr('type', 'equalto', 'frontend_proxy')|list %}
{% if matrix_nginx_proxy_synapse_workers_enabled %}
{# Setup upstream for groups of workers #}
# Round Robin "upstream" pools for workers
{% if generic_worker_workers %}
{% if generic_workers %}
upstream generic_worker_upstream {
# ensures that requests from the same client will always be passed
# to the same server (except when this server is unavailable)
ip_hash;
{% for worker in generic_worker_workers %}
{% for worker in generic_workers %}
server "matrix-synapse:{{ worker.port }}";
{% endfor %}
}
@ -250,7 +256,6 @@
{% if frontend_proxy_workers %}
upstream frontend_proxy_upstream {
# Round Robin
{% for worker in frontend_proxy_workers %}
server "matrix-synapse:{{ worker.port }}";
{% endfor %}
@ -259,7 +264,6 @@
{% if media_repository_workers %}
upstream media_repository_upstream {
# Round Robin
{% for worker in media_repository_workers %}
server "matrix-synapse:{{ worker.port }}";
{% endfor %}
@ -268,7 +272,6 @@
{% if user_dir_workers %}
upstream user_dir_upstream {
# Round Robin
{% for worker in user_dir_workers %}
server "matrix-synapse:{{ worker.port }}";
{% endfor %}
@ -358,8 +361,8 @@ server {
{% endif %}
{% if matrix_nginx_proxy_synapse_workers_enabled %}
{% if generic_worker_workers %}
{# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappgeneric_worker #}
{% if generic_workers %}
# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappgeneric_worker
{% for location in matrix_nginx_proxy_synapse_generic_worker_locations %}
location ~ {{ location }} {
proxy_pass http://generic_worker_upstream$request_uri;
@ -367,7 +370,7 @@ server {
proxy_set_header X-Forwarded-For $remote_addr;
}
{% endfor %}
{# ToDo: add GET ^/_matrix/federation/v1/groups/ #}
# FIXME: add GET ^/_matrix/federation/v1/groups/
{% endif %}
{% endif %}

@ -275,31 +275,22 @@ matrix_synapse_manhole_enabled: false
# Enable support for Synapse workers
matrix_synapse_workers_enabled: false
# Default list of workers to spawn
# (worker with ports ending on 00 are single-instance)
# Default list of workers to spawn (order in accord to docs)
# - no endpoints / doesn't need port mapping if port ends on 0
# - single-instance-only if 2nd last digit of port number is 0
matrix_synapse_workers_enabled_list:
- { worker: generic_worker, port: 18101 }
- { worker: generic_worker, port: 18102 }
- { worker: generic_worker, port: 18103 }
- { worker: generic_worker, port: 18104 }
- { worker: generic_worker, port: 18105 }
- { worker: generic_worker, port: 18106 }
- { worker: appservice, port: 18200 }
- { worker: federation_sender, port: 18301 }
- { worker: frontend_proxy, port: 18400 }
- { worker: media_repository, port: 18501 }
- { worker: pusher, port: 18600 }
- { worker: user_dir, port: 18700 }
# The list of available workers (2020-08-28)
matrix_synapse_workers_avail_list:
- generic_worker
- appservice
- federation_sender
- frontend_proxy
- media_repository
- pusher
- user_dir
- { type: generic_worker, port: 18111 }
- { type: generic_worker, port: 18112 }
- { type: generic_worker, port: 18113 }
- { type: generic_worker, port: 18114 }
- { type: generic_worker, port: 18115 }
- { type: generic_worker, port: 18116 }
- { type: pusher, port: 00 }
- { type: appservice, port: 00 }
- { type: federation_sender, port: 0 }
- { type: media_repository, port: 18221 }
- { type: user_dir, port: 18331 }
- { type: frontend_proxy, port: 18441 }
# Redis information
matrix_synapse_redis_enabled: false

@ -0,0 +1,137 @@
#!/usr/bin/awk
# Hackish approach to get a machine-readable list of current matrix
# synapse REST API endpoints from the official documentation at
# https://github.com/matrix-org/synapse/raw/master/docs/workers.md
#
# invoke in shell with:
# URL=https://github.com/matrix-org/synapse/raw/master/docs/workers.md
# curl -L ${URL} | awk -f parse-workers-docs.awk -
function worker_stanza_append(string) {
worker_stanza = worker_stanza string
}
function line_is_endpoint_url(line) {
# probably API endpoint if it starts with white-space and ^ or /
return (line ~ /^ +[\^/].*\//)
}
# Put YAML marker at beginning of file.
BEGIN {
print "---"
endpoint_conditional_comment = " # FIXME: ADDITIONAL CONDITIONS REQUIRED: to be enabled manually\n"
}
# Enable further processing after the introductory text.
# Read each synapse worker section as record and its lines as fields.
/Available worker applications/ {
enable_parsing = 1
# set record separator to markdown section header
RS = "\n### "
# set field separator to newline
FS = "\n"
}
# Once parsing is active, this will process each section as record.
enable_parsing {
# Each worker section starts with a synapse.app.X headline
if ($1 ~ /synapse\.app\./) {
# get rid of the backticks and extract worker type from headline
gsub("`", "", $1)
gsub("synapse.app.", "", $1)
worker_type = $1
# initialize empty worker stanza
worker_stanza = ""
# track if any endpoints are mentioned in a specific section
worker_has_urls = 0
# some endpoint descriptions contain flag terms
endpoints_seem_conditional = 0
# also, collect a list of available workers
workers = (workers ? workers "\n" : "") " - " worker_type
# loop through the lines (2 - number of fields in record)
for (i = 1; i < NF + 1; i++) {
# copy line for gsub replacements
line = $i
# end all lines but the last with a linefeed
linefeed = (i < NF - 1) ? "\n" : ""
# line starts with white-space and a hash: endpoint block headline
if (line ~ /^ +#/) {
# copy to output verbatim, normalizing white-space
gsub(/^ +/, "", line)
worker_stanza_append(" " line linefeed)
} else if (line_is_endpoint_url(line)) {
# mark section for special output formatting
worker_has_urls = 1
# remove leading white-space
gsub(/^ +/, "", line)
api_endpoint_regex = line
# FIXME: https://github.com/matrix-org/synapse/issues/new
# munge inconsistent media_repository endpoint notation
if (api_endpoint_regex == "/_matrix/media/") {
api_endpoint_regex = "^" line
}
# disable endpoints which specify complications
if (endpoints_seem_conditional) {
# only add notice if previous line didn't match
if (!line_is_endpoint_url($(i - 1))) {
worker_stanza_append(endpoint_conditional_comment)
}
worker_stanza_append(" # " api_endpoint_regex linefeed)
} else {
# output endpoint regex
worker_stanza_append(" - " api_endpoint_regex linefeed)
}
# white-space only line?
} else if (line ~ /^\w*$/) {
if (i > 3 && i < NF) {
# print white-space lines unless 1st or last line in section
worker_stanza_append(line linefeed)
}
# nothing of the above: the line is regular documentation text
} else {
# include this text line as comment
worker_stanza_append(" # " line linefeed)
# and take note of words hinting at additional conditions to be met
if (line ~ /\<[Ii]f\>|\<[Ff]or\>/) {
endpoints_seem_conditional = 1
}
}
}
if (worker_has_urls) {
print "\nmatrix_synapse_workers_" worker_type "_endpoints:"
print worker_stanza
} else {
# include workers without endpoints as well for reference
print "\n# " worker_type " worker (no API endpoints) ["
print worker_stanza
print "# ]"
}
}
}
END {
print "\nmatrix_synapse_workers_avail_list:"
print workers | "sort"
}
# vim: tabstop=4 shiftwidth=4 expandtab autoindent

@ -1,5 +1,18 @@
---
- name: Download synapse workers doc
get_url:
url: https://github.com/matrix-org/synapse/raw/master/docs/workers.md
dest: "{{ role_path }}/files/workers.upstream-documentation.md"
- name: Download synapse workers doc and convert into YAML
shell:
cmd: "awk -f {{ role_path }}/files/workers-doc-to-yaml.awk -- {{ role_path }}/files/workers.upstream-documentation.md > {{ role_path }}/vars/workers.yml"
creates: "{{ role_path }}/vars/workers.yml"
- name: Load list of available worker apps and endpoints
include_vars: "{{ role_path }}/vars/workers.yml"
- name: Ensure synapse worker base service file installed
template:
src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker@.service.j2"
@ -23,7 +36,7 @@
- name: Ensure individual worker service symlinks exist
service:
name: "matrix-synapse-worker@{{ item.worker }}:{{ item.port }}.service"
name: "matrix-synapse-worker@{{ item.type }}:{{ item.port }}.service"
enabled: true
with_items: "{{ matrix_synapse_workers_enabled_list }}"
@ -37,14 +50,14 @@
- name: Ensure creation of specific worker configs
template:
src: "{{ role_path }}/templates/synapse/worker.yaml.j2"
dest: "{{ matrix_synapse_config_dir_path }}/worker.{{ item.worker }}:{{ item.port }}.yaml"
dest: "{{ matrix_synapse_config_dir_path }}/worker.{{ item.type }}:{{ item.port }}.yaml"
with_list: "{{ matrix_synapse_workers_enabled_list }}"
- name: Add workers to synapse.wants list
set_fact:
matrix_synapse_systemd_wanted_services_list: >
{{ matrix_synapse_systemd_wanted_services_list +
['matrix-synapse-worker@' + item.worker + ':' + item.port|string + '.service'] }}
['matrix-synapse-worker@' + item.type + ':' + item.port|string + '.service'] }}
with_items: "{{ matrix_synapse_workers_enabled_list }}"
- name: Ensure matrix-synapse-worker-write-pid script is created

@ -249,19 +249,19 @@ worker_app: synapse.app.homeserver
# thx https://oznetnerd.com/2017/04/18/jinja2-selectattr-filter/
# reduce the main worker's offerings to core homeserver business
{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'appservice')|list %}
{% if matrix_synapse_workers_enabled_list|selectattr('type', 'equalto', 'appservice')|list %}
notify_appservices: false
{% endif %}
{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_sender')|list %}
{% if matrix_synapse_workers_enabled_list|selectattr('type', 'equalto', 'federation_sender')|list %}
send_federation: false
{% endif %}
{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|list %}
{% if matrix_synapse_workers_enabled_list|selectattr('type', 'equalto', 'media_repository')|list %}
enable_media_repo: false
{% endif %}
{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'pusher')|list %}
{% if matrix_synapse_workers_enabled_list|selectattr('type', 'equalto', 'pusher')|list %}
start_pushers: false
{% endif %}
{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|list %}
{% if matrix_synapse_workers_enabled_list|selectattr('type', 'equalto', 'user_dir')|list %}
update_user_directory: false
{% endif %}

@ -43,9 +43,12 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \
{% if matrix_synapse_manhole_enabled and matrix_synapse_container_manhole_api_host_bind_port %}
-p {{ matrix_synapse_container_manhole_api_host_bind_port }}:9000 \
{% endif %}
{% if matrix_synapse_workers_enabled %}
{% if matrix_synapse_workers_enabled and not matrix_nginx_proxy_enabled|default(False) %}
{# Expose worker (by default 18xxx range) ports on host if not using internal nginx proxy #}
{% for worker in matrix_synapse_workers_enabled_list %}
{% if worker.port != 0 %}
-p {{ worker.port }}:{{ worker.port }} \
{% endif %}
{% endfor %}
{% endif %}
-v {{ matrix_synapse_config_dir_path }}:/data:ro \

@ -1,27 +1,27 @@
#jinja2: lstrip_blocks: "True"
worker_app: synapse.app.{{ item.worker }}
worker_name: {{ item.worker ~ ':' ~ item.port }}
worker_app: synapse.app.{{ item.type }}
worker_name: {{ item.type ~ ':' ~ item.port }}
worker_replication_host: 127.0.0.1
worker_replication_http_port: {{ matrix_synapse_replication_http_port }}
{% if item.worker not in [ 'appservice', 'federation_sender', 'pusher' ] %}
{% if item.type not in [ 'appservice', 'federation_sender', 'pusher' ] %}
worker_listeners:
- type: http
port: {{ item.port }}
resources:
- names:
{% if item.worker in [ 'generic_worker', 'frontend_proxy', 'user_dir' ] %}
{% if item.type in [ 'generic_worker', 'frontend_proxy', 'user_dir' ] %}
- client
{% endif %}
{% if item.worker in [ 'generic_worker' ] %}
{% if item.type in [ 'generic_worker' ] %}
- federation
{% elif item.worker in [ 'media_repository' ] %}
{% elif item.type in [ 'media_repository' ] %}
- media
{% endif %}
{% endif %}
{% if item.worker == 'frontend_proxy' %}
{% if item.type == 'frontend_proxy' %}
worker_main_http_uri: http://127.0.0.1:8008
{% endif %}

Loading…
Cancel
Save