From ee088d5d46c76f732eef6592116ad4c844ba8203 Mon Sep 17 00:00:00 2001 From: Ahmad Haghighi Date: Tue, 25 May 2021 13:14:28 +0430 Subject: [PATCH 01/22] Synapse LDAP auth: add support for Active Directory Signed-off-by: Ahmad Haghighi --- roles/matrix-synapse/defaults/main.yml | 2 ++ roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index d966665f..89f0320a 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -497,6 +497,8 @@ matrix_synapse_ext_password_provider_ldap_attributes_name: "cn" matrix_synapse_ext_password_provider_ldap_bind_dn: "" matrix_synapse_ext_password_provider_ldap_bind_password: "" matrix_synapse_ext_password_provider_ldap_filter: "" +matrix_synapse_ext_password_provider_ldap_is_active_directory: false +matrix_synapse_ext_password_provider_ldap_default_domain: "" # Enable this to activate the Synapse Antispam spam-checker module. # See: https://github.com/t2bot/synapse-simple-antispam diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index 22f5f1ea..66730d36 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -2596,6 +2596,12 @@ password_providers: uri: {{ matrix_synapse_ext_password_provider_ldap_uri|string|to_json }} start_tls: {{ matrix_synapse_ext_password_provider_ldap_start_tls|to_json }} base: {{ matrix_synapse_ext_password_provider_ldap_base|string|to_json }} + {% if matrix_synapse_ext_password_provider_ldap_is_active_directory %} + active_directory: true + {% endif %} + {% if matrix_synapse_ext_password_provider_ldap_default_domain != '' %} + default_domain: {{ matrix_synapse_ext_password_provider_ldap_default_domain|string|to_json }} + {% endif %} attributes: uid: {{ matrix_synapse_ext_password_provider_ldap_attributes_uid|string|to_json }} mail: {{ matrix_synapse_ext_password_provider_ldap_attributes_mail|string|to_json }} From 209d59070e7ba6147c34242515418554028e787a Mon Sep 17 00:00:00 2001 From: Ahmad Haghighi Date: Tue, 25 May 2021 15:41:38 +0430 Subject: [PATCH 02/22] Avoiding if(s), fix #1083 Conversation: https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/1083#discussion_r638671860 Signed-off-by: Ahmad Haghighi --- roles/matrix-synapse/defaults/main.yml | 2 +- roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 89f0320a..e112df5c 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -497,7 +497,7 @@ matrix_synapse_ext_password_provider_ldap_attributes_name: "cn" matrix_synapse_ext_password_provider_ldap_bind_dn: "" matrix_synapse_ext_password_provider_ldap_bind_password: "" matrix_synapse_ext_password_provider_ldap_filter: "" -matrix_synapse_ext_password_provider_ldap_is_active_directory: false +matrix_synapse_ext_password_provider_ldap_active_directory: false matrix_synapse_ext_password_provider_ldap_default_domain: "" # Enable this to activate the Synapse Antispam spam-checker module. diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index 66730d36..aa07ae5b 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -2596,12 +2596,8 @@ password_providers: uri: {{ matrix_synapse_ext_password_provider_ldap_uri|string|to_json }} start_tls: {{ matrix_synapse_ext_password_provider_ldap_start_tls|to_json }} base: {{ matrix_synapse_ext_password_provider_ldap_base|string|to_json }} - {% if matrix_synapse_ext_password_provider_ldap_is_active_directory %} - active_directory: true - {% endif %} - {% if matrix_synapse_ext_password_provider_ldap_default_domain != '' %} + active_directory: {{ matrix_synapse_ext_password_provider_ldap_active_directory|to_json }} default_domain: {{ matrix_synapse_ext_password_provider_ldap_default_domain|string|to_json }} - {% endif %} attributes: uid: {{ matrix_synapse_ext_password_provider_ldap_attributes_uid|string|to_json }} mail: {{ matrix_synapse_ext_password_provider_ldap_attributes_mail|string|to_json }} From 36910348cfd2987657c0bfeb050521b4b78b536b Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 25 May 2021 16:21:11 +0300 Subject: [PATCH 03/22] Switch synapse-admin to tagged release (0.8.1) Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/1084 --- roles/matrix-synapse-admin/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse-admin/defaults/main.yml b/roles/matrix-synapse-admin/defaults/main.yml index dc4cc770..069b6279 100644 --- a/roles/matrix-synapse-admin/defaults/main.yml +++ b/roles/matrix-synapse-admin/defaults/main.yml @@ -8,7 +8,7 @@ matrix_synapse_admin_container_self_build_repo: "https://github.com/Awesome-Tech matrix_synapse_admin_docker_src_files_path: "{{ matrix_base_data_path }}/synapse-admin/docker-src" -matrix_synapse_admin_version: latest +matrix_synapse_admin_version: 0.8.1 matrix_synapse_admin_docker_image: "{{ matrix_synapse_admin_docker_image_name_prefix }}awesometechnologies/synapse-admin:{{ matrix_synapse_admin_version }}" matrix_synapse_admin_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_admin_container_self_build else matrix_container_global_registry_prefix }}" matrix_synapse_admin_docker_image_force_pull: "{{ matrix_synapse_admin_docker_image.endswith(':latest') }}" From 763952395b6eff3f70cd08fd3572f9eeeeac5ea6 Mon Sep 17 00:00:00 2001 From: BG Date: Tue, 25 May 2021 16:42:04 +0200 Subject: [PATCH 04/22] Adding mjolnir antispam synapse modul. --- roles/matrix-synapse/defaults/main.yml | 21 ++++++++ .../tasks/ext/mjolnir-antispam/setup.yml | 7 +++ .../ext/mjolnir-antispam/setup_install.yml | 52 +++++++++++++++++++ .../ext/mjolnir-antispam/setup_uninstall.yml | 6 +++ roles/matrix-synapse/tasks/ext/setup.yml | 2 + 5 files changed, 88 insertions(+) create mode 100644 roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup.yml create mode 100644 roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup_install.yml create mode 100644 roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup_uninstall.yml diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index d966665f..c7f0a9f7 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -505,6 +505,27 @@ matrix_synapse_ext_spam_checker_synapse_simple_antispam_git_repository_url: "htt matrix_synapse_ext_spam_checker_synapse_simple_antispam_git_version: "923ca5c85b08f157181721abbae50dd89c31e4b5" matrix_synapse_ext_spam_checker_synapse_simple_antispam_config_blocked_homeservers: [] +# Enable this to activate the Mjolnir Antispam spam-checker module. +# See: https://github.com/matrix-org/mjolnir#synapse-module +matrix_synapse_ext_spam_checker_mjolnir_antispam_enabled: false +matrix_synapse_ext_spam_checker_mjolnir_antispam_git_repository_url: "https://github.com/matrix-org/mjolnir" +matrix_synapse_ext_spam_checker_mjolnir_antispam_git_version: "70f353fbbad0af469b1001080dea194d512b2815" +matrix_synapse_ext_spam_checker_mjolnir_antispam_config_block_invites: true +# Flag messages sent by servers/users in the ban lists as spam. Currently +# this means that spammy messages will appear as empty to users. Default +# false. +matrix_synapse_ext_spam_checker_mjolnir_antispam_config_block_messages: false +# Remove users from the user directory search by filtering matrix IDs and +# display names by the entries in the user ban list. Default false. +matrix_synapse_ext_spam_checker_mjolnir_antispam_config_block_usernames: false +# The room IDs of the ban lists to honour. Unlike other parts of Mjolnir, +# this list cannot be room aliases or permalinks. This server is expected +# to already be joined to the room - Mjolnir will not automatically join +# these rooms. +# ["!roomid:example.org"] +matrix_synapse_ext_spam_checker_mjolnir_antispam_config_ban_lists: [] + + matrix_s3_media_store_enabled: false matrix_s3_media_store_custom_endpoint_enabled: false matrix_s3_goofys_docker_image: "ewoutp/goofys:latest" diff --git a/roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup.yml b/roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup.yml new file mode 100644 index 00000000..6c45f469 --- /dev/null +++ b/roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup.yml @@ -0,0 +1,7 @@ +--- + +- import_tasks: "{{ role_path }}/tasks/ext/mjolnir-antispam/setup_install.yml" + when: matrix_synapse_ext_spam_checker_mjolnir_antispam_enabled|bool + +- import_tasks: "{{ role_path }}/tasks/ext/mjolnir-antispam/setup_uninstall.yml" + when: "not matrix_synapse_ext_spam_checker_mjolnir_antispam_enabled|bool" diff --git a/roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup_install.yml b/roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup_install.yml new file mode 100644 index 00000000..a416e42b --- /dev/null +++ b/roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup_install.yml @@ -0,0 +1,52 @@ +--- + +- name: Ensure git installed (RedHat) + yum: + name: + - git + state: present + update_cache: no + when: "ansible_os_family == 'RedHat'" + +- name: Ensure git installed (Debian) + apt: + name: + - git + state: present + update_cache: no + when: "ansible_os_family == 'Debian'" + +- name: Ensure git installed (Archlinux) + pacman: + name: + - git + state: present + update_cache: no + when: "ansible_distribution == 'Archlinux'" + +- name: Clone mjolnir-antispam git repository + git: + repo: "{{ matrix_synapse_ext_spam_checker_mjolnir_antispam_git_repository_url }}" + version: "{{ matrix_synapse_ext_spam_checker_mjolnir_antispam_git_version }}" + dest: "{{ matrix_synapse_ext_path }}/mjolnir" + become: true + become_user: "{{ matrix_user_username }}" + +- set_fact: + matrix_synapse_spam_checker: > + {{ matrix_synapse_spam_checker }} + + + [{ + "module": "mjolnir.AntiSpam", + "config": { + "block_invites": {{ matrix_synapse_ext_spam_checker_mjolnir_antispam_config_block_invites }}, + "block_messages": {{ matrix_synapse_ext_spam_checker_mjolnir_antispam_config_block_messages }}, + "block_usernames": {{ matrix_synapse_ext_spam_checker_mjolnir_antispam_config_block_usernames }}, + "ban_lists": {{ matrix_synapse_ext_spam_checker_mjolnir_antispam_config_ban_lists }} + } + }] + + matrix_synapse_container_extra_arguments: > + {{ matrix_synapse_container_extra_arguments|default([]) }} + + + ["--mount type=bind,src={{ matrix_synapse_ext_path }}/mjolnir/synapse_antispam/mjolnir,dst={{ matrix_synapse_in_container_python_packages_path }}/mjolnir,ro"] diff --git a/roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup_uninstall.yml b/roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup_uninstall.yml new file mode 100644 index 00000000..d3c52e5f --- /dev/null +++ b/roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup_uninstall.yml @@ -0,0 +1,6 @@ +--- + +- name: Ensure mjolnir-antispam doesn't exist + file: + path: "{{ matrix_synapse_ext_path }}/mjolnir-antispam" + state: absent diff --git a/roles/matrix-synapse/tasks/ext/setup.yml b/roles/matrix-synapse/tasks/ext/setup.yml index 5e06ea9d..31637fa9 100644 --- a/roles/matrix-synapse/tasks/ext/setup.yml +++ b/roles/matrix-synapse/tasks/ext/setup.yml @@ -7,3 +7,5 @@ - import_tasks: "{{ role_path }}/tasks/ext/ldap-auth/setup.yml" - import_tasks: "{{ role_path }}/tasks/ext/synapse-simple-antispam/setup.yml" + +- import_tasks: "{{ role_path }}/tasks/ext/mjolnir-antispam/setup.yml" From 9a7eddd563e68b370432d7666fd53247f1356211 Mon Sep 17 00:00:00 2001 From: BG Date: Tue, 25 May 2021 16:59:49 +0200 Subject: [PATCH 05/22] Updating docs. --- docs/configuring-playbook-bot-mjolnir.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/configuring-playbook-bot-mjolnir.md b/docs/configuring-playbook-bot-mjolnir.md index 1b0241e1..6dc4e3a6 100644 --- a/docs/configuring-playbook-bot-mjolnir.md +++ b/docs/configuring-playbook-bot-mjolnir.md @@ -90,8 +90,21 @@ matrix_bot_mjolnir_access_token: "ACCESS_TOKEN_FROM_STEP_2_GOES_HERE" matrix_bot_mjolnir_management_room: "ROOM_ID_FROM_STEP_4_GOES_HERE" ``` +## 6. Adding mjolnir synapse antispam module (optional) -## 6. Installing +Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars.yml` file (adapt to your needs): + + +```yaml +matrix_synapse_ext_spam_checker_mjolnir_antispam_enabled: true +matrix_synapse_ext_spam_checker_mjolnir_antispam_config_block_invites: true +matrix_synapse_ext_spam_checker_mjolnir_antispam_config_block_messages: false +matrix_synapse_ext_spam_checker_mjolnir_antispam_config_block_usernames: false +matrix_synapse_ext_spam_checker_mjolnir_antispam_config_ban_lists: [] +``` + + +## 7. Installing After configuring the playbook, run the [installation](installing.md) command: From 4ddd8bbb84844905d9ea0f43f267a40c52893642 Mon Sep 17 00:00:00 2001 From: rakshazi Date: Tue, 25 May 2021 17:06:39 +0000 Subject: [PATCH 06/22] Updated nginx-proxy (1.20.0 -> 1.21.0) --- roles/matrix-nginx-proxy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 8dc8a259..3e79ab04 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -1,5 +1,5 @@ matrix_nginx_proxy_enabled: true -matrix_nginx_proxy_version: 1.20.0-alpine +matrix_nginx_proxy_version: 1.21.0-alpine # We use an official nginx image, which we fix-up to run unprivileged. # An alternative would be an `nginxinc/nginx-unprivileged` image, but From 719ecb03a80863dc4086a2e28983943d27da3c9f Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 26 May 2021 09:55:31 +0300 Subject: [PATCH 07/22] Add note about pgtune.leopard.in.ua --- docs/maintenance-postgres.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/maintenance-postgres.md b/docs/maintenance-postgres.md index 9b748d28..e2307f78 100644 --- a/docs/maintenance-postgres.md +++ b/docs/maintenance-postgres.md @@ -99,6 +99,8 @@ Example: `--extra-vars="postgres_dump_name=matrix-postgres-dump.sql"` PostgreSQL can be tuned to make it run faster. This is done by passing extra arguments to Postgres with the `matrix_postgres_process_extra_arguments` variable. You should use a website like https://pgtune.leopard.in.ua/ or information from https://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server to determine what Postgres settings you should change. +**Note**: the configuration generator at https://pgtune.leopard.in.ua/ adds spaces around the `=` sign, which is invalid. You'll need to remove it manually (`max_connections = 300` -> `max_connections=300`) + ### Here are some examples: These are not recommended values and they may not work well for you. This is just to give you an idea of some of the options that can be set. If you are an experienced PostgreSQL admin feel free to update this documentation with better examples. From f8ba97ac14362c40d4a40a9e6cd20159e7024672 Mon Sep 17 00:00:00 2001 From: BG Date: Wed, 26 May 2021 09:05:28 +0200 Subject: [PATCH 08/22] Removing antispam note. --- docs/configuring-playbook-bot-mjolnir.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/configuring-playbook-bot-mjolnir.md b/docs/configuring-playbook-bot-mjolnir.md index 6dc4e3a6..5ddb2ad3 100644 --- a/docs/configuring-playbook-bot-mjolnir.md +++ b/docs/configuring-playbook-bot-mjolnir.md @@ -4,8 +4,6 @@ The playbook can install and configure the [Mjolnir](https://github.com/matrix-o See the project's [documentation](https://github.com/matrix-org/mjolnir) to learn what it does and why it might be useful to you. -Note: the playbook does not currently support the Mjolnir Synapse module. The playbook does support another antispam module, see [Setting up Synapse Simple Antispam](configuring-playbook-synapse-simple-antispam.md). - ## 1. Register the bot account From d61fe94bae2bbbd98220f6aed4a38996fd265574 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 26 May 2021 10:52:15 +0300 Subject: [PATCH 09/22] Fix incorrect path in Mjolnir uninstallation tasks --- .../tasks/ext/mjolnir-antispam/setup_uninstall.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup_uninstall.yml b/roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup_uninstall.yml index d3c52e5f..f8439a87 100644 --- a/roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup_uninstall.yml +++ b/roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup_uninstall.yml @@ -2,5 +2,5 @@ - name: Ensure mjolnir-antispam doesn't exist file: - path: "{{ matrix_synapse_ext_path }}/mjolnir-antispam" + path: "{{ matrix_synapse_ext_path }}/mjolnir" state: absent From af4bd50c2a7fe2ceab9e30842870b185ac665a1e Mon Sep 17 00:00:00 2001 From: Thom Wiggers Date: Wed, 26 May 2021 11:04:47 +0200 Subject: [PATCH 10/22] Update IRC appservice --- roles/matrix-bridge-appservice-irc/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-appservice-irc/defaults/main.yml b/roles/matrix-bridge-appservice-irc/defaults/main.yml index 410299b7..bdf49f65 100644 --- a/roles/matrix-bridge-appservice-irc/defaults/main.yml +++ b/roles/matrix-bridge-appservice-irc/defaults/main.yml @@ -7,7 +7,7 @@ matrix_appservice_irc_container_self_build: false matrix_appservice_irc_docker_repo: "https://github.com/matrix-org/matrix-appservice-irc.git" matrix_appservice_irc_docker_src_files_path: "{{ matrix_base_data_path }}/appservice-irc/docker-src" -matrix_appservice_irc_version: release-0.25.0 +matrix_appservice_irc_version: release-0.26.0 matrix_appservice_irc_docker_image: "{{ matrix_container_global_registry_prefix }}matrixdotorg/matrix-appservice-irc:{{ matrix_appservice_irc_version }}" matrix_appservice_irc_docker_image_force_pull: "{{ matrix_appservice_irc_docker_image.endswith(':latest') }}" From 1b41e9c7dd07ebf3696d344c7c352ef95569a087 Mon Sep 17 00:00:00 2001 From: Wobbel The Bear Date: Wed, 26 May 2021 13:50:35 +0200 Subject: [PATCH 11/22] Update PostgreSQL Maintenance page Added a mid-sized VPS configuration with configuration changes to the PostgreSQL database config. Deleted single quotes in one of the examples to unify the examples --- docs/maintenance-postgres.md | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/docs/maintenance-postgres.md b/docs/maintenance-postgres.md index e2307f78..52d2d9ee 100644 --- a/docs/maintenance-postgres.md +++ b/docs/maintenance-postgres.md @@ -108,11 +108,33 @@ These are not recommended values and they may not work well for you. This is jus Here is an example config for a small 2 core server with 4GB of RAM and SSD storage: ``` matrix_postgres_process_extra_arguments: [ - "-c 'shared_buffers=128MB'", - "-c 'effective_cache_size=2304MB'", - "-c 'effective_io_concurrency=100'", - "-c 'random_page_cost=2.0'", - "-c 'min_wal_size=500MB'", + "-c shared_buffers=128MB", + "-c effective_cache_size=2304MB", + "-c effective_io_concurrency=100", + "-c random_page_cost=2.0", + "-c min_wal_size=500MB", +] +``` + +Here is an example config for a 4 core server with 8GB of RAM on a Virtual Private Server (VPS); the paramters have been configured using https://pgtune.leopard.in.ua with the following setup: PostgreSQL version 12, OS Type: Linux, DB Type: Mixed type of application, Data Storage: SSD storage: +``` +matrix_postgres_process_extra_arguments: [ + "-c max_connections=100", + "-c shared_buffers=2GB", + "-c effective_cache_size=6GB", + "-c maintenance_work_mem=512MB", + "-c checkpoint_completion_target=0.9", + "-c wal_buffers=16MB", + "-c default_statistics_target=100", + "-c random_page_cost=1.1", + "-c effective_io_concurrency=200", + "-c work_mem=5242kB", + "-c min_wal_size=1GB", + "-c max_wal_size=4GB", + "-c max_worker_processes=4", + "-c max_parallel_workers_per_gather=2", + "-c max_parallel_workers=4", + "-c max_parallel_maintenance_workers=2", ] ``` From 8472848286483f2ba19e8b94ff1eb24febb8d61d Mon Sep 17 00:00:00 2001 From: WobbelTheBear Date: Wed, 26 May 2021 16:30:08 +0200 Subject: [PATCH 12/22] Annotate server migration Annotate server migration with a note in regard to migration between different architectures and the need to dump/restore the database. --- docs/maintenance-migrating.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/maintenance-migrating.md b/docs/maintenance-migrating.md index d6380b99..1d12ef45 100644 --- a/docs/maintenance-migrating.md +++ b/docs/maintenance-migrating.md @@ -1,3 +1,8 @@ +> **Note**: This migration guide is applicable if you migrate from server to another that are of the same architecture, i.e. e.g. the servers both are amd64. +> +> In case that you migrate between two different architectures (e.g. amd64 --> arm64), simply copying the complete folder is not possible as it would move the raw PostgreSQL data between different architectures. In this specific case, you would need to dump the database on your current server and import it properly on the new server. +> + # Migrating to new server 1. Prepare by lowering DNS TTL for your domains (`matrix.DOMAIN`, etc.), so that DNS record changes (step 4 below) would happen faster, leading to less downtime From 54e1c67e53c2ddc12444c6a215f72dd1966dc526 Mon Sep 17 00:00:00 2001 From: WobbelTheBear Date: Wed, 26 May 2021 16:31:01 +0200 Subject: [PATCH 13/22] Update maintenance-migrating.md --- docs/maintenance-migrating.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/maintenance-migrating.md b/docs/maintenance-migrating.md index 1d12ef45..56cdea79 100644 --- a/docs/maintenance-migrating.md +++ b/docs/maintenance-migrating.md @@ -1,4 +1,4 @@ -> **Note**: This migration guide is applicable if you migrate from server to another that are of the same architecture, i.e. e.g. the servers both are amd64. +> **Note**: This migration guide is applicable if you migrate from server to another that are of the same cpu architecture, e.g. the servers both are amd64. > > In case that you migrate between two different architectures (e.g. amd64 --> arm64), simply copying the complete folder is not possible as it would move the raw PostgreSQL data between different architectures. In this specific case, you would need to dump the database on your current server and import it properly on the new server. > From c791d1900ccb91d9ae7bcfccbeba2bd75bc84c23 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 26 May 2021 18:03:47 +0300 Subject: [PATCH 14/22] Improve wording --- docs/maintenance-migrating.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/maintenance-migrating.md b/docs/maintenance-migrating.md index 56cdea79..fd593691 100644 --- a/docs/maintenance-migrating.md +++ b/docs/maintenance-migrating.md @@ -1,7 +1,6 @@ -> **Note**: This migration guide is applicable if you migrate from server to another that are of the same cpu architecture, e.g. the servers both are amd64. -> -> In case that you migrate between two different architectures (e.g. amd64 --> arm64), simply copying the complete folder is not possible as it would move the raw PostgreSQL data between different architectures. In this specific case, you would need to dump the database on your current server and import it properly on the new server. +> **Note**: This migration guide is applicable if you migrate from one server to another server having the same CPU architecture (e.g. both servers being `amd64`). > +> If you're trying to migrate between different architectures (e.g. `amd64` --> `arm64`), simply copying the complete `/matrix` directory is not possible as it would move the raw PostgreSQL data between different architectures. In this specific case, you can use the guide below as a reference, but you would also need to dump the database on your current server and import it properly on the new server. See our [Backing up PostgreSQL](maintenance-postgres.md#backing-up-postgresql) docs for help with PostgreSQL backup/restore. # Migrating to new server From a8dbd93f61eb227fb41ac4426a256918fe0c035d Mon Sep 17 00:00:00 2001 From: Jez Cope Date: Wed, 26 May 2021 20:41:52 +0100 Subject: [PATCH 15/22] Check irc service is present before stopping it --- .../tasks/migrate_nedb_to_postgres.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml b/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml index 3fab195a..6b39ac62 100644 --- a/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml +++ b/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml @@ -26,10 +26,16 @@ become: false when: "matrix_postgres_service_start_result.changed|bool" +- name: Check existence of matrix-appservice-irc service + stat: + path: "{{ matrix_systemd_path }}/matrix-appservice-irc.service" + register: matrix_appservice_irc_service_stat + - name: Ensure matrix-appservice-irc is stopped service: name: matrix-appservice-irc state: stopped + when: "matrix_appservice_irc_service_stat.stat.exists" - name: Import appservice-irc NeDB database into Postgres command: From 4e2780ff8858198704f73ca4bbd4a0325376bb2a Mon Sep 17 00:00:00 2001 From: Raymond Coetzee Date: Thu, 27 May 2021 20:13:29 +0100 Subject: [PATCH 16/22] Add support for a prometheus postgres exporter This commit introduces a new role that downloads and installs the prometheus community postgres exporter https://github.com/prometheus-community/postgres_exporter. A new credential is added to matrix_postgres_additional_databases that allows the exporter access to the database to gather statistics. A new dashboard was added to the grafana role, with some refactoring to enable the dashboard only if the new role is enabled. I've included some basic instructions for how to enable the role in the Docs section. In terms of testing, I've tested enabling the role, and disabling it to make sure it cleans up the container and systemd role. --- ...onfiguring-playbook-prometheus-postgres.md | 29 ++++++++++ group_vars/matrix_servers | 40 ++++++++++++++ roles/matrix-grafana/tasks/setup.yml | 2 +- .../defaults/main.yml | 49 +++++++++++++++++ .../tasks/init.yml | 5 ++ .../tasks/main.yml | 8 +++ .../tasks/setup.yml | 54 +++++++++++++++++++ ...ix-prometheus-postgres-exporter.service.j2 | 42 +++++++++++++++ .../templates/prometheus.yml.j2 | 6 +++ setup.yml | 4 +- 10 files changed, 236 insertions(+), 3 deletions(-) create mode 100644 docs/configuring-playbook-prometheus-postgres.md create mode 100644 roles/matrix-prometheus-postgres-exporter/defaults/main.yml create mode 100644 roles/matrix-prometheus-postgres-exporter/tasks/init.yml create mode 100644 roles/matrix-prometheus-postgres-exporter/tasks/main.yml create mode 100644 roles/matrix-prometheus-postgres-exporter/tasks/setup.yml create mode 100644 roles/matrix-prometheus-postgres-exporter/templates/systemd/matrix-prometheus-postgres-exporter.service.j2 diff --git a/docs/configuring-playbook-prometheus-postgres.md b/docs/configuring-playbook-prometheus-postgres.md new file mode 100644 index 00000000..9a631296 --- /dev/null +++ b/docs/configuring-playbook-prometheus-postgres.md @@ -0,0 +1,29 @@ +# Enabling metrics and graphs for Postgres (optional) + +Expanding on the metrics exposed by the [syapse exporter and the node exporter](configuring-playbook-prometheus-grafana.md), the playbook enables the [postgres exporter](https://github.com/prometheus-community/postgres_exporter) that exposes more detailed information about what's happening on your postgres database. + +You can enable this with the following settings in your configuration file (`inventory/host_vars/matrix./vars.yml`): + + +```yaml +matrix_prometheus_postgres_exporter_enabled: true + +# the role creates a postgres user as credential. You can configure these if required: +matrix_prometheus_postgres_exporter_database_username: 'matrix_prometheus_postgres_exporter' +matrix_prometheus_postgres_exporter_database_password: 'some-password' + +``` + +## What does it do? + +Name | Description +-----|---------- +`matrix_prometheus_postgres_exporter_enabled`|Enable the postgres prometheus exporter. This sets up the docker container, connects it to the database and adds a 'job' to the prometheus config which tells prometheus about this new exporter. The default is 'false' +`matrix_prometheus_postgres_exporter_database_username`| The 'username' for the user that the exporter uses to connect to the database. The default is 'matrix_prometheus_postgres_exporter' +`matrix_prometheus_postgres_exporter_database_password`| The 'username' for the user that the exporter uses to connect to the database. + + +## More information + +- [The PostgresSQL dashboard](https://grafana.com/grafana/dashboards/9628) (generic postgres dashboard) + diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 86f1d5a3..96b933f3 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1466,6 +1466,13 @@ matrix_postgres_additional_databases: | 'username': matrix_sygnal_database_username, 'password': matrix_sygnal_database_password, }] if (matrix_sygnal_enabled and matrix_sygnal_database_engine == 'postgres' and matrix_sygnal_database_hostname == 'matrix-postgres') else []) + + + ([{ + 'name': matrix_prometheus_postgres_exporter_database_name, + 'username': matrix_prometheus_postgres_exporter_database_username, + 'password': matrix_prometheus_postgres_exporter_database_password, + }] if (matrix_prometheus_postgres_exporter_enabled and matrix_prometheus_postgres_exporter_database_hostname == 'matrix-postgres') else []) + }} matrix_postgres_import_roles_to_ignore: | @@ -1766,6 +1773,10 @@ matrix_prometheus_scraper_synapse_rules_synapse_tag: "{{ matrix_synapse_docker_i matrix_prometheus_scraper_node_enabled: "{{ matrix_prometheus_node_exporter_enabled }}" matrix_prometheus_scraper_node_targets: "{{ ['matrix-prometheus-node-exporter:9100'] if matrix_prometheus_node_exporter_enabled else [] }}" +matrix_prometheus_scraper_postgres_enabled: "{{ matrix_prometheus_postgres_exporter_enabled }}" +matrix_prometheus_scraper_postgres_targets: "{{ ['matrix-prometheus-postgres-exporter:'+ matrix_prometheus_postgres_exporter_port|string] if matrix_prometheus_scraper_postgres_enabled else [] }}" + + ###################################################################### # # /matrix-prometheus @@ -1773,6 +1784,27 @@ matrix_prometheus_scraper_node_targets: "{{ ['matrix-prometheus-node-exporter:91 ###################################################################### +###################################################################### +# +# matrix-prometheus-postgres-exporter +# +###################################################################### + +matrix_prometheus_postgres_exporter_enabled: false +matrix_prometheus_postgres_exporter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'prometheus.pg.db') | to_uuid }}" + +matrix_prometheus_postgres_exporter_systemd_required_services_list: | + {{ + ['docker.service'] + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + }} + +###################################################################### +# +# /matrix-prometheus-postgres-exporter +# +###################################################################### ###################################################################### # @@ -1787,6 +1819,14 @@ matrix_grafana_enabled: false # Grafana's HTTP port to the local host. matrix_grafana_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else '127.0.0.1:3000' }}" +matrix_grafana_dashboard_download_urls_all: | + {{ + matrix_grafana_dashboard_download_urls + + + (matrix_prometheus_postgres_exporter_dashboard_urls if matrix_prometheus_postgres_exporter_enabled else []) + }} + + ###################################################################### # # /matrix-grafana diff --git a/roles/matrix-grafana/tasks/setup.yml b/roles/matrix-grafana/tasks/setup.yml index c2eea348..00d2e230 100644 --- a/roles/matrix-grafana/tasks/setup.yml +++ b/roles/matrix-grafana/tasks/setup.yml @@ -64,7 +64,7 @@ mode: 0440 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" - with_items: "{{ matrix_grafana_dashboard_download_urls }}" + with_items: "{{ matrix_grafana_dashboard_download_urls_all }}" when: matrix_grafana_enabled|bool - name: Ensure matrix-grafana.service installed diff --git a/roles/matrix-prometheus-postgres-exporter/defaults/main.yml b/roles/matrix-prometheus-postgres-exporter/defaults/main.yml new file mode 100644 index 00000000..8aca4576 --- /dev/null +++ b/roles/matrix-prometheus-postgres-exporter/defaults/main.yml @@ -0,0 +1,49 @@ +# matrix-prometheus-postgres-exporter is an Prometheus exporter for postgres metrics +# See: https://github.com/prometheus-community/postgres_exporter + +matrix_prometheus_postgres_exporter_enabled: false + +matrix_prometheus_postgres_exporter_version: v0.9.0 +matrix_prometheus_postgres_exporter_port: 9187 + +matrix_prometheus_postgres_exporter_docker_image: "quay.io/prometheuscommunity/postgres-exporter:{{ matrix_prometheus_postgres_exporter_version }}" +matrix_prometheus_postgres_exporter_docker_image_force_pull: "{{ matrix_prometheus_postgres_exporter_docker_image.endswith(':latest') }}" + +# A list of extra arguments to pass to the container +matrix_prometheus_postgres_exporter_container_extra_arguments: ["-e PG_EXPORTER_AUTO_DISCOVER_DATABASES=true", + "-e PG_EXPORTER_WEB_LISTEN_ADDRESS=\":{{matrix_prometheus_postgres_exporter_port}}\"", + "-e DATA_SOURCE_NAME=\"postgresql://{{matrix_prometheus_postgres_exporter_database_username}}:{{matrix_prometheus_postgres_exporter_database_password}}@{{matrix_prometheus_postgres_exporter_database_hostname}}:5432/{{matrix_prometheus_postgres_exporter_database_name}}?sslmode=disable\"" ] + +# List of systemd services that matrix-prometheus-postgres-exporter.service depends on +matrix_prometheus_postgres_exporter_systemd_required_services_list: ['docker.service'] + +# List of systemd services that matrix-prometheus-postgres-exporter.service wants +matrix_prometheus_postgres_exporter_systemd_wanted_services_list: [] + +# details for connecting to the database +matrix_prometheus_postgres_exporter_database_username: 'matrix_prometheus_postgres_exporter' +matrix_prometheus_postgres_exporter_database_password: 'some-password' +matrix_prometheus_postgres_exporter_database_hostname: 'matrix-postgres' +matrix_prometheus_postgres_exporter_database_port: 5432 +matrix_prometheus_postgres_exporter_database_name: 'matrix_prometheus_postgres_exporter' + + +# Controls whether the matrix-prometheus container exposes its HTTP port (tcp/9100 in the container). +# +# Takes an ":" value (e.g. "127.0.0.1:9100"), or empty string to not expose. +# +# Official recommendations are to run this container with `--net=host`, +# but we don't do that, since it: +# - likely exposes the metrics web server way too publicly (before applying https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/1008) +# - or listens on a loopback interface only (--net=host and 127.0.0.1:9100), which is not reachable from another container (like `matrix-prometheus`) +# +# Using `--net=host` and binding to Docker's `matrix` bridge network may be a solution to both, +# but that's trickier to accomplish and won't necessarily work (hasn't been tested). +# +# Not using `--net=host` means that our network statistic reports are likely broken (inaccurate), +# because node-exporter can't see all interfaces, etc. +# For now, we'll live with that, until someone develops a better solution. +matrix_prometheus_postgres_exporter_container_http_host_bind_port: '' + +matrix_prometheus_postgres_exporter_dashboard_urls: +- "https://grafana.com/api/dashboards/9628/revisions/7/download" \ No newline at end of file diff --git a/roles/matrix-prometheus-postgres-exporter/tasks/init.yml b/roles/matrix-prometheus-postgres-exporter/tasks/init.yml new file mode 100644 index 00000000..2bd6904e --- /dev/null +++ b/roles/matrix-prometheus-postgres-exporter/tasks/init.yml @@ -0,0 +1,5 @@ +- set_fact: + matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-prometheus-postgres-exporter.service'] }}" + when: matrix_prometheus_postgres_exporter_enabled|bool + + diff --git a/roles/matrix-prometheus-postgres-exporter/tasks/main.yml b/roles/matrix-prometheus-postgres-exporter/tasks/main.yml new file mode 100644 index 00000000..e3c364fa --- /dev/null +++ b/roles/matrix-prometheus-postgres-exporter/tasks/main.yml @@ -0,0 +1,8 @@ +- import_tasks: "{{ role_path }}/tasks/init.yml" + tags: + - always + +- import_tasks: "{{ role_path }}/tasks/setup.yml" + tags: + - setup-all + - setup-prometheus-postgres-exporter diff --git a/roles/matrix-prometheus-postgres-exporter/tasks/setup.yml b/roles/matrix-prometheus-postgres-exporter/tasks/setup.yml new file mode 100644 index 00000000..076ece1a --- /dev/null +++ b/roles/matrix-prometheus-postgres-exporter/tasks/setup.yml @@ -0,0 +1,54 @@ +--- + +# +# Tasks related to setting up matrix-prometheus-postgres-exporter +# + +- name: Ensure matrix-prometheus-postgres-exporter image is pulled + docker_image: + name: "{{ matrix_prometheus_postgres_exporter_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_prometheus_postgres_exporter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_prometheus_postgres_exporter_docker_image_force_pull }}" + when: "matrix_prometheus_postgres_exporter_enabled|bool" + +- name: Ensure matrix-prometheus-postgres-exporter.service installed + template: + src: "{{ role_path }}/templates/systemd/matrix-prometheus-postgres-exporter.service.j2" + dest: "{{ matrix_systemd_path }}/matrix-prometheus-postgres-exporter.service" + mode: 0644 + register: matrix_prometheus_postgres_exporter_systemd_service_result + when: matrix_prometheus_postgres_exporter_enabled|bool + +- name: Ensure systemd reloaded after matrix-prometheus.service installation + service: + daemon_reload: yes + when: "matrix_prometheus_postgres_exporter_enabled|bool and matrix_prometheus_postgres_exporter_systemd_service_result.changed" + +# +# Tasks related to getting rid of matrix-prometheus-postgres-exporter (if it was previously enabled) +# + +- name: Check existence of matrix-prometheus-postgres-exporter service + stat: + path: "{{ matrix_systemd_path }}/matrix-prometheus-postgres-exporter.service" + register: matrix_prometheus_postgres_exporter_service_stat + +- name: Ensure matrix-prometheus-postgres-exporter is stopped + service: + name: matrix-prometheus-postgres-exporter + state: stopped + daemon_reload: yes + register: stopping_result + when: "not matrix_prometheus_postgres_exporter_enabled|bool and matrix_prometheus_postgres_exporter_service_stat.stat.exists" + +- name: Ensure matrix-prometheus-postgres-exporter.service doesn't exist + file: + path: "{{ matrix_systemd_path }}/matrix-prometheus-postgres-exporter.service" + state: absent + when: "not matrix_prometheus_postgres_exporter_enabled|bool and matrix_prometheus_postgres_exporter_service_stat.stat.exists" + +- name: Ensure systemd reloaded after matrix-prometheus-postgres-exporter.service removal + service: + daemon_reload: yes + when: "not matrix_prometheus_postgres_exporter_enabled|bool and matrix_prometheus_postgres_exporter_service_stat.stat.exists" diff --git a/roles/matrix-prometheus-postgres-exporter/templates/systemd/matrix-prometheus-postgres-exporter.service.j2 b/roles/matrix-prometheus-postgres-exporter/templates/systemd/matrix-prometheus-postgres-exporter.service.j2 new file mode 100644 index 00000000..b25cb5de --- /dev/null +++ b/roles/matrix-prometheus-postgres-exporter/templates/systemd/matrix-prometheus-postgres-exporter.service.j2 @@ -0,0 +1,42 @@ +#jinja2: lstrip_blocks: "True" +[Unit] +Description=matrix-prometheus-postgres-exporter +{% for service in matrix_prometheus_postgres_exporter_systemd_required_services_list %} +Requires={{ service }} +After={{ service }} +{% endfor %} +{% for service in matrix_prometheus_postgres_exporter_systemd_wanted_services_list %} +Wants={{ service }} +{% endfor %} +DefaultDependencies=no + +[Service] +Type=simple +Environment="HOME={{ matrix_systemd_unit_home_path }}" +ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus-postgres-exporter 2>/dev/null' +ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus-postgres-exporter 2>/dev/null' + + +ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-prometheus-postgres-exporter \ + --log-driver=none \ + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + --cap-drop=ALL \ + --read-only \ + {% for arg in matrix_prometheus_postgres_exporter_container_extra_arguments %} + {{ arg }} \ + {% endfor %} + --network={{ matrix_docker_network }} \ + {% if matrix_prometheus_postgres_exporter_container_http_host_bind_port %} + -p {{ matrix_prometheus_postgres_exporter_container_http_host_bind_port }}:{{matrix_prometheus_postgres_exporter_port}} \ + {% endif %} + --pid=host \ + {{ matrix_prometheus_postgres_exporter_docker_image }} + +ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus-postgres-exporter 2>/dev/null' +ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus-postgres-exporter 2>/dev/null' +Restart=always +RestartSec=30 +SyslogIdentifier=matrix-prometheus-postgres-exporter + +[Install] +WantedBy=multi-user.target diff --git a/roles/matrix-prometheus/templates/prometheus.yml.j2 b/roles/matrix-prometheus/templates/prometheus.yml.j2 index 9502a08b..b3ee3b86 100644 --- a/roles/matrix-prometheus/templates/prometheus.yml.j2 +++ b/roles/matrix-prometheus/templates/prometheus.yml.j2 @@ -38,3 +38,9 @@ scrape_configs: static_configs: - targets: {{ matrix_prometheus_scraper_node_targets|to_json }} {% endif %} + + {% if matrix_prometheus_scraper_postgres_enabled %} + - job_name: postgres + static_configs: + - targets: {{ matrix_prometheus_scraper_postgres_targets|to_json }} + {% endif %} \ No newline at end of file diff --git a/setup.yml b/setup.yml index 9e3cd625..142364c4 100755 --- a/setup.yml +++ b/setup.yml @@ -54,5 +54,5 @@ - matrix-coturn - matrix-aux - matrix-postgres-backup - - matrix-common-after - + - matrix-prometheus-postgres-exporter + - matrix-common-after \ No newline at end of file From cf1a9078b070d6bd9d1574e04cc530031aea4707 Mon Sep 17 00:00:00 2001 From: Aaron Raimist Date: Thu, 27 May 2021 15:21:47 -0500 Subject: [PATCH 17/22] Update IRC channel --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8448680c..096c04e3 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ When updating the playbook, refer to [the changelog](CHANGELOG.md) to catch up w - Matrix room: [#matrix-docker-ansible-deploy:devture.com](https://matrix.to/#/#matrix-docker-ansible-deploy:devture.com) -- IRC channel: `#matrix-docker-ansible-deploy` on the [Freenode](https://freenode.net/) IRC network (irc.freenode.net) +- IRC channel: `#matrix-docker-ansible-deploy` on the [Libera Chat](https://libera.chat/) IRC network (irc.libera.chat:6697) - GitHub issues: [spantaleev/matrix-docker-ansible-deploy/issues](https://github.com/spantaleev/matrix-docker-ansible-deploy/issues) From 28b8bb0bf1dd8ff2c95ba817ce6e172b59ea912c Mon Sep 17 00:00:00 2001 From: WobbelTheBear Date: Fri, 28 May 2021 09:31:55 +0200 Subject: [PATCH 18/22] Minor correction --- docs/configuring-playbook-prometheus-postgres.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuring-playbook-prometheus-postgres.md b/docs/configuring-playbook-prometheus-postgres.md index 9a631296..34407aae 100644 --- a/docs/configuring-playbook-prometheus-postgres.md +++ b/docs/configuring-playbook-prometheus-postgres.md @@ -1,6 +1,6 @@ # Enabling metrics and graphs for Postgres (optional) -Expanding on the metrics exposed by the [syapse exporter and the node exporter](configuring-playbook-prometheus-grafana.md), the playbook enables the [postgres exporter](https://github.com/prometheus-community/postgres_exporter) that exposes more detailed information about what's happening on your postgres database. +Expanding on the metrics exposed by the [synapse exporter and the node exporter](configuring-playbook-prometheus-grafana.md), the playbook enables the [postgres exporter](https://github.com/prometheus-community/postgres_exporter) that exposes more detailed information about what's happening on your postgres database. You can enable this with the following settings in your configuration file (`inventory/host_vars/matrix./vars.yml`): @@ -20,7 +20,7 @@ Name | Description -----|---------- `matrix_prometheus_postgres_exporter_enabled`|Enable the postgres prometheus exporter. This sets up the docker container, connects it to the database and adds a 'job' to the prometheus config which tells prometheus about this new exporter. The default is 'false' `matrix_prometheus_postgres_exporter_database_username`| The 'username' for the user that the exporter uses to connect to the database. The default is 'matrix_prometheus_postgres_exporter' -`matrix_prometheus_postgres_exporter_database_password`| The 'username' for the user that the exporter uses to connect to the database. +`matrix_prometheus_postgres_exporter_database_password`| The 'password' for the user that the exporter uses to connect to the database. ## More information From 4880dcceb0d91ef4c3f2b56c3ddb64c477e14f1f Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 28 May 2021 11:14:01 +0300 Subject: [PATCH 19/22] Fix OCSP-stapling-related errors due to missing resolver Related to https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/1057 --- roles/matrix-nginx-proxy/defaults/main.yml | 18 +++++++++++++++--- .../nginx/conf.d/matrix-base-domain.conf.j2 | 8 ++++---- .../templates/nginx/conf.d/nginx-http.conf.j2 | 5 +++++ 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 3e79ab04..e7ee00a5 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -289,7 +289,7 @@ matrix_nginx_proxy_floc_optout_enabled: true # HSTS Preloading Enable # -# In its strongest and recommended form, the [HSTS policy](https://www.chromium.org/hsts) includes all subdomains, and +# In its strongest and recommended form, the [HSTS policy](https://www.chromium.org/hsts) includes all subdomains, and # indicates a willingness to be “preloaded” into browsers: # `Strict-Transport-Security: max-age=31536000; includeSubDomains; preload` # For more information visit: @@ -357,6 +357,18 @@ matrix_nginx_proxy_self_check_validate_certificates: true # so we default to not following redirects as well. matrix_nginx_proxy_self_check_well_known_matrix_client_follow_redirects: none +# For OCSP purposes, we need to define a resolver at the `server{}` level or `http{}` level (we do the latter). +# +# Otherwise, we get warnings like this: +# > [warn] 22#22: no resolver defined to resolve r3.o.lencr.org while requesting certificate status, responder: r3.o.lencr.org, certificate: "/matrix/ssl/config/live/.../fullchain.pem" +# +# We point it to the internal Docker resolver, which likely delegates to nameservers defined in `/etc/resolv.conf`. +# +# When nginx proxy is disabled, our configuration is likely used by non-containerized nginx, so can't use the internal Docker resolver. +# Pointing `resolver` to some public DNS server might be an option, but for now we impose DNS servers on people. +# It might also be that no such warnings occur when not running in a container. +matrix_nginx_proxy_http_level_resolver: "{{ '127.0.0.11' if matrix_nginx_proxy_enabled else '' }}" + # By default, this playbook automatically retrieves and auto-renews # free SSL certificates from Let's Encrypt. # @@ -416,7 +428,7 @@ matrix_ssl_pre_obtaining_required_service_start_wait_time_seconds: 60 # Nginx Optimize SSL Session # # ssl_session_cache: -# - Creating a cache of TLS connection parameters reduces the number of handshakes +# - Creating a cache of TLS connection parameters reduces the number of handshakes # and thus can improve the performance of application. # - Default session cache is not optimal as it can be used by only one worker process # and can cause memory fragmentation. It is much better to use shared cache. @@ -425,7 +437,7 @@ matrix_ssl_pre_obtaining_required_service_start_wait_time_seconds: 60 # ssl_session_timeout: # - Nginx by default it is set to 5 minutes which is very low. # should be like 4h or 1d but will require you to increase the size of cache. -# - Learn More: +# - Learn More: # https://github.com/certbot/certbot/issues/6903 # https://github.com/mozilla/server-side-tls/issues/198 # diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-base-domain.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-base-domain.conf.j2 index 3b0f213d..37863d73 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-base-domain.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-base-domain.conf.j2 @@ -9,13 +9,13 @@ {% if matrix_nginx_proxy_floc_optout_enabled %} add_header Permissions-Policy interest-cohort=() always; {% endif %} - + {% if matrix_nginx_proxy_hsts_preload_enabled %} add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always; {% else %} add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; {% endif %} - + add_header X-XSS-Protection "{{ matrix_nginx_proxy_xss_protection }}"; {% for configuration_block in matrix_nginx_proxy_proxy_domain_additional_server_configuration_blocks %} @@ -77,13 +77,13 @@ server { ssl_ciphers {{ matrix_nginx_proxy_ssl_ciphers }}; {% endif %} ssl_prefer_server_ciphers {{ matrix_nginx_proxy_ssl_prefer_server_ciphers }}; - + {% if matrix_nginx_proxy_ocsp_stapling_enabled %} ssl_stapling on; ssl_stapling_verify on; ssl_trusted_certificate {{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_base_domain_hostname }}/chain.pem; {% endif %} - + {% if matrix_nginx_proxy_ssl_session_tickets_off %} ssl_session_tickets off; {% endif %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/nginx-http.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/nginx-http.conf.j2 index 4811ac99..beea6afa 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/nginx-http.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/nginx-http.conf.j2 @@ -4,6 +4,11 @@ # # Thus, we ensure a larger bucket size value is used. server_names_hash_bucket_size 64; + +{% if matrix_nginx_proxy_http_level_resolver %} + resolver {{ matrix_nginx_proxy_http_level_resolver }}; +{% endif %} + {% for configuration_block in matrix_nginx_proxy_proxy_http_additional_server_configuration_blocks %} {{- configuration_block }} {% endfor %} From bee14550abb63b903026258e4188520f36ad81bb Mon Sep 17 00:00:00 2001 From: pushytoxin Date: Fri, 28 May 2021 07:56:46 +0200 Subject: [PATCH 20/22] Fix local/bin scripts autocompletion by adding rx perms to everyone It's mildly annoying when trying to execute these scripts while logged in as a regular user, as the missing execute permissions will hinder autocompletion even when trying to use with sudo. These shell scripts don't contain secrets, but may fail when ran by a regular user. The failure is due to the lack of access to the /matrix directory, and does not result in any damage. --- .../matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml | 2 +- roles/matrix-postgres/tasks/setup_postgres.yml | 6 +++--- roles/matrix-synapse/tasks/synapse/setup_install.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml b/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml index 5bf9102d..bfd25894 100644 --- a/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml +++ b/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml @@ -34,7 +34,7 @@ template: src: "{{ role_path }}/templates/usr-local-bin/matrix-ssl-lets-encrypt-certificates-renew.j2" dest: "{{ matrix_local_bin_path }}/matrix-ssl-lets-encrypt-certificates-renew" - mode: 0750 + mode: 0755 - name: Ensure SSL renewal systemd units installed template: diff --git a/roles/matrix-postgres/tasks/setup_postgres.yml b/roles/matrix-postgres/tasks/setup_postgres.yml index c072b2ea..4294bc11 100644 --- a/roles/matrix-postgres/tasks/setup_postgres.yml +++ b/roles/matrix-postgres/tasks/setup_postgres.yml @@ -77,14 +77,14 @@ template: src: "{{ role_path }}/templates/usr-local-bin/matrix-postgres-cli.j2" dest: "{{ matrix_local_bin_path }}/matrix-postgres-cli" - mode: 0750 + mode: 0755 when: matrix_postgres_enabled|bool - name: Ensure matrix-change-user-admin-status script created template: src: "{{ role_path }}/templates/usr-local-bin/matrix-change-user-admin-status.j2" dest: "{{ matrix_local_bin_path }}/matrix-change-user-admin-status" - mode: 0750 + mode: 0755 when: matrix_postgres_enabled|bool - name: (Migration) Ensure old matrix-make-user-admin script deleted @@ -97,7 +97,7 @@ template: src: "{{ role_path }}/templates/usr-local-bin/matrix-postgres-update-user-password-hash.j2" dest: "{{ matrix_local_bin_path }}/matrix-postgres-update-user-password-hash" - mode: 0750 + mode: 0755 when: matrix_postgres_enabled|bool - name: Ensure matrix-postgres.service installed diff --git a/roles/matrix-synapse/tasks/synapse/setup_install.yml b/roles/matrix-synapse/tasks/synapse/setup_install.yml index d4a7268f..b658cfff 100644 --- a/roles/matrix-synapse/tasks/synapse/setup_install.yml +++ b/roles/matrix-synapse/tasks/synapse/setup_install.yml @@ -106,4 +106,4 @@ template: src: "{{ role_path }}/templates/synapse/usr-local-bin/matrix-synapse-register-user.j2" dest: "{{ matrix_local_bin_path }}/matrix-synapse-register-user" - mode: 0750 + mode: 0755 From 897c9825172b6872126019202ed0113d0aca3ce4 Mon Sep 17 00:00:00 2001 From: sakkiii <11132948+sakkiii@users.noreply.github.com> Date: Sun, 30 May 2021 14:32:51 +0530 Subject: [PATCH 21/22] prometheus security update 2.27.1 --- roles/matrix-prometheus/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-prometheus/defaults/main.yml b/roles/matrix-prometheus/defaults/main.yml index 02afd9e9..6c0fef31 100644 --- a/roles/matrix-prometheus/defaults/main.yml +++ b/roles/matrix-prometheus/defaults/main.yml @@ -3,7 +3,7 @@ matrix_prometheus_enabled: false -matrix_prometheus_version: v2.27.0 +matrix_prometheus_version: v2.27.1 matrix_prometheus_docker_image: "{{ matrix_container_global_registry_prefix }}prom/prometheus:{{ matrix_prometheus_version }}" matrix_prometheus_docker_image_force_pull: "{{ matrix_prometheus_docker_image.endswith(':latest') }}" From 3581e1b37b2a4c6d9d30333ba4f8fa18b3080586 Mon Sep 17 00:00:00 2001 From: Toni Spets Date: Mon, 31 May 2021 08:43:55 +0300 Subject: [PATCH 22/22] Heisenbridge identd on unprivileged port Fixes running the container as an unprivileged user. --- .../templates/systemd/matrix-heisenbridge.service.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/matrix-bridge-heisenbridge/templates/systemd/matrix-heisenbridge.service.j2 b/roles/matrix-bridge-heisenbridge/templates/systemd/matrix-heisenbridge.service.j2 index 91ba39b3..e27b88f1 100644 --- a/roles/matrix-bridge-heisenbridge/templates/systemd/matrix-heisenbridge.service.j2 +++ b/roles/matrix-bridge-heisenbridge/templates/systemd/matrix-heisenbridge.service.j2 @@ -22,7 +22,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-heisenbridge \ --cap-drop=ALL \ --network={{ matrix_docker_network }} \ {% if matrix_heisenbridge_identd_enabled %} - -p 113:113 \ + -p 113:13113 \ {% endif %} -v {{ matrix_heisenbridge_base_path }}:/config:z \ {% for arg in matrix_heisenbridge_container_extra_arguments %} @@ -31,6 +31,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-heisenbridge \ {{ matrix_heisenbridge_docker_image }} \ {% if matrix_heisenbridge_identd_enabled %} --identd \ + --identd-port 13113 \ {% endif %} {% if matrix_heisenbridge_owner %} -o {{ matrix_heisenbridge_owner }} \