From 9c52a8b3a11ba51e0fd3de90e28994f1373bce21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Darveau?= Date: Mon, 10 Oct 2022 16:05:40 -0400 Subject: [PATCH 001/118] update Jitsi on LAN configuration doc --- docs/configuring-playbook-jitsi.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuring-playbook-jitsi.md b/docs/configuring-playbook-jitsi.md index f7e8a949..51ad29b7 100644 --- a/docs/configuring-playbook-jitsi.md +++ b/docs/configuring-playbook-jitsi.md @@ -87,7 +87,7 @@ For more information refer to the [docker-jitsi-meet](https://github.com/jitsi/d By default the Jitsi Meet instance does not work with a client in LAN (Local Area Network), even if others are connected from WAN. There are no video and audio. In the case of WAN to WAN everything is ok. -The reason is the Jitsi VideoBridge git to LAN client the IP address of the docker image instead of the host. The [documentation](https://github.com/jitsi/docker-jitsi-meet#running-behind-nat-or-on-a-lan-environment) of Jitsi in docker suggest to add `DOCKER_HOST_ADDRESS` in enviornment variable to make it work. +The reason is the Jitsi VideoBridge git to LAN client the IP address of the docker image instead of the host. The [documentation](https://jitsi.github.io/handbook/docs/devops-guide/devops-guide-docker/#running-behind-nat-or-on-a-lan-environment) of Jitsi in docker suggest to add `JVB_ADVERTISE_IPS` in enviornment variable to make it work. Here is how to do it in the playbook. @@ -95,7 +95,7 @@ Add these two lines to your `inventory/host_vars/matrix.DOMAIN/vars.yml` configu ```yaml matrix_jitsi_jvb_container_extra_arguments: - - '--env "DOCKER_HOST_ADDRESS="' + - '--env "JVB_ADVERTISE_IPS="' ``` ## (Optional) Fine tune Jitsi From 3a1847e193f6d99bdf5e4551856c302a7a60de55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Darveau?= Date: Mon, 10 Oct 2022 16:25:17 -0400 Subject: [PATCH 002/118] moved bind port variables documentation for webserver on different docker network or host in a more generic section --- docs/configuring-playbook-own-webserver.md | 32 ++++++++++++---------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/docs/configuring-playbook-own-webserver.md b/docs/configuring-playbook-own-webserver.md index 7e5d6001..ecae105b 100644 --- a/docs/configuring-playbook-own-webserver.md +++ b/docs/configuring-playbook-own-webserver.md @@ -27,11 +27,23 @@ No matter which external webserver you decide to go with, you'll need to: 1) Make sure your web server user (something like `http`, `apache`, `www-data`, `nginx`) is part of the `matrix` group. You should run something like this: `usermod -a -G matrix nginx`. This allows your webserver user to access files owned by the `matrix` group. When using an external nginx webserver, this allows it to read configuration files from `/matrix/nginx-proxy/conf.d`. When using another server, it would make other files, such as `/matrix/static-files/.well-known`, accessible to it. -2) Edit your configuration file (`inventory/host_vars/matrix./vars.yml`) to disable the integrated nginx server: - -```yaml -matrix_nginx_proxy_enabled: false -``` +2) Edit your configuration file (`inventory/host_vars/matrix./vars.yml`) + - to disable the integrated nginx server: + + ```yaml + matrix_nginx_proxy_enabled: false + ``` + - if using an external server on another docker network or host, add the `_http_host_bind_port` or `_http_bind_port` variables for the services that will be exposed by the external server on the other docker network or host. The actual name of the variable is listed in the `roles//defaults/vars.yml` file for each service. Most variables follow the `_http_host_bind_port` format. + + These variables will make Docker expose the ports to the local network instead of localhost only. + [Keep in mind that there are some security concerns if you simply proxy everything.](https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.md#synapse-administration-endpoints) + + Here are the variables required for the default configuration (Synapse and Element) + ``` + matrix_synapse_container_client_api_host_bind_port: '8008' + matrix_synapse_container_federation_api_plain_host_bind_port: '8048' + matrix_client_element_container_http_host_bind_port: "8765" + ``` 3) **If you'll manage SSL certificates by yourself**, edit your configuration file (`inventory/host_vars/matrix./vars.yml`) to disable SSL certificate retrieval: @@ -41,7 +53,6 @@ matrix_ssl_retrieval_method: none **Note**: During [installation](installing.md), unless you've disabled SSL certificate management (`matrix_ssl_retrieval_method: none`), the playbook would need 80 to be available, in order to retrieve SSL certificates. **Please manually stop your other webserver while installing**. You can start it back up afterwards. - ### Using your own external nginx webserver Once you've followed the [Preparation](#preparation) guide above, it's time to set up your external nginx server. @@ -60,15 +71,6 @@ matrix_nginx_proxy_ssl_protocols: "TLSv1.2" If you are experiencing issues, try updating to a newer version of Nginx. As a data point in May 2021 a user reported that Nginx 1.14.2 was not working for them. They were getting errors about socket leaks. Updating to Nginx 1.19 fixed their issue. -If you are not going to be running your webserver on the same docker network, or the same machine as matrix, these variables can be set to bind synapse to an exposed port. [Keep in mind that there are some security concerns if you simply proxy everything to it](https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.md#synapse-administration-endpoints) -```yaml -# Takes an ":" or "" value (e.g. "127.0.0.1:8048" or "192.168.1.3:80"), or empty string to not expose. -matrix_synapse_container_client_api_host_bind_port: '' -matrix_synapse_container_federation_api_plain_host_bind_port: '' -``` - - - ### Using your own external Apache webserver Once you've followed the [Preparation](#preparation) guide above, you can take a look at the [examples/apache](../examples/apache) directory for a sample configuration. From 527184097131b22ef913cc031355d3e5ace8fd0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Darveau?= Date: Mon, 10 Oct 2022 16:43:42 -0400 Subject: [PATCH 003/118] caddy2 Caddyfile : add CORS handling, update click-jacking protection to handle different subdomains and add example for cases with other well-knowns --- examples/caddy2/Caddyfile | 45 ++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/examples/caddy2/Caddyfile b/examples/caddy2/Caddyfile index 162e539e..a19ce6dd 100644 --- a/examples/caddy2/Caddyfile +++ b/examples/caddy2/Caddyfile @@ -1,3 +1,15 @@ +(cors) { + @cors_preflight method OPTIONS + + handle @cors_preflight { + header Access-Control-Allow-Origin "{args.0}" + header Access-Control-Allow-Methods "HEAD, GET, POST, PUT, PATCH, DELETE" + header Access-Control-Allow-Headers "Content-Type, Authorization" + header Access-Control-Max-Age "3600" + } +} + + matrix.DOMAIN.tld { # creates letsencrypt certificate @@ -81,6 +93,13 @@ matrix.DOMAIN.tld { header Access-Control-Allow-Origin * file_server } + + # If you have other well-knowns already handled by your base domain, you can replace the above block by this one, along with the replacement suggested in the base domain + #handle @wellknown { + # # .well-known is handled by base domain + # reverse_proxy https://DOMAIN.tld { + # header_up Host {http.reverse_proxy.upstream.hostport} + #} handle { encode zstd gzip @@ -114,6 +133,8 @@ element.DOMAIN.tld { # creates letsencrypt certificate # tls your@email.com + import cors https://*.DOMAIN.tld + header { # Enable HTTP Strict Transport Security (HSTS) to force clients to always connect via HTTPS Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" @@ -123,6 +144,8 @@ element.DOMAIN.tld { X-Content-Type-Options "nosniff" # Disallow the site to be rendered within a frame (clickjacking protection) X-Frame-Options "DENY" + # If using integrations that add frames to Element, such as Dimension and its integrations running on the same domain, it can be a good idea to limit sources allowed to be rendered + # Content-Security-Policy frame-src https://*.DOMAIN.tld # X-Robots-Tag X-Robots-Tag "noindex, noarchive, nofollow" } @@ -144,6 +167,8 @@ element.DOMAIN.tld { # # creates letsencrypt certificate # # tls your@email.com # +# import cors https://*.DOMAIN.tld +# # header { # # Enable HTTP Strict Transport Security (HSTS) to force clients to always connect via HTTPS # Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" @@ -151,8 +176,8 @@ element.DOMAIN.tld { # X-XSS-Protection "1; mode=block" # # Prevent some browsers from MIME-sniffing a response away from the declared Content-Type # X-Content-Type-Options "nosniff" -# # Disallow the site to be rendered within a frame (clickjacking protection) -# X-Frame-Options "DENY" +# # Only allow same base domain to render this website in a frame; Can be removed if the client (Element for example) is hosted on another domain (clickjacking protection) +# # Content-Security-Policy frame-ancestors https://*.DOMAIN.tld # # X-Robots-Tag # X-Robots-Tag "noindex, noarchive, nofollow" # } @@ -176,6 +201,8 @@ element.DOMAIN.tld { # creates letsencrypt certificate # tls your@email.com # +# import cors https://*.DOMAIN.tld +# # header { # # Enable HTTP Strict Transport Security (HSTS) to force clients to always connect via HTTPS # Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" @@ -185,9 +212,9 @@ element.DOMAIN.tld { # # # Prevent some browsers from MIME-sniffing a response away from the declared Content-Type # X-Content-Type-Options "nosniff" -# -# # Disallow the site to be rendered within a frame (clickjacking protection) -# X-Frame-Options "SAMEORIGIN" + +# # Only allow same base domain to render this website in a frame; Can be removed if the client (Element for example) is hosted on another domain +# # Content-Security-Policy frame-ancestors https://*.DOMAIN.tld # # # Disable some features # Feature-Policy "accelerometer 'none';ambient-light-sensor 'none'; autoplay 'none';camera 'none';encrypted-media 'none';focus-without-user-activation 'none'; geolocation 'none';gyroscope #'none';magnetometer 'none';microphone 'none';midi 'none';payment 'none';picture-in-picture 'none'; speaker 'none';sync-xhr 'none';usb 'none';vr 'none'" @@ -225,6 +252,14 @@ element.DOMAIN.tld { # header_up Host {http.reverse_proxy.upstream.hostport} # } # } +# # If you have other well-knowns already handled by your base domain, you can replace the above block by this one, along with the replacement suggested in the matrix subdomain +# # handle /.well-known/* { +# # encode zstd gzip +# # header Cache-Control max-age=14400 +# # header Content-Type application/json +# # header Access-Control-Allow-Origin * +# #} +# # # Configration for the base domain goes here # # handle { # # header -Server From a47ce70cd26a83148785d112e8b647377fe50300 Mon Sep 17 00:00:00 2001 From: Jim Myhrberg Date: Tue, 11 Oct 2022 00:43:22 +0100 Subject: [PATCH 004/118] fix(goofys): fix synapse systemd unit file to correctly require goofys On Debian 10 (buster) at least, while the Synapse systemd service unit was referring to Goofys as "matrix-goofys" without a ".service" suffix, systemd was ignoring the goofys dependency, starting Synapse before Goofys. All other dependant units which work are using the ".service" suffix. This generally leads to the mount path goofys using having been populated by Synapse before Goofys starts, causing it to fail due to the mount target not being empty. The fix seems to be to ensure that the Synapse service unit refers to Goofys as "matrix-goofys.service". This change causes the following two lines in "/etc/systemd/system/matrix-synapse.service": Requires=matrix-goofys After=matrix-goofys To be changed to: Requires=matrix-goofys.service After=matrix-goofys.service --- group_vars/matrix_servers | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index cfccd387..b055245b 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -2364,7 +2364,7 @@ matrix_synapse_systemd_required_services_list: | + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + - (['matrix-goofys'] if matrix_s3_media_store_enabled else []) + (['matrix-goofys.service'] if matrix_s3_media_store_enabled else []) }} matrix_synapse_systemd_wanted_services_list: | @@ -2650,7 +2650,7 @@ matrix_dendrite_systemd_required_services_list: | + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + - (['matrix-goofys'] if matrix_s3_media_store_enabled else []) + (['matrix-goofys.service'] if matrix_s3_media_store_enabled else []) }} matrix_dendrite_systemd_wanted_services_list: | From 9412140477902e9aff7177c9a039527ba40c4c8d Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 11 Oct 2022 06:00:49 +0000 Subject: [PATCH 005/118] Enable hookshot feeds by default as per documentation: "Services that need no further configuration are enabled by default" and feeds service doesn't require it --- roles/matrix-bridge-hookshot/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-hookshot/defaults/main.yml b/roles/matrix-bridge-hookshot/defaults/main.yml index b9b30866..0854edbf 100644 --- a/roles/matrix-bridge-hookshot/defaults/main.yml +++ b/roles/matrix-bridge-hookshot/defaults/main.yml @@ -128,7 +128,7 @@ matrix_hookshot_generic_allow_js_transformation_functions: false matrix_hookshot_generic_user_id_prefix: '_webhooks_' -matrix_hookshot_feeds_enabled: false +matrix_hookshot_feeds_enabled: true # polling interval in seconds matrix_hookshot_feeds_interval: 600 From 5ff03c0b15eb16e5e3a7f016925f6df78f36e1c6 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 11 Oct 2022 06:04:18 +0000 Subject: [PATCH 006/118] Change hookshot logging level to warn all other bridges use that level --- roles/matrix-bridge-hookshot/templates/config.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-hookshot/templates/config.yml.j2 b/roles/matrix-bridge-hookshot/templates/config.yml.j2 index 6fbce770..5cd11a87 100644 --- a/roles/matrix-bridge-hookshot/templates/config.yml.j2 +++ b/roles/matrix-bridge-hookshot/templates/config.yml.j2 @@ -108,7 +108,7 @@ metrics: logging: # (Optional) Logging settings. You can have a severity debug,info,warn,error # - level: info + level: warn {% if matrix_hookshot_widgets_enabled %} widgets: # (Optional) EXPERIMENTAL support for complimentary widgets From da041bc60c67f674ac9f748e528ee3d42ebafb28 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 11 Oct 2022 09:38:11 +0300 Subject: [PATCH 007/118] Upgrade mautrix-instagram (0.2.1 -> latest) We don't like updating to untagged releases, but.. 0.2.1 has some regression and upstream is not releasing 0.2.2 or 0.3.0 just yet, so we either need to downgrade to 0.2.0 or go `latest`. We can hopefully switch back to a tagged release soon. Related to https://github.com/mautrix/instagram/issues/56 --- roles/matrix-bridge-mautrix-instagram/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-mautrix-instagram/defaults/main.yml b/roles/matrix-bridge-mautrix-instagram/defaults/main.yml index b739a4a3..7305d0fb 100644 --- a/roles/matrix-bridge-mautrix-instagram/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-instagram/defaults/main.yml @@ -8,7 +8,7 @@ matrix_mautrix_instagram_container_image_self_build: false matrix_mautrix_instagram_container_image_self_build_repo: "https://github.com/mautrix/instagram.git" matrix_mautrix_instagram_container_image_self_build_repo_version: "{{ 'master' if matrix_mautrix_instagram_version == 'latest' else matrix_mautrix_instagram_version }}" -matrix_mautrix_instagram_version: v0.2.1 +matrix_mautrix_instagram_version: latest # See: https://mau.dev/tulir/mautrix-instagram/container_registry matrix_mautrix_instagram_docker_image: "{{ matrix_mautrix_instagram_docker_image_name_prefix }}mautrix/instagram:{{ matrix_mautrix_instagram_version }}" matrix_mautrix_instagram_docker_image_name_prefix: "{{ 'localhost/' if matrix_mautrix_instagram_container_image_self_build else 'dock.mau.dev/' }}" From 8311c59a8fa3a9b13b9076d0d6fbaefff66e74be Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 11 Oct 2022 10:43:01 +0300 Subject: [PATCH 008/118] Add Usage section to Hookshot docs --- docs/configuring-playbook-bridge-hookshot.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/configuring-playbook-bridge-hookshot.md b/docs/configuring-playbook-bridge-hookshot.md index 3e8a54a2..51d7a335 100644 --- a/docs/configuring-playbook-bridge-hookshot.md +++ b/docs/configuring-playbook-bridge-hookshot.md @@ -8,6 +8,7 @@ See the project's [documentation](https://matrix-org.github.io/matrix-hookshot/l Note: the playbook also supports [matrix-appservice-webhooks](configuring-playbook-bridge-appservice-webhooks.md), which however is soon to be archived by its author and to be replaced by hookshot. + ## Setup Instructions Refer to the [official instructions](https://matrix-org.github.io/matrix-hookshot/latest/setup.html) to learn what the individual options do. @@ -16,10 +17,25 @@ Refer to the [official instructions](https://matrix-org.github.io/matrix-hooksho 2. Take special note of the `matrix_hookshot_*_enabled` variables. Services that need no further configuration are enabled by default (GitLab, Generic), while you must first add the required configuration and enable the others (GitHub, Jira, Figma). 3. If you're setting up the GitHub bridge, you'll need to generate and download a private key file after you created your GitHub app. Copy the contents of that file to the variable `matrix_hookshot_github_private_key` so the playbook can install it for you, or use one of the [other methods](#manage-github-private-key-with-matrix-aux-role) explained below. 4. If you've already installed Matrix services using the playbook before, you'll need to re-run it (`--tags=setup-all,start`). If not, proceed with [configuring other playbook services](configuring-playbook.md) and then with [Installing](installing.md). Get back to this guide once ready. Hookshot can be set up individually using the tag `setup-hookshot`. -5. Refer to [Hookshot's official instructions](https://matrix-org.github.io/matrix-hookshot/latest/usage.html) to start using the bridge. **Important:** Note that the different listeners are bound to certain paths which might differ from those assumed by the hookshot documentation, see [URLs for bridges setup](urls-for-bridges-setup) below. Other configuration options are available via the `matrix_hookshot_configuration_extension_yaml` and `matrix_hookshot_registration_extension_yaml` variables, see the comments in [main.yml](/roles/matrix-bridge-hookshot/defaults/main.yml) for how to use them. + +## Usage + +Create a room and invite the Hookshot bot (`@hookshot:DOMAIN`) to it. + +Make sure the bot is able to send state events (usually the Moderator power level in clients). + +Send a `!hookshot help` message to see a list of help commands. + +Refer to [Hookshot's documentation](https://matrix-org.github.io/matrix-hookshot/latest/usage.html) for more details about using the brige's various features. + +**Important:** Note that the different listeners are bound to certain paths which might differ from those assumed by the hookshot documentation, see [URLs for bridges setup](#urls-for-bridges-setup) below. + + +## More setup documentation + ### URLs for bridges setup Unless indicated otherwise, the following endpoints are reachable on your `matrix.` subdomain (if the feature is enabled). From 21bed90e13c46b6ed5fb3f0945bf9435e76ede90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Darveau?= Date: Tue, 11 Oct 2022 07:26:29 -0400 Subject: [PATCH 009/118] remove mention of other docker host in external webserver documentation (binding ports section) --- docs/configuring-playbook-own-webserver.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/configuring-playbook-own-webserver.md b/docs/configuring-playbook-own-webserver.md index ecae105b..50d9d1da 100644 --- a/docs/configuring-playbook-own-webserver.md +++ b/docs/configuring-playbook-own-webserver.md @@ -33,16 +33,16 @@ No matter which external webserver you decide to go with, you'll need to: ```yaml matrix_nginx_proxy_enabled: false ``` - - if using an external server on another docker network or host, add the `_http_host_bind_port` or `_http_bind_port` variables for the services that will be exposed by the external server on the other docker network or host. The actual name of the variable is listed in the `roles//defaults/vars.yml` file for each service. Most variables follow the `_http_host_bind_port` format. + - if using an external server on another host, add the `_http_host_bind_port` or `_http_bind_port` variables for the services that will be exposed by the external server on the other host. The actual name of the variable is listed in the `roles//defaults/vars.yml` file for each service. Most variables follow the `_http_host_bind_port` format. These variables will make Docker expose the ports to the local network instead of localhost only. [Keep in mind that there are some security concerns if you simply proxy everything.](https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.md#synapse-administration-endpoints) Here are the variables required for the default configuration (Synapse and Element) ``` - matrix_synapse_container_client_api_host_bind_port: '8008' - matrix_synapse_container_federation_api_plain_host_bind_port: '8048' - matrix_client_element_container_http_host_bind_port: "8765" + matrix_synapse_container_client_api_host_bind_port: '0.0.0.0:8008' + matrix_synapse_container_federation_api_plain_host_bind_port: '0.0.0.0:8048' + matrix_client_element_container_http_host_bind_port: "0.0.0.0:8765" ``` 3) **If you'll manage SSL certificates by yourself**, edit your configuration file (`inventory/host_vars/matrix./vars.yml`) to disable SSL certificate retrieval: From a061ea54b3e302e1582daa442690ad8d71d6ddc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Darveau?= Date: Tue, 11 Oct 2022 07:32:02 -0400 Subject: [PATCH 010/118] Caddyfile example : enable Content-Security-Policy by default instead of having the line commented --- examples/caddy2/Caddyfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/caddy2/Caddyfile b/examples/caddy2/Caddyfile index a19ce6dd..43005ca4 100644 --- a/examples/caddy2/Caddyfile +++ b/examples/caddy2/Caddyfile @@ -177,7 +177,7 @@ element.DOMAIN.tld { # # Prevent some browsers from MIME-sniffing a response away from the declared Content-Type # X-Content-Type-Options "nosniff" # # Only allow same base domain to render this website in a frame; Can be removed if the client (Element for example) is hosted on another domain (clickjacking protection) -# # Content-Security-Policy frame-ancestors https://*.DOMAIN.tld +# Content-Security-Policy frame-ancestors https://*.DOMAIN.tld # # X-Robots-Tag # X-Robots-Tag "noindex, noarchive, nofollow" # } @@ -214,7 +214,7 @@ element.DOMAIN.tld { # X-Content-Type-Options "nosniff" # # Only allow same base domain to render this website in a frame; Can be removed if the client (Element for example) is hosted on another domain -# # Content-Security-Policy frame-ancestors https://*.DOMAIN.tld +# Content-Security-Policy frame-ancestors https://*.DOMAIN.tld # # # Disable some features # Feature-Policy "accelerometer 'none';ambient-light-sensor 'none'; autoplay 'none';camera 'none';encrypted-media 'none';focus-without-user-activation 'none'; geolocation 'none';gyroscope #'none';magnetometer 'none';microphone 'none';midi 'none';payment 'none';picture-in-picture 'none'; speaker 'none';sync-xhr 'none';usb 'none';vr 'none'" From 19f1c8ea5ee3ffbd520cc6c54fc45957bea8b0e5 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 11 Oct 2022 14:29:52 +0000 Subject: [PATCH 011/118] Update element 1.11.8 -> 1.11.9 --- roles/matrix-client-element/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index e4b6315b..acdaec83 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -10,7 +10,7 @@ matrix_client_element_container_image_self_build_repo: "https://github.com/vecto # - https://github.com/vector-im/element-web/issues/19544 matrix_client_element_container_image_self_build_low_memory_system_patch_enabled: "{{ ansible_memtotal_mb < 4096 }}" -matrix_client_element_version: v1.11.8 +matrix_client_element_version: v1.11.9 matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:{{ matrix_client_element_version }}" matrix_client_element_docker_image_name_prefix: "{{ 'localhost/' if matrix_client_element_container_image_self_build else matrix_container_global_registry_prefix }}" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" From fe8f91a9dac67e6b48a8704e7010768575eed1ab Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 11 Oct 2022 14:30:55 +0000 Subject: [PATCH 012/118] Update postmoogle 0.9.5 -> 0.9.6 * show recipient's email (header `TO`) * add `norecipient` option * add anti-spam options (`spamcheck:` and `spamlist:`) * fix bugs --- roles/matrix-bot-postmoogle/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bot-postmoogle/defaults/main.yml b/roles/matrix-bot-postmoogle/defaults/main.yml index 949f0913..8e76998e 100644 --- a/roles/matrix-bot-postmoogle/defaults/main.yml +++ b/roles/matrix-bot-postmoogle/defaults/main.yml @@ -9,7 +9,7 @@ matrix_bot_postmoogle_docker_repo: "https://gitlab.com/etke.cc/postmoogle.git" matrix_bot_postmoogle_docker_repo_version: "{{ 'main' if matrix_bot_postmoogle_version == 'latest' else matrix_bot_postmoogle_version }}" matrix_bot_postmoogle_docker_src_files_path: "{{ matrix_base_data_path }}/postmoogle/docker-src" -matrix_bot_postmoogle_version: v0.9.5 +matrix_bot_postmoogle_version: v0.9.6 matrix_bot_postmoogle_docker_image: "{{ matrix_bot_postmoogle_docker_image_name_prefix }}postmoogle:{{ matrix_bot_postmoogle_version }}" matrix_bot_postmoogle_docker_image_name_prefix: "{{ 'localhost/' if matrix_bot_postmoogle_container_image_self_build else 'registry.gitlab.com/etke.cc/' }}" matrix_bot_postmoogle_docker_image_force_pull: "{{ matrix_bot_postmoogle_docker_image.endswith(':latest') }}" From 3c669761cec4ffd6e7811776934c8927890d1180 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 11 Oct 2022 18:31:46 +0000 Subject: [PATCH 013/118] Update element 1.11.9 -> 1.11.10 --- roles/matrix-client-element/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index acdaec83..c8e476a1 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -10,7 +10,7 @@ matrix_client_element_container_image_self_build_repo: "https://github.com/vecto # - https://github.com/vector-im/element-web/issues/19544 matrix_client_element_container_image_self_build_low_memory_system_patch_enabled: "{{ ansible_memtotal_mb < 4096 }}" -matrix_client_element_version: v1.11.9 +matrix_client_element_version: v1.11.10 matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:{{ matrix_client_element_version }}" matrix_client_element_docker_image_name_prefix: "{{ 'localhost/' if matrix_client_element_container_image_self_build else matrix_container_global_registry_prefix }}" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" From 43b1be9a3c121764c9c48eb7a3542aaee7e0f469 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 11 Oct 2022 18:57:52 +0000 Subject: [PATCH 014/118] Update grafana 9.1.7 -> 9.2.0 --- roles/matrix-grafana/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-grafana/defaults/main.yml b/roles/matrix-grafana/defaults/main.yml index 2b139cd4..677435e8 100644 --- a/roles/matrix-grafana/defaults/main.yml +++ b/roles/matrix-grafana/defaults/main.yml @@ -5,7 +5,7 @@ matrix_grafana_enabled: true -matrix_grafana_version: 9.1.7 +matrix_grafana_version: 9.2.0 matrix_grafana_docker_image: "{{ matrix_container_global_registry_prefix }}grafana/grafana:{{ matrix_grafana_version }}" matrix_grafana_docker_image_force_pull: "{{ matrix_grafana_docker_image.endswith(':latest') }}" From 7c7b9d119f73a42222f089548e4a97be959ca5fe Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Wed, 12 Oct 2022 10:57:25 +0000 Subject: [PATCH 015/118] Update postmoogle 0.9.6 -> 0.9.7 * fix room account data retrieving --- roles/matrix-bot-postmoogle/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bot-postmoogle/defaults/main.yml b/roles/matrix-bot-postmoogle/defaults/main.yml index 8e76998e..1bcd570a 100644 --- a/roles/matrix-bot-postmoogle/defaults/main.yml +++ b/roles/matrix-bot-postmoogle/defaults/main.yml @@ -9,7 +9,7 @@ matrix_bot_postmoogle_docker_repo: "https://gitlab.com/etke.cc/postmoogle.git" matrix_bot_postmoogle_docker_repo_version: "{{ 'main' if matrix_bot_postmoogle_version == 'latest' else matrix_bot_postmoogle_version }}" matrix_bot_postmoogle_docker_src_files_path: "{{ matrix_base_data_path }}/postmoogle/docker-src" -matrix_bot_postmoogle_version: v0.9.6 +matrix_bot_postmoogle_version: v0.9.7 matrix_bot_postmoogle_docker_image: "{{ matrix_bot_postmoogle_docker_image_name_prefix }}postmoogle:{{ matrix_bot_postmoogle_version }}" matrix_bot_postmoogle_docker_image_name_prefix: "{{ 'localhost/' if matrix_bot_postmoogle_container_image_self_build else 'registry.gitlab.com/etke.cc/' }}" matrix_bot_postmoogle_docker_image_force_pull: "{{ matrix_bot_postmoogle_docker_image.endswith(':latest') }}" From 35e80b708781e7db6c04a778ce0892c7db225605 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Darveau?= Date: Wed, 12 Oct 2022 09:15:03 -0400 Subject: [PATCH 016/118] Update docs/configuring-playbook-own-webserver.md Co-authored-by: Slavi Pantaleev --- docs/configuring-playbook-own-webserver.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuring-playbook-own-webserver.md b/docs/configuring-playbook-own-webserver.md index 50d9d1da..87201e3e 100644 --- a/docs/configuring-playbook-own-webserver.md +++ b/docs/configuring-playbook-own-webserver.md @@ -35,7 +35,7 @@ No matter which external webserver you decide to go with, you'll need to: ``` - if using an external server on another host, add the `_http_host_bind_port` or `_http_bind_port` variables for the services that will be exposed by the external server on the other host. The actual name of the variable is listed in the `roles//defaults/vars.yml` file for each service. Most variables follow the `_http_host_bind_port` format. - These variables will make Docker expose the ports to the local network instead of localhost only. + These variables will make Docker expose the ports on all network interfaces instead of localhost only. [Keep in mind that there are some security concerns if you simply proxy everything.](https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.md#synapse-administration-endpoints) Here are the variables required for the default configuration (Synapse and Element) From 337016da28e2cb18c4cf3624a47c2faec4fed272 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 09:27:46 +0300 Subject: [PATCH 017/118] Improve S3 docs a bit Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/1869 --- docs/configuring-playbook-s3.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/configuring-playbook-s3.md b/docs/configuring-playbook-s3.md index 9132ff71..43aaa879 100644 --- a/docs/configuring-playbook-s3.md +++ b/docs/configuring-playbook-s3.md @@ -13,7 +13,7 @@ If you'd like to move your locally-stored media store data to Amazon S3 (or anot ## Amazon S3 -You'll need an Amazon S3 bucket and some IAM user credentials (access key + secret key) with full write access to the bucket. Example security policy: +You'll need an Amazon S3 bucket and some IAM user credentials (access key + secret key) with full write access to the bucket. Example IAM security policy: ```json { @@ -34,6 +34,8 @@ You'll need an Amazon S3 bucket and some IAM user credentials (access key + secr } ``` +**NOTE**: This policy needs to be attached to an IAM user creted from the **Security Credentials** menu. This is not a **Bucket Policy**. + You then need to enable S3 support in your configuration file (`inventory/host_vars/matrix./vars.yml`). It would be something like this: @@ -91,8 +93,13 @@ It's a good idea to [make a complete server backup](faq.md#how-do-i-backup-the-d Follow one of the guides below for a migration path from a locally-stored media store to one stored on S3-compatible storage: -- [Migrating to any S3-compatible storage (universal, but likely slow)](#migrating-to-any-s3-compatible-storage-universal-but-likely-slow) -- [Migrating to Backblaze B2](#migrating-to-backblaze-b2) +- [Storing Matrix media files on Amazon S3 (optional)](#storing-matrix-media-files-on-amazon-s3-optional) + - [Amazon S3](#amazon-s3) + - [Using other S3-compatible object stores](#using-other-s3-compatible-object-stores) + - [Backblaze B2](#backblaze-b2) + - [Migrating from local filesystem storage to S3](#migrating-from-local-filesystem-storage-to-s3) + - [Migrating to any S3-compatible storage (universal, but likely slow)](#migrating-to-any-s3-compatible-storage-universal-but-likely-slow) + - [Migrating to Backblaze B2](#migrating-to-backblaze-b2) ### Migrating to any S3-compatible storage (universal, but likely slow) From 62215a87fed6cbeffdb124261209df00f82ba1a5 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 16:25:57 +0300 Subject: [PATCH 018/118] Upgrade Postgres minor versions (14.4 -> 14.5, 13.7 -> 13.8, etc.) --- roles/matrix-postgres/defaults/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index 117611a3..de28d7ad 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -24,11 +24,11 @@ matrix_postgres_architecture: amd64 matrix_postgres_docker_image_suffix: "{{ '-alpine' if matrix_postgres_architecture in ['amd64', 'arm64'] else '' }}" matrix_postgres_docker_image_v9: "{{ matrix_container_global_registry_prefix }}postgres:9.6.24{{ matrix_postgres_docker_image_suffix }}" -matrix_postgres_docker_image_v10: "{{ matrix_container_global_registry_prefix }}postgres:10.21{{ matrix_postgres_docker_image_suffix }}" -matrix_postgres_docker_image_v11: "{{ matrix_container_global_registry_prefix }}postgres:11.16{{ matrix_postgres_docker_image_suffix }}" -matrix_postgres_docker_image_v12: "{{ matrix_container_global_registry_prefix }}postgres:12.11{{ matrix_postgres_docker_image_suffix }}" -matrix_postgres_docker_image_v13: "{{ matrix_container_global_registry_prefix }}postgres:13.7{{ matrix_postgres_docker_image_suffix }}" -matrix_postgres_docker_image_v14: "{{ matrix_container_global_registry_prefix }}postgres:14.4{{ matrix_postgres_docker_image_suffix }}" +matrix_postgres_docker_image_v10: "{{ matrix_container_global_registry_prefix }}postgres:10.22{{ matrix_postgres_docker_image_suffix }}" +matrix_postgres_docker_image_v11: "{{ matrix_container_global_registry_prefix }}postgres:11.17{{ matrix_postgres_docker_image_suffix }}" +matrix_postgres_docker_image_v12: "{{ matrix_container_global_registry_prefix }}postgres:12.12{{ matrix_postgres_docker_image_suffix }}" +matrix_postgres_docker_image_v13: "{{ matrix_container_global_registry_prefix }}postgres:13.8{{ matrix_postgres_docker_image_suffix }}" +matrix_postgres_docker_image_v14: "{{ matrix_container_global_registry_prefix }}postgres:14.5{{ matrix_postgres_docker_image_suffix }}" matrix_postgres_docker_image_latest: "{{ matrix_postgres_docker_image_v14 }}" # This variable is assigned at runtime. Overriding its value has no effect. From 0da068c7e61a35c2fdb226fe95df4f06b6527953 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 16:28:47 +0300 Subject: [PATCH 019/118] Add support for configuring Synapse's media_storage_providers In the future, we'll also inject media storage providers that we support into the `matrix_synapse_media_storage_providers_auto` list. --- roles/matrix-synapse/defaults/main.yml | 20 +++++++++++++++++++ .../templates/synapse/homeserver.yaml.j2 | 1 + 2 files changed, 21 insertions(+) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 4c5fc09e..d117eaaf 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -798,6 +798,26 @@ matrix_synapse_spam_checker: [] # Certain Synapse extensions that you can enable below auto-inject themselves into `matrix_synapse_modules` at runtime. matrix_synapse_modules: [] +# matrix_synapse_media_storage_providers contains the Synapse `media_storage_providers` configuration setting. +# To add your own custom `media_storage_providers`, use `matrix_synapse_media_storage_providers_custom`. +matrix_synapse_media_storage_providers: "{{ matrix_synapse_media_storage_providers_auto + matrix_synapse_media_storage_providers_custom }}" + +# matrix_synapse_media_storage_providers_auto contains a list of storage providers that are added by the playbook based on other configuration +matrix_synapse_media_storage_providers_auto: | + {{ + [] + }} + +# matrix_synapse_media_storage_providers_custom contains your own custom list of storage providers. +# You're meant to define each custom module as valid keys and values, not as a YAML string that needs to be parsed. +# +# Example: +# matrix_synapse_media_storage_providers_custom: +# - module: module.SomeModule +# store_local: True +# # ... +matrix_synapse_media_storage_providers_custom: [] + matrix_synapse_encryption_enabled_by_default_for_room_type: "off" matrix_synapse_trusted_key_servers: diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index ae71b7ae..9b02346c 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -1029,6 +1029,7 @@ media_store_path: "/matrix-media-store-parent/{{ matrix_synapse_media_store_dire # store_synchronous: false # config: # directory: /mnt/some/other/directory +media_storage_providers: {{ matrix_synapse_media_storage_providers | to_json }} # The largest allowed upload size in bytes # From 881fdd28f0bf2e10635ac7a6c3941fa8636cdbff Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 16:33:19 +0300 Subject: [PATCH 020/118] Add support for Synapse container image customization This allows people to augment the Synapse image with custom tools and addons without having to rebuild it from scratch. If customizations are enabled, the playbook will build a new `localhost/matrixdotorg/synapse:VERSION-customized` image on top of the default one (`FROM matrixdotorg/synapse:VERSION`) and with custom Dockerfile build steps. For servers that self-build the Synapse image, the Synapse image will be built first, before proceding to extend it the same way. In the future, we'll also have easy to enable Dockerfile build steps for modules that the playbook supports. --- .../tasks/import_synapse_sqlite_db.yml | 2 +- roles/matrix-synapse/defaults/main.yml | 33 +++++++++++++++++++ roles/matrix-synapse/tasks/setup_synapse.yml | 1 + .../tasks/synapse/setup_install.yml | 19 +++++++++++ .../tasks/synapse/setup_uninstall.yml | 5 ++- .../synapse/customizations/Dockerfile.j2 | 3 ++ .../systemd/matrix-synapse-worker.service.j2 | 2 +- .../synapse/systemd/matrix-synapse.service.j2 | 2 +- 8 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 diff --git a/roles/matrix-postgres/tasks/import_synapse_sqlite_db.yml b/roles/matrix-postgres/tasks/import_synapse_sqlite_db.yml index a459b6e1..b885ea97 100644 --- a/roles/matrix-postgres/tasks/import_synapse_sqlite_db.yml +++ b/roles/matrix-postgres/tasks/import_synapse_sqlite_db.yml @@ -83,7 +83,7 @@ --mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/data --mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/matrix-media-store-parent/media-store --mount type=bind,src={{ server_path_homeserver_db }},dst=/{{ server_path_homeserver_db | basename }} - {{ matrix_synapse_docker_image }} + {{ matrix_synapse_docker_image_final }} /usr/local/bin/synapse_port_db --sqlite-database /{{ server_path_homeserver_db | basename }} --postgres-config /data/homeserver.yaml register: matrix_postgres_import_synapse_sqlite_db_result changed_when: matrix_postgres_import_synapse_sqlite_db_result.rc == 0 diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index d117eaaf..40e05be7 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -7,14 +7,47 @@ matrix_synapse_enabled: true matrix_synapse_container_image_self_build: false matrix_synapse_container_image_self_build_repo: "https://github.com/matrix-org/synapse.git" +# matrix_synapse_container_image_customizations_enabled controls whether a customized Synapse image will be built. +# +# We toggle this variable to `true` when certain features which require a custom build are enabled. +# Feel free to toggle this to `true` yourself and specify build steps in `matrix_synapse_container_image_customizations_dockerfile_body_custom`. +# +# See: +# - `roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2` +# - `matrix_synapse_container_image_customizations_dockerfile_body_custom` +# - `matrix_synapse_docker_image_customized` +# - `matrix_synapse_docker_image_final` +matrix_synapse_container_image_customizations_enabled: "{{ matrix_synapse_ext_synapse_s3_storage_provider_enabled }}" + +# matrix_synapse_container_image_customizations_dockerfile_body contains your custom Dockerfile steps +# for building your customized Synapse image based on the original (upstream) image (`matrix_synapse_docker_image`). +# A `FROM ...` clause is included automatically so you don't have to. +# +# Example: +# matrix_synapse_container_image_customizations_dockerfile_body_custom: | +# RUN echo 'This is a custom step for building the customized Docker image for Synapse.' +# RUN echo 'You can override matrix_synapse_container_image_customizations_dockerfile_body_custom to add your own steps.' +# RUN echo 'Note that matrix_synapse_container_image_customizations_dockerfile_body_auto injects steps before this.' +# RUN echo 'You do NOT need to include a FROM clause yourself.' +matrix_synapse_container_image_customizations_dockerfile_body_custom: '' + matrix_synapse_docker_image: "{{ matrix_synapse_docker_image_name_prefix }}matrixdotorg/synapse:{{ matrix_synapse_docker_image_tag }}" matrix_synapse_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_container_image_self_build else matrix_container_global_registry_prefix }}" matrix_synapse_version: v1.68.0 matrix_synapse_docker_image_tag: "{{ matrix_synapse_version }}" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" +# matrix_synapse_docker_image_customized is the name of the locally built Synapse image +# which adds various customizations on top of the original (upstream) Synapse image. +# This image will be based on the upstream `matrix_synapse_docker_image` image, only if `matrix_synapse_container_image_customizations_enabled: true`. +matrix_synapse_docker_image_customized: "localhost/matrixdotorg/synapse:{{ matrix_synapse_docker_image_tag }}-customized" + +# matrix_synapse_docker_image_final holds the name of the Synapse image to run depending on whether or not customizations are enabled. +matrix_synapse_docker_image_final: "{{ matrix_synapse_docker_image_customized if matrix_synapse_container_image_customizations_enabled else matrix_synapse_docker_image }} " + matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse" matrix_synapse_docker_src_files_path: "{{ matrix_synapse_base_path }}/docker-src" +matrix_synapse_customized_docker_src_files_path: "{{ matrix_synapse_base_path }}/customized-docker-src" matrix_synapse_config_dir_path: "{{ matrix_synapse_base_path }}/config" matrix_synapse_storage_path: "{{ matrix_synapse_base_path }}/storage" matrix_synapse_media_store_path: "{{ matrix_synapse_storage_path }}/media-store" diff --git a/roles/matrix-synapse/tasks/setup_synapse.yml b/roles/matrix-synapse/tasks/setup_synapse.yml index d4e6ae95..7b887f30 100644 --- a/roles/matrix-synapse/tasks/setup_synapse.yml +++ b/roles/matrix-synapse/tasks/setup_synapse.yml @@ -11,6 +11,7 @@ - {path: "{{ matrix_synapse_config_dir_path }}", when: true} - {path: "{{ matrix_synapse_ext_path }}", when: true} - {path: "{{ matrix_synapse_docker_src_files_path }}", when: "{{ matrix_synapse_container_image_self_build }}"} + - {path: "{{ matrix_synapse_customized_docker_src_files_path }}", when: "{{ matrix_synapse_container_image_customizations_enabled }}"} # We handle matrix_synapse_media_store_path elsewhere (in ./synapse/setup_install.yml), # because if it's using Goofys and it's already mounted (from before), # trying to chown/chmod it here will cause trouble. diff --git a/roles/matrix-synapse/tasks/synapse/setup_install.yml b/roles/matrix-synapse/tasks/synapse/setup_install.yml index aea03f05..4d0e749f 100644 --- a/roles/matrix-synapse/tasks/synapse/setup_install.yml +++ b/roles/matrix-synapse/tasks/synapse/setup_install.yml @@ -62,6 +62,25 @@ delay: "{{ matrix_container_retries_delay }}" until: result is not failed +- when: "matrix_synapse_container_image_customizations_enabled | bool" + block: + - name: Ensure customizations Dockerfile is created + ansible.builtin.template: + src: "{{ role_path }}/templates/synapse/customizations/Dockerfile.j2" + dest: "{{ matrix_synapse_customized_docker_src_files_path }}/Dockerfile" + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + mode: 0640 + + - name: Ensure customized Docker image for Synapse is built + docker_image: + name: "{{ matrix_synapse_docker_image_customized }}" + source: build + build: + dockerfile: Dockerfile + path: "{{ matrix_synapse_customized_docker_src_files_path }}" + pull: true + - name: Check if a Synapse signing key exists ansible.builtin.stat: path: "{{ matrix_synapse_config_dir_path }}/{{ matrix_server_fqn_matrix }}.signing.key" diff --git a/roles/matrix-synapse/tasks/synapse/setup_uninstall.yml b/roles/matrix-synapse/tasks/synapse/setup_uninstall.yml index 17b1b8c4..06e55014 100644 --- a/roles/matrix-synapse/tasks/synapse/setup_uninstall.yml +++ b/roles/matrix-synapse/tasks/synapse/setup_uninstall.yml @@ -27,8 +27,11 @@ - name: Ensure Synapse Docker image doesn't exist docker_image: - name: "{{ matrix_synapse_docker_image }}" + name: "{{ item }}" state: absent + with_items: + - "{{ matrix_synapse_docker_image_final }}" + - "{{ matrix_synapse_docker_image }}" - name: Ensure sample prometheus.yml for external scraping is deleted ansible.builtin.file: diff --git a/roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 b/roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 new file mode 100644 index 00000000..7cce2086 --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 @@ -0,0 +1,3 @@ +FROM {{ matrix_synapse_docker_image }} + +{{ matrix_synapse_container_image_customizations_dockerfile_body_custom }} diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 index 96b65a0a..3855b850 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 @@ -42,7 +42,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name {{ matrix_synapse_wor {% for arg in matrix_synapse_container_arguments %} {{ arg }} \ {% endfor %} - {{ matrix_synapse_docker_image }} \ + {{ matrix_synapse_docker_image_final }} \ run -m synapse.app.{{ matrix_synapse_worker_details.app }} -c /data/homeserver.yaml -c /data/{{ matrix_synapse_worker_config_file_name }} diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 index eed50ad1..f41cc54c 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 @@ -60,7 +60,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \ {% for arg in matrix_synapse_container_arguments %} {{ arg }} \ {% endfor %} - {{ matrix_synapse_docker_image }} \ + {{ matrix_synapse_docker_image_final }} \ run -m synapse.app.homeserver -c /data/homeserver.yaml ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-synapse 2>/dev/null || true' From ac7cb3619dfe522af280875566e859007f7a0374 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 17:41:41 +0300 Subject: [PATCH 021/118] Add support for synapse-s3-storage-provider --- docs/configuring-playbook-s3-goofys.md | 137 +++++++++++++ docs/configuring-playbook-s3.md | 193 +++++------------- ...ng-playbook-synapse-s3-storage-provider.md | 104 ++++++++++ roles/matrix-synapse/defaults/main.yml | 35 ++++ .../tasks/ext/s3-storage-provider/init.yml | 5 + .../tasks/ext/s3-storage-provider/setup.yml | 10 + .../ext/s3-storage-provider/setup_install.yml | 54 +++++ .../s3-storage-provider/setup_uninstall.yml | 24 +++ .../s3-storage-provider/validate_config.yml | 18 ++ roles/matrix-synapse/tasks/ext/setup.yml | 2 + roles/matrix-synapse/tasks/init.yml | 3 + roles/matrix-synapse/tasks/setup_synapse.yml | 1 + .../synapse/customizations/Dockerfile.j2 | 4 + .../ext/s3-storage-provider/database.yaml.j2 | 5 + .../synapse/ext/s3-storage-provider/env.j2 | 16 ++ .../media_storage_provider.yaml.j2 | 14 ++ ...pse-s3-storage-provider-migrate.service.j2 | 7 + ...napse-s3-storage-provider-migrate.timer.j2 | 10 + ...rix-synapse-s3-storage-provider-migrate.j2 | 13 ++ ...atrix-synapse-s3-storage-provider-shell.j2 | 13 ++ 20 files changed, 528 insertions(+), 140 deletions(-) create mode 100644 docs/configuring-playbook-s3-goofys.md create mode 100644 docs/configuring-playbook-synapse-s3-storage-provider.md create mode 100644 roles/matrix-synapse/tasks/ext/s3-storage-provider/init.yml create mode 100644 roles/matrix-synapse/tasks/ext/s3-storage-provider/setup.yml create mode 100644 roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_install.yml create mode 100644 roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_uninstall.yml create mode 100644 roles/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml create mode 100644 roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/database.yaml.j2 create mode 100644 roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 create mode 100644 roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 create mode 100644 roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.service.j2 create mode 100644 roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.timer.j2 create mode 100644 roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 create mode 100644 roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-shell.j2 diff --git a/docs/configuring-playbook-s3-goofys.md b/docs/configuring-playbook-s3-goofys.md new file mode 100644 index 00000000..2efacddc --- /dev/null +++ b/docs/configuring-playbook-s3-goofys.md @@ -0,0 +1,137 @@ +# Storing Matrix media files on Amazon S3 with Goofys (optional) + +If you'd like to store Synapse's content repository (`media_store`) files on Amazon S3 (or other S3-compatible service), +you can let this playbook configure [Goofys](https://github.com/kahing/goofys) for you. + +Another (and better performing) way to use S3 storage with Synapse is [synapse-s3-storage-provider](configuring-playbook-synapse-s3-storage-provider.md). + +Using a Goofys-backed media store works, but performance may not be ideal. If possible, try to use a region which is close to your Matrix server. + +If you'd like to move your locally-stored media store data to Amazon S3 (or another S3-compatible object store), we also provide some migration instructions below. + + +## Usage + +After [creating the S3 bucket and configuring it](configuring-playbook-s3.md#bucket-creation-and-security-configuration), you can proceed to configure Goofys in your configuration file (`inventory/host_vars/matrix./vars.yml`): + +```yaml +matrix_s3_media_store_enabled: true +matrix_s3_media_store_bucket_name: "your-bucket-name" +matrix_s3_media_store_aws_access_key: "access-key-goes-here" +matrix_s3_media_store_aws_secret_key: "secret-key-goes-here" +matrix_s3_media_store_region: "eu-central-1" +``` + +You can use any S3-compatible object store by **additionally** configuring these variables: + +```yaml +matrix_s3_media_store_custom_endpoint_enabled: true +matrix_s3_media_store_custom_endpoint: "https://your-custom-endpoint" +``` + +If you have local media store files and wish to migrate to Backblaze B2 subsequently, follow our [migration guide to Backblaze B2](#migrating-to-backblaze-b2) below instead of applying this configuration as-is. + + +## Migrating from local filesystem storage to S3 + +It's a good idea to [make a complete server backup](faq.md#how-do-i-backup-the-data-on-my-server) before migrating your local media store to an S3-backed one. + +Follow one of the guides below for a migration path from a locally-stored media store to one stored on S3-compatible storage: + +- [Storing Matrix media files on Amazon S3 with Goofys (optional)](#storing-matrix-media-files-on-amazon-s3-with-goofys-optional) + - [Usage](#usage) + - [Migrating from local filesystem storage to S3](#migrating-from-local-filesystem-storage-to-s3) + - [Migrating to any S3-compatible storage (universal, but likely slow)](#migrating-to-any-s3-compatible-storage-universal-but-likely-slow) + - [Migrating to Backblaze B2](#migrating-to-backblaze-b2) + +### Migrating to any S3-compatible storage (universal, but likely slow) + +It's a good idea to [make a complete server backup](faq.md#how-do-i-backup-the-data-on-my-server) before doing this. + +1. Proceed with the steps below without stopping Matrix services + +2. Start by adding the base S3 configuration in your `vars.yml` file (seen above, may be different depending on the S3 provider of your choice) + +3. In addition to the base configuration you see above, add this to your `vars.yml` file: + +```yaml +matrix_s3_media_store_path: /matrix/s3-media-store +``` + +This enables S3 support, but mounts the S3 storage bucket to `/matrix/s3-media-store` without hooking it to your homeserver yet. Your homeserver will still continue using your local filesystem for its media store. + +5. Run the playbook to apply the changes: `ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start` + +6. Do an **initial sync of your files** by running this **on the server** (it may take a very long time): + +```sh +sudo -u matrix -- rsync --size-only --ignore-existing -avr /matrix/synapse/storage/media-store/. /matrix/s3-media-store/. +``` + +You may need to install `rsync` manually. + +7. Stop all Matrix services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) + +8. Start the S3 service by running this **on the server**: `systemctl start matrix-goofys` + +9. Sync the files again by re-running the `rsync` command you see in step #6 + +10. Stop the S3 service by running this **on the server**: `systemctl stop matrix-goofys` + +11. Get the old media store out of the way by running this command on the server: + +```sh +mv /matrix/synapse/storage/media-store /matrix/synapse/storage/media-store-local-backup +``` + +12. Remove the `matrix_s3_media_store_path` configuration from your `vars.yml` file (undoing step #3 above) + +13. Run the playbook: `ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start` + +14. You're done! Verify that loading existing (old) media files works and that you can upload new ones. + +15. When confident that it all works, get rid of the local media store directory: `rm -rf /matrix/synapse/storage/media-store-local-backup` + + +### Migrating to Backblaze B2 + +It's a good idea to [make a complete server backup](faq.md#how-do-i-backup-the-data-on-my-server) before doing this. + +1. While all Matrix services are running, run the following command on the server: + +(you need to adjust the 3 `--env` line below with your own data) + +```sh +docker run -it --rm -w /work \ +--env='B2_KEY_ID=YOUR_KEY_GOES_HERE' \ +--env='B2_KEY_SECRET=YOUR_SECRET_GOES_HERE' \ +--env='B2_BUCKET_NAME=YOUR_BUCKET_NAME_GOES_HERE' \ +-v /matrix/synapse/storage/media-store/:/work \ +--entrypoint=/bin/sh \ +docker.io/tianon/backblaze-b2:2.1.0 \ +-c 'b2 authorize-account $B2_KEY_ID $B2_KEY_SECRET > /dev/null && b2 sync /work/ b2://$B2_BUCKET_NAME' +``` + +This is some initial file sync, which may take a very long time. + +2. Stop all Matrix services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) + +3. Run the command from step #1 again. + +Doing this will sync any new files that may have been created locally in the meantime. + +Now that Matrix services aren't running, we're sure to get Backblaze B2 and your local media store fully in sync. + +4. Get the old media store out of the way by running this command on the server: + +```sh +mv /matrix/synapse/storage/media-store /matrix/synapse/storage/media-store-local-backup +``` + +5. Put the [Backblaze B2 settings seen above](#backblaze-b2) in your `vars.yml` file + +6. Run the playbook: `ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start` + +7. You're done! Verify that loading existing (old) media files works and that you can upload new ones. + +8. When confident that it all works, get rid of the local media store directory: `rm -rf /matrix/synapse/storage/media-store-local-backup` diff --git a/docs/configuring-playbook-s3.md b/docs/configuring-playbook-s3.md index 43aaa879..539f96d3 100644 --- a/docs/configuring-playbook-s3.md +++ b/docs/configuring-playbook-s3.md @@ -1,15 +1,44 @@ -# Storing Matrix media files on Amazon S3 (optional) +# Storing Synapse media files on Amazon S3 or another compatible Object Storage (optional) By default, this playbook configures your server to store Synapse's content repository (`media_store`) files on the local filesystem. If that's alright, you can skip this. -If you'd like to store Synapse's content repository (`media_store`) files on Amazon S3 (or other S3-compatible service), -you can let this playbook configure [Goofys](https://github.com/kahing/goofys) for you. +As an alternative to storing media files on the local filesystem, you can store them on [Amazon S3](https://aws.amazon.com/s3/) or another S3-compatible object store. -Using a Goofys-backed media store works, but performance may not be ideal. If possible, try to use a region which is close to your Matrix server. +First, [choose an Object Storage provider](#choosing-an-object-storage-provider). -If you'd like to move your locally-stored media store data to Amazon S3 (or another S3-compatible object store), we also provide some migration instructions below. +Then, [create the S3 bucket](#bucket-creation-and-security-configuration). +Finally, [set up S3 storage for Synapse](#setting-up) (with [Goofys](configuring-playbook-s3-goofys.md) or [synapse-s3-storage-provider](configuring-playbook-synapse-s3-storage-provider.md)). + + +## Choosing an Object Storage provider + +You can create [Amazon S3](https://aws.amazon.com/s3/) or another S3-compatible object store like [Backblaze B2](https://www.backblaze.com/b2/cloud-storage.html), [Wasabi](https://wasabi.com), [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces), etc. + +Amazon S3 and Backblaze S3 are pay-as-you with no minimum charges for storing too little data. + +All these providers have different prices, with Backblaze B2 appearing to be the cheapest. + +Wasabi has a minimum charge of 1TB if you're storing less than 1TB, which becomes expensive if you need to store less data than that. + +Digital Ocean Spaces has a minimum charge of 250GB ($5/month as of 2022-10), which is also expensive if you're storing less data than that. + +Important aspects of choosing the right provider are: + +- a provider by a company you like and trust (or dislike less than the others) +- a provider which has a data region close to your Matrix server (if it's farther away, high latency may cause slowdowns) +- a provider which is OK pricewise +- a provider with free or cheap egress (if you need to get the data out often, for some reason) - likely not too important for the common use-case + + +## Bucket creation and Security Configuration + +Now that you've [chosen an Object Storage provider](#choosing-an-object-storage-provider), you need to create a storage bucket. + +How you do this varies from provider to provider, with Amazon S3 being the most complicated due to its vast number of services and complicated security policies. + +Below, we provider some guides for common providers. If you don't see yours, look at the others for inspiration or read some guides online about how to create a bucket. Feel free to contribute to this documentation with an update! ## Amazon S3 @@ -34,161 +63,45 @@ You'll need an Amazon S3 bucket and some IAM user credentials (access key + secr } ``` -**NOTE**: This policy needs to be attached to an IAM user creted from the **Security Credentials** menu. This is not a **Bucket Policy**. - -You then need to enable S3 support in your configuration file (`inventory/host_vars/matrix./vars.yml`). -It would be something like this: - -```yaml -matrix_s3_media_store_enabled: true -matrix_s3_media_store_bucket_name: "your-bucket-name" -matrix_s3_media_store_aws_access_key: "access-key-goes-here" -matrix_s3_media_store_aws_secret_key: "secret-key-goes-here" -matrix_s3_media_store_region: "eu-central-1" -``` - +**NOTE**: This policy needs to be attached to an IAM user created from the **Security Credentials** menu. This is not a **Bucket Policy**. -## Using other S3-compatible object stores -You can use any S3-compatible object store by **additionally** configuring these variables: +## Backblaze B2 -```yaml -matrix_s3_media_store_custom_endpoint_enabled: true -# Example: "https://storage.googleapis.com" -matrix_s3_media_store_custom_endpoint: "your-custom-endpoint" -``` +To use [Backblaze B2](https://www.backblaze.com/b2/cloud-storage.html) you first need to sign up. -### Backblaze B2 +You [can't easily change which region (US, Europe) your Backblaze account stores files in](https://old.reddit.com/r/backblaze/comments/hi1v90/make_the_choice_for_the_b2_data_center_region/), so make sure to carefully choose the region when signing up (hint: it's a hard to see dropdown below the username/password fields in the signup form). -To use [Backblaze B2](https://www.backblaze.com/b2/cloud-storage.html): +After logging in to Backblaze: - create a new **private** bucket through its user interface (you can call it something like `matrix-DOMAIN-media-store`) -- note the **Endpoint** for your bucket (something like `s3.us-west-002.backblazeb2.com`) -- adjust its lifecycle rules to use the following **custom** rules: - - File Path: *empty value* - - Days Till Hide: *empty value* - - Days Till Delete: `1` +- note the **Endpoint** for your bucket (something like `s3.us-west-002.backblazeb2.com`). +- adjust its Lifecycle Rules to: Keep only the last version of the file - go to [App Keys](https://secure.backblaze.com/app_keys.htm) and use the **Add a New Application Key** to create a new one - restrict it to the previously created bucket (e.g. `matrix-DOMAIN-media-store`) - give it *Read & Write* access -Copy the `keyID` and `applicationKey`. - -You need the following *additional* playbook configuration (on top of what you see above): - -```yaml -matrix_s3_media_store_bucket_name: "YOUR_BUCKET_NAME_GOES_HERE" -matrix_s3_media_store_aws_access_key: "YOUR_keyID_GOES_HERE" -matrix_s3_media_store_aws_secret_key: "YOUR_applicationKey_GOES_HERE" -matrix_s3_media_store_custom_endpoint_enabled: true -matrix_s3_media_store_custom_endpoint: "https://s3.us-west-002.backblazeb2.com" # this may be different for your bucket -``` - -If you have local media store files and wish to migrate to Backblaze B2 subsequently, follow our [migration guide to Backblaze B2](#migrating-to-backblaze-b2) below instead of applying this configuration as-is. +The `keyID` value is your **Access Key** and `applicationKey` is your **Secret Key**. +For configuring [Goofys](configuring-playbook-s3-goofys.md) or [s3-synapse-storage-provider](configuring-playbook-synapse-s3-storage-provider.md) you will need: -## Migrating from local filesystem storage to S3 +- **Endpoint URL** - this is the **Endpoint** value you saw above, but prefixed with `https://` -It's a good idea to [make a complete server backup](faq.md#how-do-i-backup-the-data-on-my-server) before migrating your local media store to an S3-backed one. +- **Region** - use the value you see in the Endpoint (e.g. `us-west-002`) -Follow one of the guides below for a migration path from a locally-stored media store to one stored on S3-compatible storage: +- **Storage Class** - use `STANDARD`. Backblaze B2 does not have different storage classes, so it doesn't make sense to use any other value. -- [Storing Matrix media files on Amazon S3 (optional)](#storing-matrix-media-files-on-amazon-s3-optional) - - [Amazon S3](#amazon-s3) - - [Using other S3-compatible object stores](#using-other-s3-compatible-object-stores) - - [Backblaze B2](#backblaze-b2) - - [Migrating from local filesystem storage to S3](#migrating-from-local-filesystem-storage-to-s3) - - [Migrating to any S3-compatible storage (universal, but likely slow)](#migrating-to-any-s3-compatible-storage-universal-but-likely-slow) - - [Migrating to Backblaze B2](#migrating-to-backblaze-b2) - -### Migrating to any S3-compatible storage (universal, but likely slow) - -It's a good idea to [make a complete server backup](faq.md#how-do-i-backup-the-data-on-my-server) before doing this. - -1. Proceed with the steps below without stopping Matrix services - -2. Start by adding the base S3 configuration in your `vars.yml` file (seen above, may be different depending on the S3 provider of your choice) - -3. In addition to the base configuration you see above, add this to your `vars.yml` file: - -```yaml -matrix_s3_media_store_path: /matrix/s3-media-store -``` -This enables S3 support, but mounts the S3 storage bucket to `/matrix/s3-media-store` without hooking it to your homeserver yet. Your homeserver will still continue using your local filesystem for its media store. +## Other providers -5. Run the playbook to apply the changes: `ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start` +For other S3-compatible providers, you may not need to configure security policies, etc. (just like for [Backblaze B2](#backblaze-b2)). -6. Do an **initial sync of your files** by running this **on the server** (it may take a very long time): - -```sh -sudo -u matrix -- rsync --size-only --ignore-existing -avr /matrix/synapse/storage/media-store/. /matrix/s3-media-store/. -``` - -You may need to install `rsync` manually. - -7. Stop all Matrix services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - -8. Start the S3 service by running this **on the server**: `systemctl start matrix-goofys` - -9. Sync the files again by re-running the `rsync` command you see in step #6 - -10. Stop the S3 service by running this **on the server**: `systemctl stop matrix-goofys` - -11. Get the old media store out of the way by running this command on the server: - -```sh -mv /matrix/synapse/storage/media-store /matrix/synapse/storage/media-store-local-backup -``` - -12. Remove the `matrix_s3_media_store_path` configuration from your `vars.yml` file (undoing step #3 above) - -13. Run the playbook: `ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start` - -14. You're done! Verify that loading existing (old) media files works and that you can upload new ones. - -15. When confident that it all works, get rid of the local media store directory: `rm -rf /matrix/synapse/storage/media-store-local-backup` - - -### Migrating to Backblaze B2 - -It's a good idea to [make a complete server backup](faq.md#how-do-i-backup-the-data-on-my-server) before doing this. - -1. While all Matrix services are running, run the following command on the server: - -(you need to adjust the 3 `--env` line below with your own data) - -```sh -docker run -it --rm -w /work \ ---env='B2_KEY_ID=YOUR_KEY_GOES_HERE' \ ---env='B2_KEY_SECRET=YOUR_SECRET_GOES_HERE' \ ---env='B2_BUCKET_NAME=YOUR_BUCKET_NAME_GOES_HERE' \ --v /matrix/synapse/storage/media-store/:/work \ ---entrypoint=/bin/sh \ -docker.io/tianon/backblaze-b2:2.1.0 \ --c 'b2 authorize-account $B2_KEY_ID $B2_KEY_SECRET > /dev/null && b2 sync /work/ b2://$B2_BUCKET_NAME' -``` - -This is some initial file sync, which may take a very long time. - -2. Stop all Matrix services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - -3. Run the command from step #1 again. - -Doing this will sync any new files that may have been created locally in the meantime. - -Now that Matrix services aren't running, we're sure to get Backblaze B2 and your local media store fully in sync. - -4. Get the old media store out of the way by running this command on the server: - -```sh -mv /matrix/synapse/storage/media-store /matrix/synapse/storage/media-store-local-backup -``` +You most likely just need to create an S3 bucket and get some credentials (access key and secret key) for accessing the bucket in a read/write manner. -5. Put the [Backblaze B2 settings seen above](#backblaze-b2) in your `vars.yml` file -6. Run the playbook: `ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start` +## Setting up -7. You're done! Verify that loading existing (old) media files works and that you can upload new ones. +To set up Synapse to store files in S3, follow the instructions for the method of your choice: -8. When confident that it all works, get rid of the local media store directory: `rm -rf /matrix/synapse/storage/media-store-local-backup` +- using [synapse-s3-storage-provider](configuring-playbook-synapse-s3-storage-provider.md) (recommended) +- using [Goofys to mount the S3 store to the local filesystem](configuring-playbook-s3-goofys.md) diff --git a/docs/configuring-playbook-synapse-s3-storage-provider.md b/docs/configuring-playbook-synapse-s3-storage-provider.md new file mode 100644 index 00000000..bc0250f1 --- /dev/null +++ b/docs/configuring-playbook-synapse-s3-storage-provider.md @@ -0,0 +1,104 @@ +# Storing Synapse media files on Amazon S3 with synapse-s3-storage-provider (optional) + +If you'd like to store Synapse's content repository (`media_store`) files on Amazon S3 (or other S3-compatible service), +you can use the [synapse-s3-storage-provider](https://github.com/matrix-org/synapse-s3-storage-provider) media provider module for Synapse. + +An alternative (which has worse performance) is to use [Goofys to mount the S3 store to the local filesystem](configuring-playbook-s3-goofys.md). + + +## How it works? + +Summarized writings here are inspired by [this article](https://quentin.dufour.io/blog/2021-09-14/matrix-synapse-s3-storage/). + +The way media storage providers in Synapse work has some caveats: + +- Synapse still continues to use locally-stored files (for creating thumbnails, serving files, etc) +- the media storage provider is just an extra storage mechanism (in addition to the local filesystem) +- all files are stored locally at first, and then copied to the media storage provider (either synchronously or asynchronously) +- if a file is not available on the local filesystem, it's pulled from a media storage provider + +You may be thinking **if all files are stored locally as well, what's the point**? + +You can run some scripts to delete the local files once in a while, thus freeing up local disk space. If these files are needed in the future (for serving them to users, etc.), Synapse will pull them from the media storage provider on demand. + +While you will need some local disk space around, it's only to accommodate usage, etc., and won't grow as large as your S3 store. + + +## Installing + +After [creating the S3 bucket and configuring it](configuring-playbook-s3.md#bucket-creation-and-security-configuration), you can proceed to configure Goofys in your configuration file (`inventory/host_vars/matrix./vars.yml`): + +```yaml +matrix_synapse_ext_synapse_s3_storage_provider_enabled: true +matrix_synapse_ext_synapse_s3_storage_provider_config_bucket: your-bucket-name +matrix_synapse_ext_synapse_s3_storage_provider_config_region_name: some-region-name # e.g. eu-central-1 +matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url: https://.. # delete this whole line for Amazon S3 +matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id: access-key-goes-here +matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key: secret-key-goes-here +matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class: STANDARD # or STANDARD_IA, etc. + +# For additional advanced settings, take a look at `roles/matrix-synapse/defaults/main.yml` +``` + +If you have existing files in Synapse's media repository (`/matrix/synapse/media-store/..`): + +- new files will start being stored both locally and on the S3 store +- the existing files will remain on the local filesystem only until [migrating them to the S3 store](#migrating-your-existing-media-files-to-the-s3-store) +- at some point (and periodically in the future), you can delete local files which have been uploaded to the S3 store already + + +## Migrating your existing media files to the S3 store + +Migrating your existing data can happen in multiple ways: + +- [using the `s3_media_upload` script from `synapse-s3-storage-provider`](#using-the-s3_media_upload-script-from-synapse-s3-storage-provider) (very slow when dealing with lots of data) +- [using another tool in combination with `s3_media_upload`](#using-another-tool-in-combination-with-s3_media_upload) (quicker when dealing with lots of data) + +### Using the `s3_media_upload` script from `synapse-s3-storage-provider` + +Instead of using `s3_media_upload` directly, which is very slow and painful for an initial data migration, we recommend [using another tool in combination with `s3_media_upload`](#using-another-tool-in-combination-with-s3_media_upload). + +To copy your existing files, SSH into the server and run `/usr/local/bin/matrix-synapse-s3-storage-provider-shell`. + +This launches a Synapse container, which has access to the local media store, Postgres database, S3 store and has some convenient environment variables configured for you to use (`MEDIA_PATH`, `BUCKET`, `ENDPOINT`, `UPDATE_DB_DAYS`, etc). + +Then use the following commands (`$` values come from environment variables - they're **not placeholders** that you need to substitute): + +- `s3_media_upload update-db $UPDATE_DB_DURATION` - create a local SQLite database (`cache.db`) with a list of media repository files (from the `synapse` Postgres database) eligible for operating on + - `$UPDATE_DB_DURATION` is influenced by the `matrix_synapse_ext_synapse_s3_storage_provider_update_db_day_count` variable (defaults to `0`) + - `$UPDATE_DB_DURATION` defaults to `0d` (0 days), which means **include files which haven't been accessed for more than 0 days** (that is, **all files will be included**). +- `s3_media_upload check-deleted $MEDIA_PATH` - check whether files in the local cache still exist in the local media repository directory +- `s3_media_upload upload $MEDIA_PATH $BUCKET --delete --endpoint-url $ENDPOINT` - uploads locally-stored files to S3 and deletes them from the local media repository directory + +The `upload` command may take a lot of time to complete. + + +### Using another tool in combination with `s3_media_upload` + +To migrate your existing local data to S3, we recommend to: + +- **first** use another tool ([`aws s3`](#copying-data-to-amazon-s3) or [`b2 sync`](#copying-data-to-backblaze-b2), etc.) to copy the local files to the S3 bucket + +- **only then** [use the `s3_media_upload` tool to finish the migration](#using-the-s3_media_upload-script-from-synapse-s3-storage-provider) (this checks to ensure all files are uploaded and then deletes the local files) + +#### Copying data to Amazon S3 + +Generally, you need to use the `aws s3` tool. + +This documentation section could use an improvement. Ideally, we'd come up with a guide like the one used in [Copying data to Backblaze B2](#copying-data-to-backblaze-b2) - running `aws s3` in a container, etc. + +#### Copying data to Backblaze B2 + +To copy to Backblaze B2, start a container like this: + +```sh +docker run -it --rm \ +-w /work \ +--env='B2_KEY_ID=YOUR_KEY_GOES_HERE' \ +--env='B2_KEY_SECRET=YOUR_SECRET_GOES_HERE' \ +--env='B2_BUCKET_NAME=YOUR_BUCKET_NAME_GOES_HERE' \ +--mount type=bind,src=/matrix/synapse/storage/media-store,dst=/work,ro \ +--entrypoint=/bin/sh \ +tianon/backblaze-b2:3.6.0 \ +-c 'b2 authorize-account $B2_KEY_ID $B2_KEY_SECRET > /dev/null && b2 sync /work b2://$B2_BUCKET_NAME --skipNewer' +``` diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 40e05be7..383e67ab 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -19,6 +19,10 @@ matrix_synapse_container_image_self_build_repo: "https://github.com/matrix-org/s # - `matrix_synapse_docker_image_final` matrix_synapse_container_image_customizations_enabled: "{{ matrix_synapse_ext_synapse_s3_storage_provider_enabled }}" +# Controls whether custom build steps will be added to the Dockerfile for installing s3-storage-provider. +# The version that will be installed is specified in `matrix_synapse_ext_synapse_s3_storage_provider_version`. +matrix_synapse_container_image_customizations_s3_storage_provider_installation_enabled: "{{ matrix_synapse_ext_synapse_s3_storage_provider_enabled }}" + # matrix_synapse_container_image_customizations_dockerfile_body contains your custom Dockerfile steps # for building your customized Synapse image based on the original (upstream) image (`matrix_synapse_docker_image`). # A `FROM ...` clause is included automatically so you don't have to. @@ -52,6 +56,7 @@ matrix_synapse_config_dir_path: "{{ matrix_synapse_base_path }}/config" matrix_synapse_storage_path: "{{ matrix_synapse_base_path }}/storage" matrix_synapse_media_store_path: "{{ matrix_synapse_storage_path }}/media-store" matrix_synapse_ext_path: "{{ matrix_synapse_base_path }}/ext" +matrix_synapse_ext_s3_storage_provider_path: "{{ matrix_synapse_base_path }}/ext/s3-storage-provider" matrix_synapse_container_client_api_port: 8008 @@ -787,6 +792,32 @@ matrix_synapse_ext_encryption_config_yaml: | patch_power_levels: {{ matrix_synapse_ext_encryption_disabler_patch_power_levels | to_json }} +# matrix_synapse_ext_synapse_s3_storage_provider_enabled controls whether to enable https://github.com/matrix-org/synapse-s3-storage-provider +# Installing it requires building a customized Docker image for Synapse (see `matrix_synapse_container_image_customizations_enabled`). +# Enabling this will enable customizations and inject the appropriate Dockerfile clauses for installing synapse-s3-storage-provider. +matrix_synapse_ext_synapse_s3_storage_provider_enabled: false +matrix_synapse_ext_synapse_s3_storage_provider_version: 1.1.2 +# Controls whether media from this (local) server is stored in s3-storage-provider +matrix_synapse_ext_synapse_s3_storage_provider_store_local: true +# Controls whether media from remote servers is stored in s3-storage-provider +matrix_synapse_ext_synapse_s3_storage_provider_store_remote: true +# Controls whether files are stored to S3 at the same time they are stored on the local filesystem. +# For slightly improved reliability, consider setting this to `true`. +# Even with asynchronous uploading to S3 (`false` value), data loss shouldn't be possible, +# because the local filesystem is a reliable data store anyway. +matrix_synapse_ext_synapse_s3_storage_provider_store_synchronous: false +matrix_synapse_ext_synapse_s3_storage_provider_config_bucket: '' +matrix_synapse_ext_synapse_s3_storage_provider_config_region_name: '' +matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url: '' +matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id: '' +matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key: '' +matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class: STANDARD +matrix_synapse_ext_synapse_s3_storage_provider_config_threadpool_size: 40 +# matrix_synapse_ext_synapse_s3_storage_provider_update_db_day_count is a day value (number) for the `s3_media_upload update-db` command. +# It specifies how old files need to have been inactive to be eligible for migration from the local filesystem to the S3 data store. +# By default, we use `0` which says "all files are eligible for migration". +matrix_synapse_ext_synapse_s3_storage_provider_update_db_day_count: 0 + matrix_s3_media_store_enabled: false matrix_s3_media_store_custom_endpoint_enabled: false matrix_s3_goofys_docker_image: "ewoutp/goofys:latest" @@ -839,6 +870,10 @@ matrix_synapse_media_storage_providers: "{{ matrix_synapse_media_storage_provide matrix_synapse_media_storage_providers_auto: | {{ [] + + + [ + lookup('ansible.builtin.template', role_path + '/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2') | from_yaml + ] if matrix_synapse_ext_synapse_s3_storage_provider_enabled else [] }} # matrix_synapse_media_storage_providers_custom contains your own custom list of storage providers. diff --git a/roles/matrix-synapse/tasks/ext/s3-storage-provider/init.yml b/roles/matrix-synapse/tasks/ext/s3-storage-provider/init.yml new file mode 100644 index 00000000..008161cb --- /dev/null +++ b/roles/matrix-synapse/tasks/ext/s3-storage-provider/init.yml @@ -0,0 +1,5 @@ +--- + +- ansible.builtin.set_fact: + matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-synapse-s3-storage-provider-migrate.timer'] }}" + when: matrix_synapse_ext_synapse_s3_storage_provider_enabled | bool diff --git a/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup.yml b/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup.yml new file mode 100644 index 00000000..aefa49fe --- /dev/null +++ b/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup.yml @@ -0,0 +1,10 @@ +--- + +- ansible.builtin.import_tasks: "{{ role_path }}/tasks/ext/s3-storage-provider/validate_config.yml" + when: matrix_synapse_ext_synapse_s3_storage_provider_enabled | bool + +- ansible.builtin.import_tasks: "{{ role_path }}/tasks/ext/s3-storage-provider/setup_install.yml" + when: matrix_synapse_ext_synapse_s3_storage_provider_enabled | bool + +- ansible.builtin.import_tasks: "{{ role_path }}/tasks/ext/s3-storage-provider/setup_uninstall.yml" + when: not matrix_synapse_ext_synapse_s3_storage_provider_enabled | bool diff --git a/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_install.yml b/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_install.yml new file mode 100644 index 00000000..31f72181 --- /dev/null +++ b/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_install.yml @@ -0,0 +1,54 @@ +--- + +# We install this into Synapse by making `matrix_synapse_ext_synapse_s3_storage_provider_enabled` influence other variables: +# - `matrix_synapse_media_storage_providers` (via `matrix_synapse_media_storage_providers_auto`) +# - `matrix_synapse_container_image_customizations_enabled` +# - `matrix_synapse_container_image_customizations_s3_storage_provider_installation_enabled` +# +# Below are additional tasks for setting up various helper scripts, etc. + +- name: Ensure s3-storage-provider env file installed + ansible.builtin.template: + src: "{{ role_path }}/templates/synapse/ext/s3-storage-provider/env.j2" + dest: "{{ matrix_synapse_ext_s3_storage_provider_path }}/env" + mode: 0640 + +- name: Ensure s3-storage-provider data path exists + ansible.builtin.file: + path: "{{ matrix_synapse_ext_s3_storage_provider_path }}/data" + state: directory + mode: 0750 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + +- name: Ensure s3-storage-provider database.yaml file installed + ansible.builtin.template: + src: "{{ role_path }}/templates/synapse/ext/s3-storage-provider/database.yaml.j2" + dest: "{{ matrix_synapse_ext_s3_storage_provider_path }}/data/database.yaml" + mode: 0640 + +- name: Ensure s3-storage-provider scripts installed + ansible.builtin.template: + src: "{{ role_path }}/templates/synapse/ext/s3-storage-provider/usr-local-bin/{{ item }}.j2" + dest: "{{ matrix_local_bin_path }}/{{ item }}" + mode: 0750 + with_items: + - matrix-synapse-s3-storage-provider-shell + - matrix-synapse-s3-storage-provider-migrate + +- name: Ensure matrix-synapse-s3-storage-provider-migrate.service and timer are installed + ansible.builtin.template: + src: "{{ role_path }}/templates/systemd/.j2" + src: "{{ role_path }}/templates/synapse/ext/s3-storage-provider/systemd/{{ item }}.j2" + dest: "{{ matrix_systemd_path }}/{{ item }}" + mode: 0640 + with_items: + - matrix-synapse-s3-storage-provider-migrate.service + - matrix-synapse-s3-storage-provider-migrate.timer + register: matrix_synapse_s3_storage_provider_systemd_service_result + +- name: Ensure systemd reloaded after matrix-synapse-s3-storage-provider-migrate.service installation + ansible.builtin.service: + daemon_reload: true + when: matrix_synapse_s3_storage_provider_systemd_service_result.changed | bool + diff --git a/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_uninstall.yml b/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_uninstall.yml new file mode 100644 index 00000000..205a5541 --- /dev/null +++ b/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_uninstall.yml @@ -0,0 +1,24 @@ +--- + +- name: Ensure matrix-synapse-s3-storage-provider-migrate.service and timer don't exist + ansible.builtin.file: + path: "{{ matrix_systemd_path }}/{{ item }}" + state: absent + with_items: + - matrix-synapse-s3-storage-provider-migrate.timer + - matrix-synapse-s3-storage-provider-migrate.service + register: matrix_synapse_s3_storage_provider_migrate_sevice_removal + +- name: Ensure systemd reloaded after matrix-synapse-s3-storage-provider-migrate.service removal + ansible.builtin.service: + daemon_reload: true + when: matrix_synapse_s3_storage_provider_migrate_sevice_removal.changed | bool + +- name: Ensure s3-storage-provider files don't exist + ansible.builtin.file: + path: "{{ item }}" + state: absent + with_items: + - "{{ matrix_local_bin_path }}/matrix-synapse-s3-storage-provider-shell" + - "{{ matrix_local_bin_path }}/matrix-synapse-s3-storage-provider-migrate" + - "{{ matrix_synapse_ext_s3_storage_provider_path }}" diff --git a/roles/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml b/roles/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml new file mode 100644 index 00000000..d71809fe --- /dev/null +++ b/roles/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml @@ -0,0 +1,18 @@ +--- + +- name: Fail if required s3-storage-provider settings not defined + ansible.builtin.fail: + msg: >- + You need to define a required configuration setting (`{{ item }}`) for using s3-storage-provider. + when: "vars[item] == ''" + with_items: + - "matrix_synapse_ext_synapse_s3_storage_provider_config_bucket" + - "matrix_synapse_ext_synapse_s3_storage_provider_config_region_name" + - "matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id" + - "matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key" + +- name: Fail if required matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url looks invalid + ansible.builtin.fail: + msg: >- + `matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url` needs to look like a URL (`http://` or `https://` prefix). + when: "matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url != '' and not matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url.startswith('http')" diff --git a/roles/matrix-synapse/tasks/ext/setup.yml b/roles/matrix-synapse/tasks/ext/setup.yml index d944f257..6cf1afaa 100644 --- a/roles/matrix-synapse/tasks/ext/setup.yml +++ b/roles/matrix-synapse/tasks/ext/setup.yml @@ -11,3 +11,5 @@ - ansible.builtin.import_tasks: "{{ role_path }}/tasks/ext/synapse-simple-antispam/setup.yml" - ansible.builtin.import_tasks: "{{ role_path }}/tasks/ext/mjolnir-antispam/setup.yml" + +- ansible.builtin.import_tasks: "{{ role_path }}/tasks/ext/s3-storage-provider/setup.yml" diff --git a/roles/matrix-synapse/tasks/init.yml b/roles/matrix-synapse/tasks/init.yml index a77320c2..9146936a 100644 --- a/roles/matrix-synapse/tasks/init.yml +++ b/roles/matrix-synapse/tasks/init.yml @@ -26,6 +26,9 @@ matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-goofys.service'] }}" when: matrix_s3_media_store_enabled | bool +- ansible.builtin.include_tasks: "{{ role_path }}/tasks/ext/s3-storage-provider/init.yml" + when: matrix_synapse_ext_synapse_s3_storage_provider_enabled | bool + - when: matrix_synapse_enabled | bool and matrix_synapse_metrics_proxying_enabled | bool block: - name: Fail if matrix-nginx-proxy role already executed diff --git a/roles/matrix-synapse/tasks/setup_synapse.yml b/roles/matrix-synapse/tasks/setup_synapse.yml index 7b887f30..13a5819e 100644 --- a/roles/matrix-synapse/tasks/setup_synapse.yml +++ b/roles/matrix-synapse/tasks/setup_synapse.yml @@ -12,6 +12,7 @@ - {path: "{{ matrix_synapse_ext_path }}", when: true} - {path: "{{ matrix_synapse_docker_src_files_path }}", when: "{{ matrix_synapse_container_image_self_build }}"} - {path: "{{ matrix_synapse_customized_docker_src_files_path }}", when: "{{ matrix_synapse_container_image_customizations_enabled }}"} + - {path: "{{ matrix_synapse_ext_s3_storage_provider_path }}", when: "{{ matrix_synapse_ext_synapse_s3_storage_provider_enabled }}"} # We handle matrix_synapse_media_store_path elsewhere (in ./synapse/setup_install.yml), # because if it's using Goofys and it's already mounted (from before), # trying to chown/chmod it here will cause trouble. diff --git a/roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 b/roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 index 7cce2086..3919e955 100644 --- a/roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 +++ b/roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 @@ -1,3 +1,7 @@ FROM {{ matrix_synapse_docker_image }} +{% if matrix_synapse_container_image_customizations_s3_storage_provider_installation_enabled %} +RUN pip install synapse-s3-storage-provider=={{ matrix_synapse_ext_synapse_s3_storage_provider_version }} +{% endif %} + {{ matrix_synapse_container_image_customizations_dockerfile_body_custom }} diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/database.yaml.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/database.yaml.j2 new file mode 100644 index 00000000..ed11645e --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/database.yaml.j2 @@ -0,0 +1,5 @@ +user: {{ matrix_synapse_database_user | to_json }} +password: {{ matrix_synapse_database_password | to_json }} +database: {{ matrix_synapse_database_database | to_json }} +host: {{ matrix_synapse_database_host | to_json }} +port: {{ matrix_synapse_database_port | to_json }} diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 new file mode 100644 index 00000000..4b09688b --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 @@ -0,0 +1,16 @@ +AWS_ACCESS_KEY_ID={{ matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id }} +AWS_SECRET_ACCESS_KEY={{ matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key }} +AWS_DEFAULT_REGION={{ matrix_synapse_ext_synapse_s3_storage_provider_config_region_name }} + +ENDPOINT={{ matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url }} +BUCKET={{ matrix_synapse_ext_synapse_s3_storage_provider_config_bucket }} + +PG_USER={{ matrix_synapse_database_user }} +PG_PASS={{ matrix_synapse_database_password }} +PG_DB={{ matrix_synapse_database_database }} +PG_HOST={{ matrix_synapse_database_host }} +PG_PORT={{ matrix_synapse_database_port }} + +MEDIA_PATH=/matrix-media-store-parent/{{ matrix_synapse_media_store_directory_name }} + +UPDATE_DB_DURATION={{ matrix_synapse_ext_synapse_s3_storage_provider_update_db_day_count }}d diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 new file mode 100644 index 00000000..97b0f5f2 --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 @@ -0,0 +1,14 @@ +module: s3_storage_provider.S3StorageProviderBackend +store_local: {{ matrix_synapse_ext_synapse_s3_storage_provider_store_local | to_json }} +store_remote: {{ matrix_synapse_ext_synapse_s3_storage_provider_store_remote | to_json }} +store_synchronous: {{ matrix_synapse_ext_synapse_s3_storage_provider_store_synchronous | to_json }} +config: + bucket: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_bucket | to_json }} + region_name: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_region_name | to_json }} + endpoint_url: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url | to_json }} + access_key_id: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id | to_json }} + secret_access_key: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key | to_json }} + + storage_class: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class | to_json }} + + threadpool_size: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_threadpool_size | to_json }} diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.service.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.service.j2 new file mode 100644 index 00000000..ea8f0c8c --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.service.j2 @@ -0,0 +1,7 @@ +[Unit] +Description=Migrates locally-stored Synapse media store files to S3 + +[Service] +Type=oneshot +Environment="HOME={{ matrix_systemd_unit_home_path }}" +ExecStart={{ matrix_local_bin_path }}/matrix-synapse-s3-storage-provider-migrate diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.timer.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.timer.j2 new file mode 100644 index 00000000..61526ac1 --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Migrates locally-stored Synapse media store files to S3 + +[Timer] +Unit=matrix-synapse-s3-storage-provider-migrate.service +OnCalendar=*-*-* 05:00:00 +RandomizedDelaySec=2h + +[Install] +WantedBy=timers.target diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 new file mode 100644 index 00000000..0893f5d6 --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 @@ -0,0 +1,13 @@ +#jinja2: lstrip_blocks: "True" +#!/bin/bash + +{{ matrix_host_command_docker }} run \ + --rm \ + --env-file={{ matrix_synapse_ext_s3_storage_provider_path }}/env \ + --mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \ + --mount type=bind,src={{ matrix_synapse_ext_s3_storage_provider_path }}/data,dst=/data \ + --workdir=/data \ + --network={{ matrix_docker_network }} \ + --entrypoint=/bin/bash \ + {{ matrix_synapse_docker_image_final }} \ + -c 's3_media_upload update-db $UPDATE_DB_DURATION && s3_media_upload --no-progress check-deleted $MEDIA_PATH && s3_media_upload --no-progress upload $MEDIA_PATH $BUCKET --delete --endpoint-url $ENDPOINT' diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-shell.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-shell.j2 new file mode 100644 index 00000000..c67a6dda --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-shell.j2 @@ -0,0 +1,13 @@ +#jinja2: lstrip_blocks: "True" +#!/bin/bash + +{{ matrix_host_command_docker }} run \ + -it \ + --rm \ + --env-file={{ matrix_synapse_ext_s3_storage_provider_path }}/env \ + --mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \ + --mount type=bind,src={{ matrix_synapse_ext_s3_storage_provider_path }}/data,dst=/data \ + --workdir=/data \ + --network={{ matrix_docker_network }} \ + --entrypoint=/bin/bash \ + {{ matrix_synapse_docker_image_final }} From a3759b0466fac32d482942177d7e690990d799e9 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 17:50:48 +0300 Subject: [PATCH 022/118] Announce Synapse customization and synapse-s3-storage-provider support --- CHANGELOG.md | 33 ++++++++++++++++++++++++++ roles/matrix-synapse/defaults/main.yml | 1 - 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06664634..e99cd360 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,36 @@ +# 2022-10-14 + +## synapse-s3-storage-provider support + +You can now store your Synapse media repository files on Amazon S3 (or another S3-compatible object store) using [synapse-s3-storage-provider](https://github.com/matrix-org/synapse-s3-storage-provider) - a media provider for Synapse (Python module), which should work faster and more reliably than our previous [Goofys](docs/configuring-playbook-s3-goofys.md) implementation (Goofys will continue to work). + +This is not just for initial installations. Users with existing files (stored in the local filesystem) can also migrate their files to `synapse-s3-storage-provider`. + +To get started, see our [Storing Synapse media files on Amazon S3 with synapse-s3-storage-provider](docs/configuring-playbook-synapse-s3-storage-provider.md) documentation. + + +## Synapse container image customization support + +We now support customizing the Synapse container image by adding additional build steps to its [`Dockerfile`](https://docs.docker.com/engine/reference/builder/). + +Our [synapse-s3-storage-provider support](#synapse-s3-storage-provider-support) is actually built on this. When `s3-storage-provider` is enabled, we automatically add additional build steps to install its Python module into the Synapse image. + +Besides this kind of auto-added build steps (for components supported by the playbook), we also let you inject your own custom build steps using configuration like this: + +```yaml +matrix_synapse_container_image_customizations_enabled: true + +matrix_synapse_container_image_customizations_dockerfile_body_custom: | + RUN echo 'This is a custom step for building the customized Docker image for Synapse.' + RUN echo 'You can override matrix_synapse_container_image_customizations_dockerfile_body_custom to add your own steps.' + RUN echo 'You do NOT need to include a FROM clause yourself.' +``` + +People who have needed to customize Synapse previously had to fork the git repository, make their changes to the `Dockerfile` there, point the playbook to the new repository (`matrix_synapse_container_image_self_build_repo`) and enable self-building from scratch (`matrix_synapse_container_image_self_build: true`). This is harder and slower. + +With the new Synapse-customization feature in the playbook, we use the original upstream (pre-built, if available) Synapse image and only build on top of it, right on the Matrix server. This is much faster than building all of Synapse from scratch. + + # 2022-10-02 ## matrix-ldap-registration-proxy support diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 383e67ab..4b979159 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -31,7 +31,6 @@ matrix_synapse_container_image_customizations_s3_storage_provider_installation_e # matrix_synapse_container_image_customizations_dockerfile_body_custom: | # RUN echo 'This is a custom step for building the customized Docker image for Synapse.' # RUN echo 'You can override matrix_synapse_container_image_customizations_dockerfile_body_custom to add your own steps.' -# RUN echo 'Note that matrix_synapse_container_image_customizations_dockerfile_body_auto injects steps before this.' # RUN echo 'You do NOT need to include a FROM clause yourself.' matrix_synapse_container_image_customizations_dockerfile_body_custom: '' From 63a0e5c4f6b1b3384a8178ac6414712094736bc6 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 17:58:47 +0300 Subject: [PATCH 023/118] Add warnings to synapse-s3-storage-provider support feature --- CHANGELOG.md | 2 ++ docs/configuring-playbook-synapse-s3-storage-provider.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e99cd360..3ee9afac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## synapse-s3-storage-provider support +**`synapse-s3-storage-provider` support is very new and still relatively untested. Using it may cause data loss.** + You can now store your Synapse media repository files on Amazon S3 (or another S3-compatible object store) using [synapse-s3-storage-provider](https://github.com/matrix-org/synapse-s3-storage-provider) - a media provider for Synapse (Python module), which should work faster and more reliably than our previous [Goofys](docs/configuring-playbook-s3-goofys.md) implementation (Goofys will continue to work). This is not just for initial installations. Users with existing files (stored in the local filesystem) can also migrate their files to `synapse-s3-storage-provider`. diff --git a/docs/configuring-playbook-synapse-s3-storage-provider.md b/docs/configuring-playbook-synapse-s3-storage-provider.md index bc0250f1..3490705a 100644 --- a/docs/configuring-playbook-synapse-s3-storage-provider.md +++ b/docs/configuring-playbook-synapse-s3-storage-provider.md @@ -3,6 +3,8 @@ If you'd like to store Synapse's content repository (`media_store`) files on Amazon S3 (or other S3-compatible service), you can use the [synapse-s3-storage-provider](https://github.com/matrix-org/synapse-s3-storage-provider) media provider module for Synapse. +**`synapse-s3-storage-provider` support is very new and still relatively untested. Using it may cause data loss.** + An alternative (which has worse performance) is to use [Goofys to mount the S3 store to the local filesystem](configuring-playbook-s3-goofys.md). From fe9647559538756ca2f0bb547c621de638506c14 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 18:01:51 +0300 Subject: [PATCH 024/118] Remove some useless PG_ variables fron the s3-storage-provider env-file These Postgres configuration variables are part of the `database.yaml` file and don't need to be here. --- .../templates/synapse/ext/s3-storage-provider/env.j2 | 6 ------ 1 file changed, 6 deletions(-) diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 index 4b09688b..3788499b 100644 --- a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 @@ -5,12 +5,6 @@ AWS_DEFAULT_REGION={{ matrix_synapse_ext_synapse_s3_storage_provider_config_regi ENDPOINT={{ matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url }} BUCKET={{ matrix_synapse_ext_synapse_s3_storage_provider_config_bucket }} -PG_USER={{ matrix_synapse_database_user }} -PG_PASS={{ matrix_synapse_database_password }} -PG_DB={{ matrix_synapse_database_database }} -PG_HOST={{ matrix_synapse_database_host }} -PG_PORT={{ matrix_synapse_database_port }} - MEDIA_PATH=/matrix-media-store-parent/{{ matrix_synapse_media_store_directory_name }} UPDATE_DB_DURATION={{ matrix_synapse_ext_synapse_s3_storage_provider_update_db_day_count }}d From 2e0fc5c11c9d7dd165b09c8dc7bcbf8fcf6c2020 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 18:04:49 +0300 Subject: [PATCH 025/118] Fix some ansible-lint-reported warnings --- .../tasks/ext/s3-storage-provider/setup_install.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_install.yml b/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_install.yml index 31f72181..79684270 100644 --- a/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_install.yml +++ b/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_install.yml @@ -38,7 +38,6 @@ - name: Ensure matrix-synapse-s3-storage-provider-migrate.service and timer are installed ansible.builtin.template: - src: "{{ role_path }}/templates/systemd/.j2" src: "{{ role_path }}/templates/synapse/ext/s3-storage-provider/systemd/{{ item }}.j2" dest: "{{ matrix_systemd_path }}/{{ item }}" mode: 0640 @@ -51,4 +50,3 @@ ansible.builtin.service: daemon_reload: true when: matrix_synapse_s3_storage_provider_systemd_service_result.changed | bool - From 3cf0dcda6d39506cc71c249d3001ec30d826b554 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 18:07:10 +0300 Subject: [PATCH 026/118] Update "b2 sync" commands --- docs/configuring-playbook-s3-goofys.md | 6 +++--- docs/configuring-playbook-synapse-s3-storage-provider.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/configuring-playbook-s3-goofys.md b/docs/configuring-playbook-s3-goofys.md index 2efacddc..ef8f20c2 100644 --- a/docs/configuring-playbook-s3-goofys.md +++ b/docs/configuring-playbook-s3-goofys.md @@ -106,10 +106,10 @@ docker run -it --rm -w /work \ --env='B2_KEY_ID=YOUR_KEY_GOES_HERE' \ --env='B2_KEY_SECRET=YOUR_SECRET_GOES_HERE' \ --env='B2_BUCKET_NAME=YOUR_BUCKET_NAME_GOES_HERE' \ --v /matrix/synapse/storage/media-store/:/work \ +--mount type=bind,src=/matrix/synapse/storage/media-store,dst=/work,ro \ --entrypoint=/bin/sh \ -docker.io/tianon/backblaze-b2:2.1.0 \ --c 'b2 authorize-account $B2_KEY_ID $B2_KEY_SECRET > /dev/null && b2 sync /work/ b2://$B2_BUCKET_NAME' +docker.io/tianon/backblaze-b2:3.6.0 \ +-c 'b2 authorize-account $B2_KEY_ID $B2_KEY_SECRET && b2 sync /work b2://$B2_BUCKET_NAME --skipNewer' ``` This is some initial file sync, which may take a very long time. diff --git a/docs/configuring-playbook-synapse-s3-storage-provider.md b/docs/configuring-playbook-synapse-s3-storage-provider.md index 3490705a..7f2af49c 100644 --- a/docs/configuring-playbook-synapse-s3-storage-provider.md +++ b/docs/configuring-playbook-synapse-s3-storage-provider.md @@ -102,5 +102,5 @@ docker run -it --rm \ --mount type=bind,src=/matrix/synapse/storage/media-store,dst=/work,ro \ --entrypoint=/bin/sh \ tianon/backblaze-b2:3.6.0 \ --c 'b2 authorize-account $B2_KEY_ID $B2_KEY_SECRET > /dev/null && b2 sync /work b2://$B2_BUCKET_NAME --skipNewer' +-c 'b2 authorize-account $B2_KEY_ID $B2_KEY_SECRET && b2 sync /work b2://$B2_BUCKET_NAME --skipNewer' ``` From de8fd519c56e5905b729ce18c558cc478dc05de7 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 18:14:34 +0300 Subject: [PATCH 027/118] Document matrix-synapse-s3-storage-provider-migrate.{service,timer} --- ...iguring-playbook-synapse-s3-storage-provider.md | 14 +++++++++++++- ...ix-synapse-s3-storage-provider-migrate.timer.j2 | 1 - 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/configuring-playbook-synapse-s3-storage-provider.md b/docs/configuring-playbook-synapse-s3-storage-provider.md index 7f2af49c..6ab2820a 100644 --- a/docs/configuring-playbook-synapse-s3-storage-provider.md +++ b/docs/configuring-playbook-synapse-s3-storage-provider.md @@ -21,7 +21,7 @@ The way media storage providers in Synapse work has some caveats: You may be thinking **if all files are stored locally as well, what's the point**? -You can run some scripts to delete the local files once in a while, thus freeing up local disk space. If these files are needed in the future (for serving them to users, etc.), Synapse will pull them from the media storage provider on demand. +You can run some scripts to delete the local files once in a while (which we do automatically by default - see [Periodically cleaning up the local filesystem](#periodically-cleaning-up-the-local-filesystem)), thus freeing up local disk space. If these files are needed in the future (for serving them to users, etc.), Synapse will pull them from the media storage provider on demand. While you will need some local disk space around, it's only to accommodate usage, etc., and won't grow as large as your S3 store. @@ -104,3 +104,15 @@ docker run -it --rm \ tianon/backblaze-b2:3.6.0 \ -c 'b2 authorize-account $B2_KEY_ID $B2_KEY_SECRET && b2 sync /work b2://$B2_BUCKET_NAME --skipNewer' ``` + +## Periodically cleaning up the local filesystem + +As described in [How it works?](#how-it-works) above, when new media is uploaded to the Synapse homeserver, it's first stored locally and then also stored on the remote S3 storage. + +By default, we periodically ensure that all local files are uploaded to S3 and are then removed from the local filesystem. This is done automatically using: + +- the `/usr/local/bin/matrix-synapse-s3-storage-provider-migrate` script +- .. invoked via the `matrix-synapse-s3-storage-provider-migrate.service` service +- .. triggered by the `matrix-synapse-s3-storage-provider-migrate.timer` timer, every day at 05:00 + +So.. you don't need to perform any maintenance yourself. diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.timer.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.timer.j2 index 61526ac1..5013c7a8 100644 --- a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.timer.j2 +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.timer.j2 @@ -4,7 +4,6 @@ Description=Migrates locally-stored Synapse media store files to S3 [Timer] Unit=matrix-synapse-s3-storage-provider-migrate.service OnCalendar=*-*-* 05:00:00 -RandomizedDelaySec=2h [Install] WantedBy=timers.target From 6f56b9fb38fbc5216054067a1658e8530a98f4fc Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 18:17:08 +0300 Subject: [PATCH 028/118] Upgrade Dendrite (0.10.2 -> 0.10.3) --- roles/matrix-dendrite/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-dendrite/defaults/main.yml b/roles/matrix-dendrite/defaults/main.yml index 92a9ff9f..2adf13da 100644 --- a/roles/matrix-dendrite/defaults/main.yml +++ b/roles/matrix-dendrite/defaults/main.yml @@ -6,7 +6,7 @@ matrix_dendrite_enabled: true matrix_dendrite_docker_image: "{{ matrix_dendrite_docker_image_name_prefix }}matrixdotorg/dendrite-monolith:{{ matrix_dendrite_docker_image_tag }}" matrix_dendrite_docker_image_name_prefix: "docker.io/" -matrix_dendrite_docker_image_tag: "v0.10.2" +matrix_dendrite_docker_image_tag: "v0.10.3" matrix_dendrite_docker_image_force_pull: "{{ matrix_dendrite_docker_image.endswith(':latest') }}" matrix_dendrite_base_path: "{{ matrix_base_data_path }}/dendrite" From 30745db6d5f098fa68ac228b900f6efa82fd5ed1 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 18:29:30 +0300 Subject: [PATCH 029/118] Add storage-class support to the s3_media_upload command and mention matrix-synapse-s3-storage-provider-migrate script --- ...configuring-playbook-synapse-s3-storage-provider.md | 10 ++++++++-- .../templates/synapse/ext/s3-storage-provider/env.j2 | 1 + .../matrix-synapse-s3-storage-provider-migrate.j2 | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/configuring-playbook-synapse-s3-storage-provider.md b/docs/configuring-playbook-synapse-s3-storage-provider.md index 6ab2820a..fce5b4ed 100644 --- a/docs/configuring-playbook-synapse-s3-storage-provider.md +++ b/docs/configuring-playbook-synapse-s3-storage-provider.md @@ -70,10 +70,16 @@ Then use the following commands (`$` values come from environment variables - th - `$UPDATE_DB_DURATION` is influenced by the `matrix_synapse_ext_synapse_s3_storage_provider_update_db_day_count` variable (defaults to `0`) - `$UPDATE_DB_DURATION` defaults to `0d` (0 days), which means **include files which haven't been accessed for more than 0 days** (that is, **all files will be included**). - `s3_media_upload check-deleted $MEDIA_PATH` - check whether files in the local cache still exist in the local media repository directory -- `s3_media_upload upload $MEDIA_PATH $BUCKET --delete --endpoint-url $ENDPOINT` - uploads locally-stored files to S3 and deletes them from the local media repository directory +- `s3_media_upload upload $MEDIA_PATH $BUCKET --delete --storage-class $STORAGE_CLASS --endpoint-url $ENDPOINT` - uploads locally-stored files to S3 and deletes them from the local media repository directory -The `upload` command may take a lot of time to complete. +The `s3_media_upload upload` command may take a lot of time to complete. +Instead of running the above commands manually in the shell, you can also run the `/usr/local/bin/matrix-synapse-s3-storage-provider-migrate` script which will run the same commands automatically. We demonstrate how to do it manually, because: + +- it's what the upstream project demonstrates and it teaches you how to use the `s3_media_upload` tool +- allows you to check and verify the output of each command, to catch mistakes +- includes progress bars and detailed output for each command +- allows you to easily interrupt slow-running commands, etc. (the `/usr/local/bin/matrix-synapse-s3-storage-provider-migrate` starts a container without interactive TTY support, so `Ctrl+C` may not work and you and require killing via `docker kill ..`) ### Using another tool in combination with `s3_media_upload` diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 index 3788499b..6dfcbe41 100644 --- a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 @@ -4,6 +4,7 @@ AWS_DEFAULT_REGION={{ matrix_synapse_ext_synapse_s3_storage_provider_config_regi ENDPOINT={{ matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url }} BUCKET={{ matrix_synapse_ext_synapse_s3_storage_provider_config_bucket }} +STORAGE_CLASS={{ matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class }} MEDIA_PATH=/matrix-media-store-parent/{{ matrix_synapse_media_store_directory_name }} diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 index 0893f5d6..e6684e69 100644 --- a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 @@ -10,4 +10,4 @@ --network={{ matrix_docker_network }} \ --entrypoint=/bin/bash \ {{ matrix_synapse_docker_image_final }} \ - -c 's3_media_upload update-db $UPDATE_DB_DURATION && s3_media_upload --no-progress check-deleted $MEDIA_PATH && s3_media_upload --no-progress upload $MEDIA_PATH $BUCKET --delete --endpoint-url $ENDPOINT' + -c 's3_media_upload update-db $UPDATE_DB_DURATION && s3_media_upload --no-progress check-deleted $MEDIA_PATH && s3_media_upload --no-progress upload $MEDIA_PATH $BUCKET --delete --storage-class $STORAGE_CLASS --endpoint-url $ENDPOINT' From 5a7b80d9e48326f8d6bedf1e31a234fbe735c8d6 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 18:31:17 +0300 Subject: [PATCH 030/118] Update configuring-playbook-synapse-s3-storage-provider.md --- docs/configuring-playbook-synapse-s3-storage-provider.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuring-playbook-synapse-s3-storage-provider.md b/docs/configuring-playbook-synapse-s3-storage-provider.md index fce5b4ed..a71e5a0b 100644 --- a/docs/configuring-playbook-synapse-s3-storage-provider.md +++ b/docs/configuring-playbook-synapse-s3-storage-provider.md @@ -48,6 +48,8 @@ If you have existing files in Synapse's media repository (`/matrix/synapse/media - the existing files will remain on the local filesystem only until [migrating them to the S3 store](#migrating-your-existing-media-files-to-the-s3-store) - at some point (and periodically in the future), you can delete local files which have been uploaded to the S3 store already +Regardless of whether you need to [Migrate your existing files to the S3 store](#migrating-your-existing-media-files-to-the-s3-store) or not, make sure you've familiarized yourself with [How it works?](#how-it-works) above and [Periodically cleaning up the local filesystem](#periodically-cleaning-up-the-local-filesystem) below. + ## Migrating your existing media files to the S3 store From 1e0d10586d4bd04dde30adb1e860ada432084865 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 18:49:27 +0300 Subject: [PATCH 031/118] Update README to mention synapse-s3-storage-provider --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b93fdd5d..569dbb3f 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Using this playbook, you can get the following services configured on your serve - (optional) a [Dendrite](https://github.com/matrix-org/dendrite) homeserver - storing your data and managing your presence in the [Matrix](http://matrix.org/) network. Dendrite is a second-generation Matrix homeserver written in Go, an alternative to Synapse. -- (optional) [Amazon S3](https://aws.amazon.com/s3/) storage for Synapse's content repository (`media_store`) files using [Goofys](https://github.com/kahing/goofys) +- (optional) [Amazon S3](https://aws.amazon.com/s3/) (or other S3-compatible object store) storage for Synapse's content repository (`media_store`) files using [Goofys](https://github.com/kahing/goofys) or [`synapse-s3-storage-provider`](https://github.com/matrix-org/synapse-s3-storage-provider) - (optional, default) [PostgreSQL](https://www.postgresql.org/) database for Synapse. [Using an external PostgreSQL server](docs/configuring-playbook-external-postgres.md) is also possible. From 8025bd19b0b93ac52230f756e86155b6b7bf17a7 Mon Sep 17 00:00:00 2001 From: smargold476 <105579587+smargold476@users.noreply.github.com> Date: Sat, 15 Oct 2022 14:33:28 +0200 Subject: [PATCH 032/118] update Docs regarding Filter-Mode-Flag --- docs/configuring-playbook-bridge-mautrix-telegram.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/configuring-playbook-bridge-mautrix-telegram.md b/docs/configuring-playbook-bridge-mautrix-telegram.md index 08ee83cc..91596afc 100644 --- a/docs/configuring-playbook-bridge-mautrix-telegram.md +++ b/docs/configuring-playbook-bridge-mautrix-telegram.md @@ -59,3 +59,8 @@ matrix_mautrix_telegram_configuration_extension_yaml: | More details about permissions in this example: https://github.com/mautrix/telegram/blob/master/mautrix_telegram/example-config.yaml#L410 + +If you like to exclude all groups from syncing and use the Telgeram-Bridge only for direct chats, you can add the following additional playbook configuration: +```yaml +matrix_mautrix_telegram_filter_mode: whitelist +``` From 01078fea8f0c32e09e20a81070ce0cc5f6712435 Mon Sep 17 00:00:00 2001 From: smargold476 <105579587+smargold476@users.noreply.github.com> Date: Sat, 15 Oct 2022 14:37:09 +0200 Subject: [PATCH 033/118] add default for TG filter_mode --- roles/matrix-bridge-mautrix-telegram/defaults/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml index 9fdfd430..0f52cc22 100644 --- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml @@ -38,6 +38,9 @@ matrix_mautrix_telegram_api_id: '' matrix_mautrix_telegram_api_hash: '' matrix_mautrix_telegram_bot_token: disabled +# Define the filter-mode +matrix_mautrix_telegram_filter_mode: "blacklist" + # Whether or not the public-facing endpoints should be enabled (web-based login) matrix_mautrix_telegram_appservice_public_enabled: true From a6e167cf91932eac58fdcb117062f2a31d62ed64 Mon Sep 17 00:00:00 2001 From: smargold476 <105579587+smargold476@users.noreply.github.com> Date: Sat, 15 Oct 2022 14:38:51 +0200 Subject: [PATCH 034/118] add option to change filter_mode --- roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 index d50be47d..2e6aae51 100644 --- a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 @@ -273,7 +273,7 @@ bridge: # Filter mode to use. Either "blacklist" or "whitelist". # If the mode is "blacklist", the listed chats will never be bridged. # If the mode is "whitelist", only the listed chats can be bridged. - mode: blacklist + mode: "{{ matrix_mautrix_telegram_filter_mode }}" # The list of group/channel IDs to filter. list: [] From 7b3b22ce072ce02d7ebb8665f899147422a84841 Mon Sep 17 00:00:00 2001 From: smargold476 <105579587+smargold476@users.noreply.github.com> Date: Sat, 15 Oct 2022 14:43:25 +0200 Subject: [PATCH 035/118] update docs to exclude grou-sync --- docs/configuring-playbook-bridge-mautrix-telegram.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/configuring-playbook-bridge-mautrix-telegram.md b/docs/configuring-playbook-bridge-mautrix-telegram.md index 08ee83cc..91596afc 100644 --- a/docs/configuring-playbook-bridge-mautrix-telegram.md +++ b/docs/configuring-playbook-bridge-mautrix-telegram.md @@ -59,3 +59,8 @@ matrix_mautrix_telegram_configuration_extension_yaml: | More details about permissions in this example: https://github.com/mautrix/telegram/blob/master/mautrix_telegram/example-config.yaml#L410 + +If you like to exclude all groups from syncing and use the Telgeram-Bridge only for direct chats, you can add the following additional playbook configuration: +```yaml +matrix_mautrix_telegram_filter_mode: whitelist +``` From a7dc9a406de907ab21804613a13c335f335c4352 Mon Sep 17 00:00:00 2001 From: Felix Stupp Date: Sat, 15 Oct 2022 20:17:58 +0000 Subject: [PATCH 036/118] docs/configuring-playbook: Add link to etherpad configuration --- docs/configuring-playbook.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuring-playbook.md b/docs/configuring-playbook.md index 43b68ee8..efb0f7e3 100644 --- a/docs/configuring-playbook.md +++ b/docs/configuring-playbook.md @@ -34,6 +34,8 @@ When you're done with all the configuration you'd like to do, continue with [Ins - [Setting up the Jitsi video-conferencing platform](configuring-playbook-jitsi.md) (optional) +- [Setting up Etherpad](configuring-playbook-etherpad.md) (optional) + - [Setting up Dynamic DNS](configuring-playbook-dynamic-dns.md) (optional) - [Enabling metrics and graphs (Prometheus, Grafana) for your Matrix server](configuring-playbook-prometheus-grafana.md) (optional) From 4d267dad4084859ca9c69b23e241c4ae2c1d2817 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 16 Oct 2022 18:24:55 +0300 Subject: [PATCH 037/118] Upgrade mautrix-whatsapp (0.7.0 -> 0.7.1) and sync bridge config with upstream --- .../matrix-bridge-mautrix-whatsapp/defaults/main.yml | 2 +- .../templates/config.yaml.j2 | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml index cb4900ee..55b7387f 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml @@ -8,7 +8,7 @@ matrix_mautrix_whatsapp_container_image_self_build: false matrix_mautrix_whatsapp_container_image_self_build_repo: "https://mau.dev/mautrix/whatsapp.git" matrix_mautrix_whatsapp_container_image_self_build_branch: "{{ 'master' if matrix_mautrix_whatsapp_version == 'latest' else matrix_mautrix_whatsapp_version }}" -matrix_mautrix_whatsapp_version: v0.7.0 +matrix_mautrix_whatsapp_version: v0.7.1 # See: https://mau.dev/mautrix/whatsapp/container_registry matrix_mautrix_whatsapp_docker_image: "{{ matrix_mautrix_whatsapp_docker_image_name_prefix }}mautrix/whatsapp:{{ matrix_mautrix_whatsapp_version }}" matrix_mautrix_whatsapp_docker_image_name_prefix: "{{ 'localhost/' if matrix_mautrix_whatsapp_container_image_self_build else 'dock.mau.dev/' }}" diff --git a/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 index 28971447..9c0b95e8 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 @@ -147,6 +147,12 @@ bridge: # provisioning endpoint is used or when a message comes in from that # chat. max_initial_conversations: -1 + # If this value is greater than 0, then if the conversation's last + # message was more than this number of hours ago, then the conversation + # will automatically be marked it as read. + # Conversations that have a last message that is less than this number + # of hours ago will have their unread status synced from WhatsApp. + unread_hours_threshold: 0 # Settings for immediate backfills. These backfills should generally be # small and their main purpose is to populate each of the initial chats # (as configured by max_initial_conversations) with a few messages so @@ -228,7 +234,10 @@ bridge: # manually. login_shared_secret_map: {{ matrix_mautrix_whatsapp_bridge_login_shared_secret_map|to_json }} # Should the bridge explicitly set the avatar and room name for private chat portal rooms? + # This is implicitly enabled in encrypted rooms. private_chat_portal_meta: false + # Should group members be synced in parallel? This makes member sync faster + parallel_member_sync: false # Should Matrix m.notice-type messages be bridged? bridge_notices: true # Set this to true to tell the bridge to re-send m.bridge events to all rooms on the next run. @@ -281,6 +290,9 @@ bridge: # Send captions in the same message as images. This will send data compatible with both MSC2530 and MSC3552. # This is currently not supported in most clients. caption_in_message: false + # Should Matrix edits be bridged to WhatsApp edits? + # Official WhatsApp clients don't render edits yet, but once they do, the bridge should work with them right away. + send_whatsapp_edits: false # Maximum time for handling Matrix events. Duration strings formatted for https://pkg.go.dev/time#ParseDuration # Null means there's no enforced timeout. message_handling_timeout: From 07a7234ba0c642f910e3381776edf5b40735c9de Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 16 Oct 2022 18:27:35 +0300 Subject: [PATCH 038/118] Use | to_json --- .../matrix-bridge-mautrix-telegram/templates/config.yaml.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 index 2e6aae51..b7af83f5 100644 --- a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 @@ -273,12 +273,12 @@ bridge: # Filter mode to use. Either "blacklist" or "whitelist". # If the mode is "blacklist", the listed chats will never be bridged. # If the mode is "whitelist", only the listed chats can be bridged. - mode: "{{ matrix_mautrix_telegram_filter_mode }}" + mode: {{ matrix_mautrix_telegram_filter_mode | to_json }} # The list of group/channel IDs to filter. list: [] # The prefix for commands. Only required in non-management rooms. - command_prefix: "{{ matrix_mautrix_telegram_command_prefix }}" + command_prefix: {{ matrix_mautrix_telegram_command_prefix | to_json }} # Permissions for using the bridge. # Permitted values: @@ -291,7 +291,7 @@ bridge: # * - All Matrix users # domain - All users on that homeserver # mxid - Specific user - permissions: {{ matrix_mautrix_telegram_bridge_permissions|to_json }} + permissions: {{ matrix_mautrix_telegram_bridge_permissions | to_json }} # Options related to the message relay Telegram bot. relaybot: From 6480cc36293da31fa58b3aa1f7fe90fffbae87bf Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 17 Oct 2022 08:03:10 +0300 Subject: [PATCH 039/118] Fix `'something' is undefined` error in matrix-bridge-appservice-slack .. when `matrix_nginx_proxy_enabled: false` Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/2177 --- roles/matrix-bridge-appservice-slack/tasks/init.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-appservice-slack/tasks/init.yml b/roles/matrix-bridge-appservice-slack/tasks/init.yml index d06e5aaf..5d03b24b 100644 --- a/roles/matrix-bridge-appservice-slack/tasks/init.yml +++ b/roles/matrix-bridge-appservice-slack/tasks/init.yml @@ -85,7 +85,7 @@ msg: >- NOTE: You've enabled the Matrix Slack bridge but are not using the matrix-nginx-proxy reverse proxy. - Please make sure that you're proxying the `{{ something }}` + Please make sure that you're proxying the `{{ matrix_appservice_slack_public_endpoint }}` URL endpoint to the matrix-appservice-slack container. You can expose the container's port using the `matrix_appservice_slack_container_http_host_bind_port` variable. when: "matrix_appservice_slack_enabled | bool and not matrix_nginx_proxy_enabled | default(False) | bool" From c781bdea992435fd9768d5a7458058528b712300 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Mon, 17 Oct 2022 14:24:57 +0000 Subject: [PATCH 040/118] Update Synapse 1.68.0 -> 1.69.0 --- roles/matrix-synapse/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 4b979159..5e5867eb 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -36,7 +36,7 @@ matrix_synapse_container_image_customizations_dockerfile_body_custom: '' matrix_synapse_docker_image: "{{ matrix_synapse_docker_image_name_prefix }}matrixdotorg/synapse:{{ matrix_synapse_docker_image_tag }}" matrix_synapse_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_container_image_self_build else matrix_container_global_registry_prefix }}" -matrix_synapse_version: v1.68.0 +matrix_synapse_version: v1.69.0 matrix_synapse_docker_image_tag: "{{ matrix_synapse_version }}" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" From bcba84389d683f9143dbb7c2ccb516e4b68773d6 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 18 Oct 2022 12:43:13 +0000 Subject: [PATCH 041/118] Update grafana 9.2.0 -> 9.2.1 --- roles/matrix-grafana/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-grafana/defaults/main.yml b/roles/matrix-grafana/defaults/main.yml index 677435e8..767a749b 100644 --- a/roles/matrix-grafana/defaults/main.yml +++ b/roles/matrix-grafana/defaults/main.yml @@ -5,7 +5,7 @@ matrix_grafana_enabled: true -matrix_grafana_version: 9.2.0 +matrix_grafana_version: 9.2.1 matrix_grafana_docker_image: "{{ matrix_container_global_registry_prefix }}grafana/grafana:{{ matrix_grafana_version }}" matrix_grafana_docker_image_force_pull: "{{ matrix_grafana_docker_image.endswith(':latest') }}" From 2eef6af23e6a8c4e5a46c9ccf49a12c0e4f24c91 Mon Sep 17 00:00:00 2001 From: David Napier Date: Wed, 19 Oct 2022 10:45:11 -0400 Subject: [PATCH 042/118] Bump postgres version to newly released 15 --- roles/matrix-postgres/defaults/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index de28d7ad..93de2612 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -29,7 +29,8 @@ matrix_postgres_docker_image_v11: "{{ matrix_container_global_registry_prefix }} matrix_postgres_docker_image_v12: "{{ matrix_container_global_registry_prefix }}postgres:12.12{{ matrix_postgres_docker_image_suffix }}" matrix_postgres_docker_image_v13: "{{ matrix_container_global_registry_prefix }}postgres:13.8{{ matrix_postgres_docker_image_suffix }}" matrix_postgres_docker_image_v14: "{{ matrix_container_global_registry_prefix }}postgres:14.5{{ matrix_postgres_docker_image_suffix }}" -matrix_postgres_docker_image_latest: "{{ matrix_postgres_docker_image_v14 }}" +matrix_postgres_docker_image_v15: "{{ matrix_container_global_registry_prefix }}postgres:15{{ matrix_postgres_docker_image_suffix }}" +matrix_postgres_docker_image_latest: "{{ matrix_postgres_docker_image_v15 }}" # This variable is assigned at runtime. Overriding its value has no effect. matrix_postgres_docker_image_to_use: '{{ matrix_postgres_docker_image_latest }}' From c4a57428a6bedac73f1f8253ab0c5e4010aa00dc Mon Sep 17 00:00:00 2001 From: array-in-a-matrix Date: Thu, 20 Oct 2022 00:34:30 -0400 Subject: [PATCH 043/118] add if hostname is not availble for SRV record --- docs/configuring-dns.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuring-dns.md b/docs/configuring-dns.md index 8d31ab3f..ea5fea88 100644 --- a/docs/configuring-dns.md +++ b/docs/configuring-dns.md @@ -45,6 +45,8 @@ If you are using Cloudflare DNS, make sure to disable the proxy and set all reco | [Postmoogle](configuring-playbook-bot-postmoogle.md) email bridge | TXT | `_dmarc.matrix` | - | - | - | `v=DMARC1; p=quarantine;` | | [Postmoogle](configuring-playbook-bot-postmoogle.md) email bridge | TXT | `postmoogle._domainkey.matrix` | - | - | - | get it from `!pm dkim` | +When setting up a SRV record, if you are asked for a service and protocol instead of a hostname split the host value from the table where the period is. For example use service as `_matrix-identity` and protocol as `_tcp`. + ## Subdomains setup As the table above illustrates, you need to create 2 subdomains (`matrix.` and `element.`) and point both of them to your new server's IP address (DNS `A` record or `CNAME` record is fine). From 20db57d288e037bd651d718e945fff96faa33d77 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 17 Oct 2022 17:35:52 +0300 Subject: [PATCH 044/118] Upgrade certbot (v1.30.0 -> v1.31.0) --- roles/matrix-nginx-proxy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 7cdc0c92..84ec4f7e 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -547,7 +547,7 @@ matrix_ssl_lets_encrypt_staging: false # Learn more here: https://eff-certbot.readthedocs.io/en/stable/using.html#changing-the-acme-server matrix_ssl_lets_encrypt_server: '' -matrix_ssl_lets_encrypt_certbot_docker_image: "{{ matrix_container_global_registry_prefix }}certbot/certbot:{{ matrix_ssl_architecture }}-v1.30.0" +matrix_ssl_lets_encrypt_certbot_docker_image: "{{ matrix_container_global_registry_prefix }}certbot/certbot:{{ matrix_ssl_architecture }}-v1.31.0" matrix_ssl_lets_encrypt_certbot_docker_image_force_pull: "{{ matrix_ssl_lets_encrypt_certbot_docker_image.endswith(':latest') }}" matrix_ssl_lets_encrypt_certbot_standalone_http_port: 2402 matrix_ssl_lets_encrypt_support_email: ~ From a32eea41fe56bdee721f8062b1199217704b0daa Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 20 Oct 2022 07:43:01 +0300 Subject: [PATCH 045/118] Make roles/matrix-postgres/tasks/detect_existing_postgres_version.yml detect Postgres v14 --- .../tasks/detect_existing_postgres_version.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/matrix-postgres/tasks/detect_existing_postgres_version.yml b/roles/matrix-postgres/tasks/detect_existing_postgres_version.yml index 687d5e3a..1be8291b 100644 --- a/roles/matrix-postgres/tasks/detect_existing_postgres_version.yml +++ b/roles/matrix-postgres/tasks/detect_existing_postgres_version.yml @@ -68,3 +68,8 @@ ansible.builtin.set_fact: matrix_postgres_detected_version_corresponding_docker_image: "{{ matrix_postgres_docker_image_v13 }}" when: "matrix_postgres_detected_version == '13' or matrix_postgres_detected_version.startswith('13.')" + +- name: Determine corresponding Docker image to detected version (use 14.x, if detected) + ansible.builtin.set_fact: + matrix_postgres_detected_version_corresponding_docker_image: "{{ matrix_postgres_docker_image_v14 }}" + when: "matrix_postgres_detected_version == '14' or matrix_postgres_detected_version.startswith('14.')" From 80da7dfb0bc4dbff1b6982d213ca5cbb98ff7f94 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 20 Oct 2022 11:48:56 +0300 Subject: [PATCH 046/118] Update docs/prerequisites.md --- docs/prerequisites.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/prerequisites.md b/docs/prerequisites.md index 1ed4befe..74954648 100644 --- a/docs/prerequisites.md +++ b/docs/prerequisites.md @@ -26,7 +26,7 @@ If your distro runs within an [LXC container](https://linuxcontainers.org/), you - Properly configured DNS records for `` (details in [Configuring DNS](configuring-dns.md)). -- Some TCP/UDP ports open. This playbook configures the server's internal firewall for you. In most cases, you don't need to do anything special. But **if your server is running behind another firewall**, you'd need to open these ports: +- Some TCP/UDP ports open. This playbook (actually [Docker itself](https://docs.docker.com/network/iptables/)) configures the server's internal firewall for you. In most cases, you don't need to do anything special. But **if your server is running behind another firewall**, you'd need to open these ports: - `80/tcp`: HTTP webserver - `443/tcp`: HTTPS webserver From af3a32cf6a7f7c6f73ee6bd85b3fede721b5e70c Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 20 Oct 2022 15:46:02 +0300 Subject: [PATCH 047/118] Fix Jinja2 interpolation for some default variables We're overriding these in the correct way in `group_vars/matrix_servers` so this wasn't causing any problem in practice. --- roles/matrix-postgres/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index 93de2612..1b11bad8 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -68,7 +68,7 @@ matrix_postgres_additional_databases: [] # If a dump file contains the roles and they've also been created beforehand (see `matrix_postgres_additional_databases`), # importing would fail. # We either need to not create them or to ignore the `CREATE ROLE` statements in the dump. -matrix_postgres_import_roles_to_ignore: [matrix_postgres_connection_username] +matrix_postgres_import_roles_to_ignore: "{{ [matrix_postgres_connection_username] }}" # When importing an existing Postgres database (when restoring a backup) or when doing a Postgres upgrade (which dumps & restores), we'd like to avoid: # - creating users (`CREATE ROLE ..`) @@ -85,7 +85,7 @@ matrix_postgres_import_roles_ignore_regex: "^(CREATE|ALTER) ROLE ({{ matrix_post # If a dump file contains the databases and they've also been created beforehand (see `matrix_postgres_additional_databases`), # importing would fail. # We either need to not create them or to ignore the `CREATE DATABASE` statements in the dump. -matrix_postgres_import_databases_to_ignore: [matrix_postgres_db_name] +matrix_postgres_import_databases_to_ignore: "{{ [matrix_postgres_db_name] }}" matrix_postgres_import_databases_ignore_regex: "^CREATE DATABASE ({{ matrix_postgres_import_databases_to_ignore | join('|') }})\\s" # noqa jinja[spacing] From b8097b0bd6f79f9efb66c4df7676129f86e07bd2 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 20 Oct 2022 16:05:55 +0300 Subject: [PATCH 048/118] Add support for binary content to matrix-aux --- roles/matrix-aux/defaults/main.yml | 9 +++++++++ roles/matrix-aux/tasks/setup.yml | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/roles/matrix-aux/defaults/main.yml b/roles/matrix-aux/defaults/main.yml index e4a4e827..4c1f8879 100644 --- a/roles/matrix-aux/defaults/main.yml +++ b/roles/matrix-aux/defaults/main.yml @@ -50,6 +50,9 @@ matrix_aux_file_default_mode: '0640' # then you likely need to add `/matrix/some/path` to `matrix_aux_directory_definitions` as well. # You don't need to do this for directories that the playbook already creates for you. # +# Use a `content` key for text content and `src` with a location to a file for binary content. +# The `content` key does not support binary content (see https://github.com/ansible/ansible/issues/11594). +# # Example: # # matrix_aux_file_definitions: @@ -69,4 +72,10 @@ matrix_aux_file_default_mode: '0640' # mode: '0600' # owner: 'some-user' # group: 'some-group' +# +# - dest: /matrix/aux/binary-file.dat +# src: "/path/to/binary.dat" +# mode: '0600' +# owner: 'some-user' +# group: 'some-group' matrix_aux_file_definitions: [] diff --git a/roles/matrix-aux/tasks/setup.yml b/roles/matrix-aux/tasks/setup.yml index ccb0bdcb..eb0adad4 100644 --- a/roles/matrix-aux/tasks/setup.yml +++ b/roles/matrix-aux/tasks/setup.yml @@ -11,8 +11,9 @@ - name: Ensure AUX files are created ansible.builtin.copy: + src: "{{ item.src if 'src' in item else omit }}" + content: "{{ item.content if 'content' in item else omit }}" dest: "{{ item.dest }}" - content: "{{ item.content }}" owner: "{{ item.owner | default(matrix_user_username) }}" group: "{{ item.group | default(matrix_user_groupname) }}" mode: "{{ item.mode | default(matrix_aux_file_default_mode) }}" From bed9c18ab04156e6e96521a380acc3be0c36e047 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 21 Oct 2022 05:31:52 +0300 Subject: [PATCH 049/118] Pin Postgres version to 15.0 Related to https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/2180 Just specifying `15` means we won't automatically re-pull `15.1` when it comes out. --- roles/matrix-postgres/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index 1b11bad8..6ddbdf9f 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -29,7 +29,7 @@ matrix_postgres_docker_image_v11: "{{ matrix_container_global_registry_prefix }} matrix_postgres_docker_image_v12: "{{ matrix_container_global_registry_prefix }}postgres:12.12{{ matrix_postgres_docker_image_suffix }}" matrix_postgres_docker_image_v13: "{{ matrix_container_global_registry_prefix }}postgres:13.8{{ matrix_postgres_docker_image_suffix }}" matrix_postgres_docker_image_v14: "{{ matrix_container_global_registry_prefix }}postgres:14.5{{ matrix_postgres_docker_image_suffix }}" -matrix_postgres_docker_image_v15: "{{ matrix_container_global_registry_prefix }}postgres:15{{ matrix_postgres_docker_image_suffix }}" +matrix_postgres_docker_image_v15: "{{ matrix_container_global_registry_prefix }}postgres:15.0{{ matrix_postgres_docker_image_suffix }}" matrix_postgres_docker_image_latest: "{{ matrix_postgres_docker_image_v15 }}" # This variable is assigned at runtime. Overriding its value has no effect. From e37e86eb1cfdabf0bbe7464037060cbb0cacf2dc Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 21 Oct 2022 07:33:10 +0300 Subject: [PATCH 050/118] Fix 'could not save history to file "//.psql_history"' errors --- roles/matrix-postgres/templates/env-postgres-psql.j2 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/roles/matrix-postgres/templates/env-postgres-psql.j2 b/roles/matrix-postgres/templates/env-postgres-psql.j2 index c61927a3..22058987 100644 --- a/roles/matrix-postgres/templates/env-postgres-psql.j2 +++ b/roles/matrix-postgres/templates/env-postgres-psql.j2 @@ -1,4 +1,8 @@ #jinja2: lstrip_blocks: "True" PGUSER={{ matrix_postgres_connection_username }} PGPASSWORD={{ matrix_postgres_connection_password }} -PGDATABASE={{ matrix_postgres_db_name }} \ No newline at end of file +PGDATABASE={{ matrix_postgres_db_name }} +# Prevent errors like this: +# > could not save history to file "//.psql_history": Permission denied +# .. due to Postgres not being able to write to the filesystem. +PSQL_HISTORY=/dev/null From 0a022db256688a9924d199828f4f555bbb94d168 Mon Sep 17 00:00:00 2001 From: Array in a Matrix Date: Fri, 21 Oct 2022 00:52:48 -0400 Subject: [PATCH 051/118] grammar correction the word "both" would refer to only 2 urls in this context --- docs/configuring-well-known.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuring-well-known.md b/docs/configuring-well-known.md index 81caf04c..fd548aa6 100644 --- a/docs/configuring-well-known.md +++ b/docs/configuring-well-known.md @@ -192,7 +192,7 @@ Make sure to: ## Confirming it works -No matter which method you've used to set up the well-known files, if you've done it correctly you should be able to see a JSON file at both of these URLs: +No matter which method you've used to set up the well-known files, if you've done it correctly you should be able to see a JSON file at these URLs: - `https:///.well-known/matrix/server` - `https:///.well-known/matrix/client` From 0b44ec19b4e1d39420710f229808f0780c5820d2 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 21 Oct 2022 10:00:49 +0300 Subject: [PATCH 052/118] Do not override matrix_postgres_import_roles_to_ignore/matrix_postgres_import_databases_to_ignore in group_vars These values that we were setting also make sense in the context of the `matrix-postgres` role even when not used within the playbook. --- group_vars/matrix_servers | 14 -------------- roles/matrix-postgres/defaults/main.yml | 14 ++++++++++++-- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index b055245b..6728eaaa 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -2112,20 +2112,6 @@ matrix_postgres_additional_databases: | }} -matrix_postgres_import_roles_to_ignore: | - {{ - [matrix_postgres_connection_username] - + - matrix_postgres_additional_databases|map(attribute='username') | list - }} - -matrix_postgres_import_databases_to_ignore: | - {{ - [matrix_postgres_db_name] - + - matrix_postgres_additional_databases|map(attribute='name') | list - }} - ###################################################################### # # /matrix-postgres diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index 6ddbdf9f..e34b3b60 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -68,7 +68,12 @@ matrix_postgres_additional_databases: [] # If a dump file contains the roles and they've also been created beforehand (see `matrix_postgres_additional_databases`), # importing would fail. # We either need to not create them or to ignore the `CREATE ROLE` statements in the dump. -matrix_postgres_import_roles_to_ignore: "{{ [matrix_postgres_connection_username] }}" +matrix_postgres_import_roles_to_ignore: | + {{ + [matrix_postgres_connection_username] + + + matrix_postgres_additional_databases|map(attribute='username') | list + }} # When importing an existing Postgres database (when restoring a backup) or when doing a Postgres upgrade (which dumps & restores), we'd like to avoid: # - creating users (`CREATE ROLE ..`) @@ -85,7 +90,12 @@ matrix_postgres_import_roles_ignore_regex: "^(CREATE|ALTER) ROLE ({{ matrix_post # If a dump file contains the databases and they've also been created beforehand (see `matrix_postgres_additional_databases`), # importing would fail. # We either need to not create them or to ignore the `CREATE DATABASE` statements in the dump. -matrix_postgres_import_databases_to_ignore: "{{ [matrix_postgres_db_name] }}" +matrix_postgres_import_databases_to_ignore: | + {{ + [matrix_postgres_db_name] + + + matrix_postgres_additional_databases|map(attribute='name') | list + }} matrix_postgres_import_databases_ignore_regex: "^CREATE DATABASE ({{ matrix_postgres_import_databases_to_ignore | join('|') }})\\s" # noqa jinja[spacing] From ff0fc88faeac6444dcee93241dca23a24356cde1 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 21 Oct 2022 11:25:30 +0300 Subject: [PATCH 053/118] Make Postgres import not break for databases with special names We haven't encountered such a problem yet, but it doesn't hurt to make things more robust. --- roles/matrix-postgres/defaults/main.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index e34b3b60..7b0660bb 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -84,7 +84,9 @@ matrix_postgres_import_roles_to_ignore: | # which is unsupported by default by newer Postgres versions (v14+). # When users are created and passwords are set by the playbook, they end up hashed as `scram-sha-256` on Postgres v14+. # If an md5-hashed password is restored on top, Postgres v14+ will refuse to authenticate users with it by default. -matrix_postgres_import_roles_ignore_regex: "^(CREATE|ALTER) ROLE ({{ matrix_postgres_import_roles_to_ignore | join('|') }})(;| WITH)" # noqa jinja[spacing] +# +# We also allow for the role name to be quoted, which is rare, but might happen for role names which are special keywords (e.g. `default`). +matrix_postgres_import_roles_ignore_regex: "^(CREATE|ALTER) ROLE \\\"?({{ matrix_postgres_import_roles_to_ignore | join('|') }})\\\"?(;| WITH)" # noqa jinja[spacing] # A list of databases to avoid creating when importing (or upgrading) the database. # If a dump file contains the databases and they've also been created beforehand (see `matrix_postgres_additional_databases`), @@ -97,7 +99,8 @@ matrix_postgres_import_databases_to_ignore: | matrix_postgres_additional_databases|map(attribute='name') | list }} -matrix_postgres_import_databases_ignore_regex: "^CREATE DATABASE ({{ matrix_postgres_import_databases_to_ignore | join('|') }})\\s" # noqa jinja[spacing] +# We also allow for the database name to be quoted, which is rare, but might happen for database names which are special keywords (e.g. `default`). +matrix_postgres_import_databases_ignore_regex: "^CREATE DATABASE \\\"?({{ matrix_postgres_import_databases_to_ignore | join('|') }})\\\"?\\s" # noqa jinja[spacing] # The number of seconds to wait after starting `matrix-postgres.service` # and before trying to run queries for creating additional databases/users against it. From 7e8184e210798da376b2fd16baa7c730811dcbb4 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 21 Oct 2022 14:58:21 +0300 Subject: [PATCH 054/118] Do not break the guided installation flow --- docs/configuring-dns.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuring-dns.md b/docs/configuring-dns.md index ea5fea88..3803ba8f 100644 --- a/docs/configuring-dns.md +++ b/docs/configuring-dns.md @@ -26,6 +26,8 @@ Be mindful as to how long it will take for the DNS records to propagate. If you are using Cloudflare DNS, make sure to disable the proxy and set all records to `DNS only`. Otherwise, fetching certificates will fail. +When you're done configuring DNS, proceed to [Configuring the playbook](configuring-playbook.md). + ## DNS settings for optional services/features | Used by component | Type | Host | Priority | Weight | Port | Target | From 4cf85605a911e998409d9e2a1cb7a94d83854491 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 21 Oct 2022 17:36:24 +0300 Subject: [PATCH 055/118] Upgrade Dendrite (0.10.3 -> 0.10.4) --- roles/matrix-dendrite/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-dendrite/defaults/main.yml b/roles/matrix-dendrite/defaults/main.yml index 2adf13da..dc0af2dc 100644 --- a/roles/matrix-dendrite/defaults/main.yml +++ b/roles/matrix-dendrite/defaults/main.yml @@ -6,7 +6,7 @@ matrix_dendrite_enabled: true matrix_dendrite_docker_image: "{{ matrix_dendrite_docker_image_name_prefix }}matrixdotorg/dendrite-monolith:{{ matrix_dendrite_docker_image_tag }}" matrix_dendrite_docker_image_name_prefix: "docker.io/" -matrix_dendrite_docker_image_tag: "v0.10.3" +matrix_dendrite_docker_image_tag: "v0.10.4" matrix_dendrite_docker_image_force_pull: "{{ matrix_dendrite_docker_image.endswith(':latest') }}" matrix_dendrite_base_path: "{{ matrix_base_data_path }}/dendrite" From 463596884028dc38099d40155cc4117976bca8d6 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 21 Oct 2022 20:07:29 +0300 Subject: [PATCH 056/118] Upgrade Hookshot (2.3.0 -> 2.4.0) --- roles/matrix-bridge-hookshot/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-hookshot/defaults/main.yml b/roles/matrix-bridge-hookshot/defaults/main.yml index 0854edbf..1c390f34 100644 --- a/roles/matrix-bridge-hookshot/defaults/main.yml +++ b/roles/matrix-bridge-hookshot/defaults/main.yml @@ -10,7 +10,7 @@ matrix_hookshot_container_image_self_build: false matrix_hookshot_container_image_self_build_repo: "https://github.com/matrix-org/matrix-hookshot.git" matrix_hookshot_container_image_self_build_branch: "{{ 'main' if matrix_hookshot_version == 'latest' else matrix_hookshot_version }}" -matrix_hookshot_version: 2.3.0 +matrix_hookshot_version: 2.4.0 matrix_hookshot_docker_image: "{{ matrix_hookshot_docker_image_name_prefix }}halfshot/matrix-hookshot:{{ matrix_hookshot_version }}" matrix_hookshot_docker_image_name_prefix: "{{ 'localhost/' if matrix_hookshot_container_image_self_build else matrix_container_global_registry_prefix }}" From 7d043489bdc5533b188fe891692d1d85ec13ed93 Mon Sep 17 00:00:00 2001 From: smargold476 <105579587+smargold476@users.noreply.github.com> Date: Fri, 21 Oct 2022 21:51:54 +0200 Subject: [PATCH 057/118] update remote-repo-infos with link to doc was a litte confusing for me, so i added the link to the syntax-descr. --- docs/configuring-playbook-backup-borg.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuring-playbook-backup-borg.md b/docs/configuring-playbook-backup-borg.md index 41ca0156..72ec9e67 100644 --- a/docs/configuring-playbook-backup-borg.md +++ b/docs/configuring-playbook-backup-borg.md @@ -56,7 +56,7 @@ where: * USER - SSH user of a provider/server * HOST - SSH host of a provider/server -* REPO - borg repository name, it will be initialized on backup start, eg: `matrix` +* REPO - borg repository name, it will be initialized on backup start, eg: `matrix`, regarding Syntax see [Remote repositories](https://borgbackup.readthedocs.io/en/stable/usage/general.html#repository-urls) * PASSPHRASE - passphrase used for encrypting backups, you may generate it with `pwgen -s 64 1` or use any password manager * PRIVATE KEY - the content of the **private** part of the SSH key you created before. The whole key (all of its belonging lines) under `matrix_backup_borg_ssh_key_private` needs to be indented with 2 spaces From ac8312d95f58d573488ef49907830cbdf876a62b Mon Sep 17 00:00:00 2001 From: smargold476 <105579587+smargold476@users.noreply.github.com> Date: Fri, 21 Oct 2022 22:19:03 +0200 Subject: [PATCH 058/118] update link to example in my environment i see the matrix. conf in that file: /matrix/nginx-proxy/conf.d/matrix-domain.conf --- docs/configuring-playbook-ssl-certificates.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuring-playbook-ssl-certificates.md b/docs/configuring-playbook-ssl-certificates.md index 30a8f0b8..606160da 100644 --- a/docs/configuring-playbook-ssl-certificates.md +++ b/docs/configuring-playbook-ssl-certificates.md @@ -99,7 +99,7 @@ The certificate files would be made available in `/matrix/ssl/config/live/ Date: Sat, 22 Oct 2022 08:45:27 +0300 Subject: [PATCH 059/118] Make ntfy not try to listen on a privileged port We're starting it with `--user` and dropped capabilities, after all. Hopefully fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/2188 --- roles/matrix-ntfy/templates/ntfy/server.yml.j2 | 1 + roles/matrix-ntfy/templates/systemd/matrix-ntfy.service.j2 | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/matrix-ntfy/templates/ntfy/server.yml.j2 b/roles/matrix-ntfy/templates/ntfy/server.yml.j2 index 4cafcd62..096991a7 100644 --- a/roles/matrix-ntfy/templates/ntfy/server.yml.j2 +++ b/roles/matrix-ntfy/templates/ntfy/server.yml.j2 @@ -1,3 +1,4 @@ base_url: {{ matrix_ntfy_base_url }} behind_proxy: true cache_file: /data/cache.db +listen-http: :8080 diff --git a/roles/matrix-ntfy/templates/systemd/matrix-ntfy.service.j2 b/roles/matrix-ntfy/templates/systemd/matrix-ntfy.service.j2 index da292e5c..f4159856 100644 --- a/roles/matrix-ntfy/templates/systemd/matrix-ntfy.service.j2 +++ b/roles/matrix-ntfy/templates/systemd/matrix-ntfy.service.j2 @@ -21,7 +21,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-ntfy \ {% endfor %} --network={{ matrix_docker_network }} \ {% if matrix_ntfy_container_http_host_bind_port %} - -p {{ matrix_ntfy_container_http_host_bind_port }}:80 \ + -p {{ matrix_ntfy_container_http_host_bind_port }}:8080 \ {% endif %} --mount type=bind,src={{ matrix_ntfy_config_dir_path }},dst=/etc/ntfy,ro \ --mount type=bind,src={{ matrix_ntfy_data_path }},dst=/data \ From 3bd0af76ddec6004e9e794fd821e0809123cc4a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian-Samuel=20Geb=C3=BChr?= Date: Sat, 22 Oct 2022 07:47:00 +0200 Subject: [PATCH 060/118] Add information on how to manually run the backup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julian-Samuel Gebühr --- docs/configuring-playbook-backup-borg.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/configuring-playbook-backup-borg.md b/docs/configuring-playbook-backup-borg.md index 41ca0156..0fda66da 100644 --- a/docs/configuring-playbook-backup-borg.md +++ b/docs/configuring-playbook-backup-borg.md @@ -73,3 +73,9 @@ After configuring the playbook, run the [installation](installing.md) command ag ``` ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start ``` + +## Manually start a backup + +For testing your setup it can be helpful to not wait until 4am. If you want to run the backup immediately, log onto the server +and run `systemctl start matrix-backup-borg`. This will not return until the backup is done, so possibly a long time. +Consider using [tmux](https://en.wikipedia.org/wiki/Tmux) if your SSH connection is unstable. From 9fd3e00b71fdd450bf0643b30179f880e63d175c Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 23 Oct 2022 07:45:54 +0300 Subject: [PATCH 061/118] Upgrade nginx (1.23.1 -> 1.23.2) --- roles/matrix-nginx-proxy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 84ec4f7e..efef89af 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -1,7 +1,7 @@ --- # Project source code URL: https://github.com/nginx/nginx matrix_nginx_proxy_enabled: true -matrix_nginx_proxy_version: 1.23.1-alpine +matrix_nginx_proxy_version: 1.23.2-alpine # We use an official nginx image, which we fix-up to run unprivileged. # An alternative would be an `nginxinc/nginx-unprivileged` image, but From 716efe7ad5923cdbdc7721fc58f0db38f588a391 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 23 Oct 2022 09:52:02 +0300 Subject: [PATCH 062/118] Upgrade ddclient (v3.9.1-ls100 -> v3.10.0-ls102) --- roles/matrix-dynamic-dns/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-dynamic-dns/defaults/main.yml b/roles/matrix-dynamic-dns/defaults/main.yml index c465fef9..af4d9592 100644 --- a/roles/matrix-dynamic-dns/defaults/main.yml +++ b/roles/matrix-dynamic-dns/defaults/main.yml @@ -7,7 +7,7 @@ matrix_dynamic_dns_enabled: true # The dynamic dns daemon interval matrix_dynamic_dns_daemon_interval: '300' -matrix_dynamic_dns_version: v3.9.1-ls100 +matrix_dynamic_dns_version: v3.10.0-ls102 # The docker container to use when in mode matrix_dynamic_dns_docker_image: "{{ matrix_dynamic_dns_docker_image_name_prefix }}linuxserver/ddclient:{{ matrix_dynamic_dns_version }}" From d44b0378505ac5df4b9f6cafb93f46248234a5c4 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 23 Oct 2022 18:45:22 +0300 Subject: [PATCH 063/118] fgrep -> grep -F --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f0aeb397..576dcbf5 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: lint help: ## Show this help. - @fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/##//' + @grep -F -h "##" $(MAKEFILE_LIST) | grep -v grep | sed -e 's/\\$$//' | sed -e 's/##//' lint: ## Runs ansible-lint against all roles in the playbook ansible-lint From 24409766eb5cfeadc1f668e077532f0b8b78091d Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Mon, 24 Oct 2022 05:42:18 +0000 Subject: [PATCH 064/118] Fix 502 ref: 57bb340343c692c9439c863cd1d13c54cecb87b8 --- .../templates/nginx/conf.d/matrix-ntfy.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-ntfy.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-ntfy.conf.j2 index 988b3b35..ae100eda 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-ntfy.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-ntfy.conf.j2 @@ -21,7 +21,7 @@ {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; - set $backend "matrix-ntfy:80"; + set $backend "matrix-ntfy:8080"; proxy_pass http://$backend; {% else %} {# Generic configuration for use outside of our container setup #} From 9c549a185f6b035cb2979d0c41f9d7e2858037fc Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 25 Oct 2022 06:28:24 +0300 Subject: [PATCH 065/118] Auto-purge orphaned Let's Encrypt renewal configuration files --- roles/matrix-nginx-proxy/defaults/main.yml | 14 ++++++++++ ...urge_ssl_lets_encrypt_orphaned_configs.yml | 27 +++++++++++++++++++ .../tasks/ssl/setup_ssl_lets_encrypt.yml | 3 +++ 3 files changed, 44 insertions(+) create mode 100644 roles/matrix-nginx-proxy/tasks/ssl/purge_ssl_lets_encrypt_orphaned_configs.yml diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index efef89af..c233dc43 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -572,6 +572,20 @@ matrix_ssl_log_dir_path: "{{ matrix_ssl_base_path }}/log" matrix_ssl_pre_obtaining_required_service_name: ~ matrix_ssl_pre_obtaining_required_service_start_wait_time_seconds: 60 +# matrix_ssl_orphaned_renewal_configs_purging_enabled controls whether the playbook will delete Let's Encryption renewal configuration files (`/matrix/ssl/config/renewal/*.conf) +# for domains that are not part of the `matrix_ssl_domains_to_obtain_certificates_for` list. +# +# As the `matrix_ssl_domains_to_obtain_certificates_for` list changes over time, the playbook obtains certificates for various domains +# and sets up "renewal" configuration files to keep these certificates fresh. +# When a domain disappears from the `matrix_ssl_domains_to_obtain_certificates_for` list (because its associated service had gotten disabled), +# the certificate files and renewal configuration still remain in the filesystem and certbot may try to renewal the certificate for this domain. +# If there's no DNS record for this domain or it doesn't point to this server anymore, the `matrix-ssl-lets-encrypt-certificates-renew.service` systemd service +# won't be able to renew the certificate and will generate an error. +# +# With `matrix_ssl_orphaned_renewal_configs_purging_enabled` enabled, orphaned renewal configurations will be purged on each playbook run. +# Some other leftover files will still remain, but we don't bother purging them because they don't cause troubles. +matrix_ssl_orphaned_renewal_configs_purging_enabled: true + # Nginx Optimize SSL Session # # ssl_session_cache: diff --git a/roles/matrix-nginx-proxy/tasks/ssl/purge_ssl_lets_encrypt_orphaned_configs.yml b/roles/matrix-nginx-proxy/tasks/ssl/purge_ssl_lets_encrypt_orphaned_configs.yml new file mode 100644 index 00000000..51fd1f31 --- /dev/null +++ b/roles/matrix-nginx-proxy/tasks/ssl/purge_ssl_lets_encrypt_orphaned_configs.yml @@ -0,0 +1,27 @@ +--- + +- name: Check if a Let's Encrypt renewal configuration directory exists + ansible.builtin.stat: + path: "{{ matrix_ssl_config_dir_path }}/renewal" + register: matrix_ssl_config_renewal_directory_stat_result + +- when: matrix_ssl_config_renewal_directory_stat_result.stat.exists | bool + block: + - name: Determine current Let's Encrypt renewal configs + ansible.builtin.find: + path: "{{ matrix_ssl_config_dir_path }}/renewal" + patterns: ".*.conf$" + use_regex: true + register: matrix_ssl_current_renewal_config_files + + - name: Determine unnecessary Let's Encrypt renewal configs + ansible.builtin.set_fact: + matrix_ssl_current_renewal_config_files_to_purge: "{{ matrix_ssl_current_renewal_config_files_to_purge | default([]) + [item.path] }}" + with_items: "{{ matrix_ssl_current_renewal_config_files.files }}" + when: "item.path | basename | replace('.conf', '') not in matrix_ssl_domains_to_obtain_certificates_for" + + - name: Purge unneceessary Let's Encrypt renewal config files + ansible.builtin.file: + path: "{{ item }}" + state: absent + with_items: "{{ matrix_ssl_current_renewal_config_files_to_purge | default([]) }}" diff --git a/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml b/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml index 029ef860..62430f4b 100644 --- a/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml +++ b/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml @@ -18,6 +18,9 @@ - when: "matrix_ssl_retrieval_method == 'lets-encrypt'" block: + - when: matrix_ssl_orphaned_renewal_configs_purging_enabled | bool + ansible.builtin.import_tasks: "{{ role_path }}/tasks/ssl/purge_ssl_lets_encrypt_orphaned_configs.yml" + - name: Ensure certbot Docker image is pulled docker_image: name: "{{ matrix_ssl_lets_encrypt_certbot_docker_image }}" From cb2fecbea96ead3e9003c22bcabda00e20dd3b01 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 25 Oct 2022 06:43:57 +0300 Subject: [PATCH 066/118] Fix some ansible-lint-reported warnings --- roles/matrix-base/tasks/server_base/setup.yml | 3 ++- .../tasks/validate_config.yml | 3 ++- roles/matrix-ldap-registration-proxy/tasks/init.yml | 9 ++++----- roles/matrix-postgres/tasks/import_generic_sqlite_db.yml | 3 ++- roles/matrix-postgres/tasks/run_vacuum.yml | 3 ++- .../tasks/rust-synapse-compress-state/compress_room.yml | 3 ++- .../tasks/rust-synapse-compress-state/main.yml | 3 ++- 7 files changed, 16 insertions(+), 11 deletions(-) diff --git a/roles/matrix-base/tasks/server_base/setup.yml b/roles/matrix-base/tasks/server_base/setup.yml index 8cc4dff4..9e3319f5 100644 --- a/roles/matrix-base/tasks/server_base/setup.yml +++ b/roles/matrix-base/tasks/server_base/setup.yml @@ -21,7 +21,8 @@ register: lsb_release_installation_result - name: Reread ansible_lsb facts if lsb-release got installed - ansible.builtin.setup: filter=ansible_lsb* + ansible.builtin.setup: + filter: ansible_lsb* when: lsb_release_installation_result.changed - ansible.builtin.include_tasks: "{{ role_path }}/tasks/server_base/setup_debian.yml" diff --git a/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml b/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml index 901b760b..e005f162 100644 --- a/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml +++ b/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml @@ -22,5 +22,6 @@ - {'old': 'matrix_appservice_discord_container_expose_client_server_api_port', 'new': ''} - name: Require a valid database engine - ansible.builtin.fail: msg="`matrix_appservice_discord_database_engine` needs to be either 'sqlite' or 'postgres'" + ansible.builtin.fail: + msg: "`matrix_appservice_discord_database_engine` needs to be either 'sqlite' or 'postgres'" when: "matrix_appservice_discord_database_engine not in ['sqlite', 'postgres']" diff --git a/roles/matrix-ldap-registration-proxy/tasks/init.yml b/roles/matrix-ldap-registration-proxy/tasks/init.yml index 0b2051c3..40623609 100644 --- a/roles/matrix-ldap-registration-proxy/tasks/init.yml +++ b/roles/matrix-ldap-registration-proxy/tasks/init.yml @@ -10,7 +10,10 @@ matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-ldap-registration-proxy.service'] }}" when: matrix_ldap_registration_proxy_enabled | bool -- block: +- when: matrix_ldap_registration_proxy_enabled | bool + tags: + - always + block: - name: Fail if matrix-nginx-proxy role already executed ansible.builtin.fail: msg: >- @@ -52,7 +55,3 @@ URL endpoint to the matrix-ldap-proxy container. You can expose the container's port using the `matrix_ldap_registration_proxy_container_http_host_bind_port` variable. when: "not matrix_nginx_proxy_enabled | default(False) | bool" - - tags: - - always - when: matrix_ldap_registration_proxy_enabled | bool diff --git a/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml b/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml index 897ebc2e..4a515c27 100644 --- a/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml +++ b/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml @@ -28,7 +28,8 @@ - when: 'postgres_connection_string_variable_name is defined' block: - name: Fail if postgres_connection_string_variable_name points to an undefined variable - ansible.builtin.fail: msg="postgres_connection_string_variable_name is defined, but there is no variable with the name `{{ postgres_connection_string_variable_name }}`" + ansible.builtin.fail: + msg: "postgres_connection_string_variable_name is defined, but there is no variable with the name `{{ postgres_connection_string_variable_name }}`" when: "postgres_connection_string_variable_name not in vars" - name: Get Postgres connection string from variable diff --git a/roles/matrix-postgres/tasks/run_vacuum.yml b/roles/matrix-postgres/tasks/run_vacuum.yml index aafa761d..43959982 100644 --- a/roles/matrix-postgres/tasks/run_vacuum.yml +++ b/roles/matrix-postgres/tasks/run_vacuum.yml @@ -82,7 +82,8 @@ changed_when: matrix_postgres_synapse_vacuum_result.finished and matrix_postgres_synapse_vacuum_result.rc == 0 # Intentionally show the results -- ansible.builtin.debug: var="matrix_postgres_synapse_vacuum_result" +- ansible.builtin.debug: + var: "matrix_postgres_synapse_vacuum_result" - name: Ensure matrix-synapse is started, if it previously was ansible.builtin.service: diff --git a/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml b/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml index 6ae016fc..9834e256 100644 --- a/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml +++ b/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml @@ -24,7 +24,8 @@ failed_when: not matrix_synapse_rust_synapse_compress_state_compress_room_command_result.finished changed_when: matrix_synapse_rust_synapse_compress_state_compress_room_command_result.finished and matrix_synapse_rust_synapse_compress_state_compress_room_command_result.rc == 0 -- ansible.builtin.debug: var="matrix_synapse_rust_synapse_compress_state_compress_room_command_result" +- ansible.builtin.debug: + var: "matrix_synapse_rust_synapse_compress_state_compress_room_command_result" - name: Generate Postgres compression SQL import command ansible.builtin.set_fact: diff --git a/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml b/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml index b48e6077..80c25f7f 100644 --- a/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml +++ b/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml @@ -88,7 +88,8 @@ - when: "matrix_synapse_rust_synapse_compress_state_find_rooms_command_result.failed or matrix_synapse_rust_synapse_compress_state_find_rooms_command_result.stdout_lines | length != 4" block: - - ansible.builtin.debug: var="matrix_synapse_rust_synapse_compress_state_find_rooms_command_result" + - ansible.builtin.debug: + var: "matrix_synapse_rust_synapse_compress_state_find_rooms_command_result" - name: Fail if room find result is not what we expect ansible.builtin.fail: From fd7e1604a5880242b67d92fa1d8df4180fc53c6f Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 25 Oct 2022 09:16:24 +0000 Subject: [PATCH 067/118] Update appservice-irc 0.35.1 -> 0.36.0 --- roles/matrix-bridge-appservice-irc/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-appservice-irc/defaults/main.yml b/roles/matrix-bridge-appservice-irc/defaults/main.yml index c4fa75fe..d54a7685 100644 --- a/roles/matrix-bridge-appservice-irc/defaults/main.yml +++ b/roles/matrix-bridge-appservice-irc/defaults/main.yml @@ -11,7 +11,7 @@ matrix_appservice_irc_docker_src_files_path: "{{ matrix_base_data_path }}/appser # matrix_appservice_irc_version used to contain the full Docker image tag (e.g. `release-X.X.X`). # It's a bare version number now. We try to somewhat retain compatibility below. -matrix_appservice_irc_version: 0.35.1 +matrix_appservice_irc_version: 0.36.0 matrix_appservice_irc_docker_image: "{{ matrix_container_global_registry_prefix }}matrixdotorg/matrix-appservice-irc:{{ matrix_appservice_irc_docker_image_tag }}" matrix_appservice_irc_docker_image_tag: "{{ 'latest' if matrix_appservice_irc_version == 'latest' else ('release-' + matrix_appservice_irc_version) }}" matrix_appservice_irc_docker_image_force_pull: "{{ matrix_appservice_irc_docker_image.endswith(':latest') }}" From 81d8785811bd3aca8a6334910e2143edda8a3dda Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 25 Oct 2022 13:03:50 +0000 Subject: [PATCH 068/118] Update grafana 9.2.1 -> 9.2.2 --- roles/matrix-grafana/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-grafana/defaults/main.yml b/roles/matrix-grafana/defaults/main.yml index 767a749b..5bccb60d 100644 --- a/roles/matrix-grafana/defaults/main.yml +++ b/roles/matrix-grafana/defaults/main.yml @@ -5,7 +5,7 @@ matrix_grafana_enabled: true -matrix_grafana_version: 9.2.1 +matrix_grafana_version: 9.2.2 matrix_grafana_docker_image: "{{ matrix_container_global_registry_prefix }}grafana/grafana:{{ matrix_grafana_version }}" matrix_grafana_docker_image_force_pull: "{{ matrix_grafana_docker_image.endswith(':latest') }}" From 145a57a7b8ad1d01a3a05c28abe6586078611418 Mon Sep 17 00:00:00 2001 From: Aine Date: Tue, 25 Oct 2022 18:25:07 +0300 Subject: [PATCH 069/118] update honoroit 0.9.15 -> 0.9.16 --- group_vars/matrix_servers | 2 +- roles/matrix-bot-honoroit/defaults/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 6728eaaa..e31e0dba 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1163,7 +1163,7 @@ matrix_bot_honoroit_systemd_required_services_list: | # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_bot_honoroit_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" matrix_bot_honoroit_database_password: "{{ '%s' | format(matrix_homeserver_generic_secret_key) | password_hash('sha512', 'honoroit.bot.db') | to_uuid }}" -matrix_bot_honoroit_container_image_self_build: "{{ matrix_architecture not in ['amd64', 'arm32', 'arm64'] }}" +matrix_bot_honoroit_container_image_self_build: "{{ matrix_architecture not in ['amd64', 'arm64'] }}" ###################################################################### # diff --git a/roles/matrix-bot-honoroit/defaults/main.yml b/roles/matrix-bot-honoroit/defaults/main.yml index 68fb8c17..3510f473 100644 --- a/roles/matrix-bot-honoroit/defaults/main.yml +++ b/roles/matrix-bot-honoroit/defaults/main.yml @@ -9,7 +9,7 @@ matrix_bot_honoroit_docker_repo: "https://gitlab.com/etke.cc/honoroit.git" matrix_bot_honoroit_docker_repo_version: "{{ matrix_bot_honoroit_version }}" matrix_bot_honoroit_docker_src_files_path: "{{ matrix_base_data_path }}/honoroit/docker-src" -matrix_bot_honoroit_version: v0.9.15 +matrix_bot_honoroit_version: v0.9.16 matrix_bot_honoroit_docker_image: "{{ matrix_bot_honoroit_docker_image_name_prefix }}honoroit:{{ matrix_bot_honoroit_version }}" matrix_bot_honoroit_docker_image_name_prefix: "{{ 'localhost/' if matrix_bot_honoroit_container_image_self_build else 'registry.gitlab.com/etke.cc/' }}" matrix_bot_honoroit_docker_image_force_pull: "{{ matrix_bot_honoroit_docker_image.endswith(':latest') }}" From a22d5b1726a207f6349e47890684759a14e1e697 Mon Sep 17 00:00:00 2001 From: Aine Date: Tue, 25 Oct 2022 18:26:57 +0300 Subject: [PATCH 070/118] update postmoogle 0.9.7 -> 0.9.8 --- group_vars/matrix_servers | 2 +- roles/matrix-bot-postmoogle/defaults/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 6728eaaa..354b6691 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1227,7 +1227,7 @@ matrix_bot_postmoogle_systemd_required_services_list: | matrix_bot_postmoogle_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" matrix_bot_postmoogle_database_password: "{{ '%s' | format(matrix_homeserver_generic_secret_key) | password_hash('sha512', 'postmoogle.db') | to_uuid }}" -matrix_bot_postmoogle_container_image_self_build: "{{ matrix_architecture not in ['amd64', 'arm32', 'arm64'] }}" +matrix_bot_postmoogle_container_image_self_build: "{{ matrix_architecture not in ['amd64', 'arm64'] }}" ###################################################################### # diff --git a/roles/matrix-bot-postmoogle/defaults/main.yml b/roles/matrix-bot-postmoogle/defaults/main.yml index 1bcd570a..af6c23ac 100644 --- a/roles/matrix-bot-postmoogle/defaults/main.yml +++ b/roles/matrix-bot-postmoogle/defaults/main.yml @@ -9,7 +9,7 @@ matrix_bot_postmoogle_docker_repo: "https://gitlab.com/etke.cc/postmoogle.git" matrix_bot_postmoogle_docker_repo_version: "{{ 'main' if matrix_bot_postmoogle_version == 'latest' else matrix_bot_postmoogle_version }}" matrix_bot_postmoogle_docker_src_files_path: "{{ matrix_base_data_path }}/postmoogle/docker-src" -matrix_bot_postmoogle_version: v0.9.7 +matrix_bot_postmoogle_version: v0.9.8 matrix_bot_postmoogle_docker_image: "{{ matrix_bot_postmoogle_docker_image_name_prefix }}postmoogle:{{ matrix_bot_postmoogle_version }}" matrix_bot_postmoogle_docker_image_name_prefix: "{{ 'localhost/' if matrix_bot_postmoogle_container_image_self_build else 'registry.gitlab.com/etke.cc/' }}" matrix_bot_postmoogle_docker_image_force_pull: "{{ matrix_bot_postmoogle_docker_image.endswith(':latest') }}" From 12fe1f417c2816939755bfc5e1e61a376b59ea90 Mon Sep 17 00:00:00 2001 From: Aine Date: Tue, 25 Oct 2022 18:39:39 +0300 Subject: [PATCH 071/118] update buscarron 1.2.1 -> 1.3.0 --- docs/configuring-playbook-bot-buscarron.md | 4 +-- group_vars/matrix_servers | 2 +- roles/matrix-bot-buscarron/defaults/main.yml | 32 ++++++++++++++------ roles/matrix-bot-buscarron/templates/env.j2 | 6 +++- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/docs/configuring-playbook-bot-buscarron.md b/docs/configuring-playbook-bot-buscarron.md index 3a5822ab..b38f25c6 100644 --- a/docs/configuring-playbook-bot-buscarron.md +++ b/docs/configuring-playbook-bot-buscarron.md @@ -36,10 +36,10 @@ matrix_bot_buscarron_forms: room: "!yourRoomID:DOMAIN" # (mandatory) Room ID where form submission will be posted redirect: https://DOMAIN # (mandatory) To what page user will be redirected after the form submission ratelimit: 1r/m # (optional) rate limit of the form, format: r/, eg: 1r/s or 54r/m + hasemail: 1 # (optional) form has "email" field that should be validated extensions: [] # (optional) list of form extensions (not used yet) -matrix_bot_buscarron_spam_hosts: [] # (optional) list of email domains/hosts that should be rejected automatically -matrix_bot_buscarron_spam_emails: [] # (optional) list of email addresses that should be rejected automatically +matrix_bot_buscarron_spamlist: [] # (optional) list of emails/domains/hosts (with wildcards support) that should be rejected automatically ``` You will also need to add a DNS record so that buscarron can be accessed. diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 6728eaaa..5d463da2 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1194,7 +1194,7 @@ matrix_bot_buscarron_systemd_required_services_list: | # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_bot_buscarron_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" matrix_bot_buscarron_database_password: "{{ '%s' | format(matrix_homeserver_generic_secret_key) | password_hash('sha512', 'buscarron.bot.db') | to_uuid }}" -matrix_bot_buscarron_container_image_self_build: "{{ matrix_architecture not in ['amd64', 'arm32', 'arm64'] }}" +matrix_bot_buscarron_container_image_self_build: "{{ matrix_architecture not in ['amd64', 'arm64'] }}" ###################################################################### # diff --git a/roles/matrix-bot-buscarron/defaults/main.yml b/roles/matrix-bot-buscarron/defaults/main.yml index 648d5344..21d9a4a3 100644 --- a/roles/matrix-bot-buscarron/defaults/main.yml +++ b/roles/matrix-bot-buscarron/defaults/main.yml @@ -9,7 +9,7 @@ matrix_bot_buscarron_docker_repo: "https://gitlab.com/etke.cc/buscarron.git" matrix_bot_buscarron_docker_repo_version: "{{ matrix_bot_buscarron_version }}" matrix_bot_buscarron_docker_src_files_path: "{{ matrix_base_data_path }}/buscarron/docker-src" -matrix_bot_buscarron_version: v1.2.1 +matrix_bot_buscarron_version: v1.3.0 matrix_bot_buscarron_docker_image: "{{ matrix_bot_buscarron_docker_image_name_prefix }}buscarron:{{ matrix_bot_buscarron_version }}" matrix_bot_buscarron_docker_image_name_prefix: "{{ 'localhost/' if matrix_bot_buscarron_container_image_self_build else 'registry.gitlab.com/etke.cc/' }}" matrix_bot_buscarron_docker_image_force_pull: "{{ matrix_bot_buscarron_docker_image.endswith(':latest') }}" @@ -78,37 +78,49 @@ matrix_bot_buscarron_homeserver: "{{ matrix_homeserver_container_url }}" matrix_bot_buscarron_forms: [] # Disable encryption -matrix_bot_buscarron_noencryption: +matrix_bot_buscarron_noencryption: false # Sentry DSN -matrix_bot_buscarron_sentry: +matrix_bot_buscarron_sentry: '' # Log level matrix_bot_buscarron_loglevel: INFO -# spam hosts/domains +# list of spammers with wildcards support, eg: *@spam.com spam@*, spam@spam.com +matrix_bot_buscarron_spamlist: [] + +# spam hosts/domains. +# deprecated, use matrix_bot_buscarron_spamlist matrix_bot_buscarron_spam_hosts: [] # spam email addresses +# deprecated, use matrix_bot_buscarron_spamlist matrix_bot_buscarron_spam_emails: [] # spam email localparts +# deprecated, use matrix_bot_buscarron_spamlist matrix_bot_buscarron_spam_localparts: [] -# Ban duration in hours -matrix_bot_buscarron_ban_duration: 24 - # Banlist size matrix_bot_buscarron_ban_size: 10000 +# Permanent banlist +matrix_bot_buscarron_ban_list: [] + # Postmark token (confirmation emails) -matrix_bot_buscarron_pm_token: +matrix_bot_buscarron_pm_token: '' # Postmark sender signature -matrix_bot_buscarron_pm_from: +matrix_bot_buscarron_pm_from: '' # Postmark confirmation email's reply-to -matrix_bot_buscarron_pm_replyto: +matrix_bot_buscarron_pm_replyto: '' + +# email address (from) for SMTP validation. Must be valid email on valid SMTP server, otherwise it will be rejected by other servers +matrix_bot_buscarron_smtp_from: '' + +# enforce SMTP validation +matrix_bot_buscarron_smtp_validation: false # Additional environment variables to pass to the buscarron container # diff --git a/roles/matrix-bot-buscarron/templates/env.j2 b/roles/matrix-bot-buscarron/templates/env.j2 index 42da0d34..90ae5f7e 100644 --- a/roles/matrix-bot-buscarron/templates/env.j2 +++ b/roles/matrix-bot-buscarron/templates/env.j2 @@ -3,22 +3,26 @@ BUSCARRON_PASSWORD={{ matrix_bot_buscarron_password }} BUSCARRON_HOMESERVER={{ matrix_bot_buscarron_homeserver }} BUSCARRON_DB_DSN={{ matrix_bot_buscarron_database_connection_string }} BUSCARRON_DB_DIALECT={{ matrix_bot_buscarron_database_dialect }} +BUSCARRON_SPAMLIST={{ matrix_bot_buscarron_spamlist|join(" ") }} BUSCARRON_SPAM_HOSTS={{ matrix_bot_buscarron_spam_hosts|join(" ") }} BUSCARRON_SPAM_EMAILS={{ matrix_bot_buscarron_spam_emails|join(" ") }} BUSCARRON_SPAM_LOCALPARTS={{ matrix_bot_buscarron_spam_localparts|join(" ") }} BUSCARRON_SENTRY={{ matrix_bot_buscarron_sentry }} BUSCARRON_LOGLEVEL={{ matrix_bot_buscarron_loglevel }} -BUSCARRON_BAN_DURATION={{ matrix_bot_buscarron_ban_duration }} BUSCARRON_BAN_SIZE={{ matrix_bot_buscarron_ban_size }} +BUSCARRON_BAN_LIST={{ matrix_bot_buscarron_ban_list }} BUSCARRON_PM_TOKEN={{ matrix_bot_buscarron_pm_token }} BUSCARRON_PM_FROM={{ matrix_bot_buscarron_pm_from }} BUSCARRON_PM_REPLYTO={{ matrix_bot_buscarron_pm_replyto }} +BUSCARRON_SMTP_FROM={{ matrix_bot_buscarron_smtp_from }} +BUSCARRON_SMTP_VALIDATION={{ matrix_bot_buscarron_smtp_validation }} BUSCARRON_NOENCRYPTION={{ matrix_bot_buscarron_noencryption }} {% set forms = [] %} {% for form in matrix_bot_buscarron_forms -%}{{- forms.append(form.name) -}} BUSCARRON_{{ form.name|upper }}_ROOM={{ form.room|default('') }} BUSCARRON_{{ form.name|upper }}_REDIRECT={{ form.redirect|default('') }} BUSCARRON_{{ form.name|upper }}_HASDOMAIN={{ form.hasdomain|default('') }} +BUSCARRON_{{ form.name|upper }}_HASEMAIL={{ form.hasemail|default('') }} BUSCARRON_{{ form.name|upper }}_RATELIMIT={{ form.ratelimit|default('') }} BUSCARRON_{{ form.name|upper }}_EXTENSIONS={{ form.extensions|default('')|join(' ') }} BUSCARRON_{{ form.name|upper }}_CONFIRMATION_SUBJECT={{ form.confirmation_subject|default('') }} From 437d177d3133827379d303bba452480bd30d3ab9 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 25 Oct 2022 18:17:47 +0000 Subject: [PATCH 072/118] update element v1.11.10 -> v1.11.11 --- roles/matrix-client-element/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index c8e476a1..7204aa10 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -10,7 +10,7 @@ matrix_client_element_container_image_self_build_repo: "https://github.com/vecto # - https://github.com/vector-im/element-web/issues/19544 matrix_client_element_container_image_self_build_low_memory_system_patch_enabled: "{{ ansible_memtotal_mb < 4096 }}" -matrix_client_element_version: v1.11.10 +matrix_client_element_version: v1.11.11 matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:{{ matrix_client_element_version }}" matrix_client_element_docker_image_name_prefix: "{{ 'localhost/' if matrix_client_element_container_image_self_build else matrix_container_global_registry_prefix }}" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" From 9a439f91ae5b412756c51bb3dbf55643cd8127a9 Mon Sep 17 00:00:00 2001 From: Array in a Matrix Date: Wed, 26 Oct 2022 12:58:09 -0400 Subject: [PATCH 073/118] update way to login to discord --- docs/configuring-playbook-bridge-mautrix-discord.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuring-playbook-bridge-mautrix-discord.md b/docs/configuring-playbook-bridge-mautrix-discord.md index 9fbf1424..065609b2 100644 --- a/docs/configuring-playbook-bridge-mautrix-discord.md +++ b/docs/configuring-playbook-bridge-mautrix-discord.md @@ -12,7 +12,7 @@ See the project's [documentation](https://docs.mau.fi/bridges/go/discord/index.h ## Prerequisites -For using this bridge, you would **need to authenticate by scanning a QR code with the Discord app on your phone**. +For using this bridge, you would need to authenticate by **scanning a QR code** with the Discord app on your phone **or** by using **discord account token**. You can delete the Discord app after the authentication process. @@ -70,7 +70,7 @@ When using this method, **each user** that wishes to enable Double Puppeting nee ## Usage 1. Start a chat with `@discordbot:YOUR_DOMAIN` (where `YOUR_DOMAIN` is your base domain, not the `matrix.` domain). -2. Send a `login` command +2. If you would like to login to Discord using a token, send `login-token` command, otherwise, send `login-qr` command. 3. You'll see a QR code which you need to scan with the Discord app on your phone. You can scan it with the camera app too, which will open Discord, which will then instruct you to scan it a 2nd time in the Discord app. 4. After confirming (in the Discord app) that you'd like to allow this login, the bot should respond with "Succcessfully authenticated as ..." 5. Now that you're logged in, you can send a `help` command to the bot again, to see additional commands you have access to From 170960be1aafcdf2b094322f8c9f268b744a29bf Mon Sep 17 00:00:00 2001 From: Joe Kappus Date: Wed, 26 Oct 2022 18:47:09 -0400 Subject: [PATCH 074/118] update element v1.11.11 -> v1.11.12 fixes jitsi issue --- roles/matrix-client-element/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index 7204aa10..2d5c99d2 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -10,7 +10,7 @@ matrix_client_element_container_image_self_build_repo: "https://github.com/vecto # - https://github.com/vector-im/element-web/issues/19544 matrix_client_element_container_image_self_build_low_memory_system_patch_enabled: "{{ ansible_memtotal_mb < 4096 }}" -matrix_client_element_version: v1.11.11 +matrix_client_element_version: v1.11.12 matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:{{ matrix_client_element_version }}" matrix_client_element_docker_image_name_prefix: "{{ 'localhost/' if matrix_client_element_container_image_self_build else matrix_container_global_registry_prefix }}" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" From e5e5ee4f7237c8047abfe85fc2aa0e25eb38edb2 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 27 Oct 2022 06:45:37 +0300 Subject: [PATCH 075/118] Use fixed image tags for prodrigestivill/postgres-backup-local At some point, we can drop v9.6 support and use a single variable that holds this value. Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/2191 --- roles/matrix-postgres-backup/defaults/main.yml | 18 +++++++++--------- .../tasks/validate_config.yml | 11 +++++++++++ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/roles/matrix-postgres-backup/defaults/main.yml b/roles/matrix-postgres-backup/defaults/main.yml index ed42266f..d59c84e9 100644 --- a/roles/matrix-postgres-backup/defaults/main.yml +++ b/roles/matrix-postgres-backup/defaults/main.yml @@ -24,18 +24,18 @@ matrix_postgres_backup_postgres_data_path: "" matrix_postgres_backup_architecture: amd64 -# matrix_postgres_docker_image_suffix controls whether we use Alpine-based images (`-alpine`) or the normal Debian-based images. +# matrix_postgres_backup_docker_image_distro controls whether we use Alpine-based images (`-alpine`) or the normal Debian-based images. # Alpine-based Postgres images are smaller and we usually prefer them, but they don't work on ARM32 (tested on a Raspberry Pi 3 running Raspbian 10.7). # On ARM32, `-alpine` images fail with the following error: # > LOG: startup process (PID 37) was terminated by signal 11: Segmentation fault -matrix_postgres_backup_docker_image_suffix: "{{ '-alpine' if matrix_postgres_backup_architecture in ['amd64', 'arm64'] else '' }}" - -matrix_postgres_backup_docker_image_v9: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:9.6{{ matrix_postgres_backup_docker_image_suffix }}" -matrix_postgres_backup_docker_image_v10: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:10{{ matrix_postgres_backup_docker_image_suffix }}" -matrix_postgres_backup_docker_image_v11: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:11{{ matrix_postgres_backup_docker_image_suffix }}" -matrix_postgres_backup_docker_image_v12: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:12{{ matrix_postgres_backup_docker_image_suffix }}" -matrix_postgres_backup_docker_image_v13: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:13{{ matrix_postgres_backup_docker_image_suffix }}" -matrix_postgres_backup_docker_image_v14: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:14{{ matrix_postgres_backup_docker_image_suffix }}" +matrix_postgres_backup_docker_image_distro: "{{ 'alpine' if matrix_postgres_backup_architecture in ['amd64', 'arm64'] else 'debian' }}" + +matrix_postgres_backup_docker_image_v9: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:9.6-{{ matrix_postgres_backup_docker_image_distro }}-2aa03d1" +matrix_postgres_backup_docker_image_v10: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:10-{{ matrix_postgres_backup_docker_image_distro }}-2cf00a5" +matrix_postgres_backup_docker_image_v11: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:11-{{ matrix_postgres_backup_docker_image_distro }}-2cf00a5" +matrix_postgres_backup_docker_image_v12: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:12-{{ matrix_postgres_backup_docker_image_distro }}-2cf00a5" +matrix_postgres_backup_docker_image_v13: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:13-{{ matrix_postgres_backup_docker_image_distro }}-2cf00a5" +matrix_postgres_backup_docker_image_v14: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:14-{{ matrix_postgres_backup_docker_image_distro }}-2cf00a5" matrix_postgres_backup_docker_image_latest: "{{ matrix_postgres_backup_docker_image_v14 }}" # This variable is assigned at runtime. Overriding its value has no effect. diff --git a/roles/matrix-postgres-backup/tasks/validate_config.yml b/roles/matrix-postgres-backup/tasks/validate_config.yml index 8a2ddb5a..2ba64374 100644 --- a/roles/matrix-postgres-backup/tasks/validate_config.yml +++ b/roles/matrix-postgres-backup/tasks/validate_config.yml @@ -16,3 +16,14 @@ - "matrix_postgres_backup_keep_months" - "matrix_postgres_backup_path" - "matrix_postgres_backup_databases" + +- name: (Deprecation) Catch and report renamed settings + ansible.builtin.fail: + msg: >- + Your configuration contains a variable, which now has a different name. + Please change your configuration to rename the variable (`{{ item.old }}` -> `{{ item.new }}`). + when: "item.old in vars" + with_items: + - {'old': 'matrix_postgres_backup_docker_image_suffix', 'new': 'matrix_postgres_backup_docker_image_distro'} + + From 6cacf7297d7dc88a37537569fd9d9fc81db62176 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 27 Oct 2022 06:50:04 +0300 Subject: [PATCH 076/118] Add support for postgres-backup-local v15 Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/2200 --- roles/matrix-postgres-backup/defaults/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/matrix-postgres-backup/defaults/main.yml b/roles/matrix-postgres-backup/defaults/main.yml index d59c84e9..abdfa74c 100644 --- a/roles/matrix-postgres-backup/defaults/main.yml +++ b/roles/matrix-postgres-backup/defaults/main.yml @@ -36,7 +36,8 @@ matrix_postgres_backup_docker_image_v11: "{{ matrix_container_global_registry_pr matrix_postgres_backup_docker_image_v12: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:12-{{ matrix_postgres_backup_docker_image_distro }}-2cf00a5" matrix_postgres_backup_docker_image_v13: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:13-{{ matrix_postgres_backup_docker_image_distro }}-2cf00a5" matrix_postgres_backup_docker_image_v14: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:14-{{ matrix_postgres_backup_docker_image_distro }}-2cf00a5" -matrix_postgres_backup_docker_image_latest: "{{ matrix_postgres_backup_docker_image_v14 }}" +matrix_postgres_backup_docker_image_v15: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:15-{{ matrix_postgres_backup_docker_image_distro }}-2cf00a5" +matrix_postgres_backup_docker_image_latest: "{{ matrix_postgres_backup_docker_image_v15 }}" # This variable is assigned at runtime. Overriding its value has no effect. matrix_postgres_backup_docker_image_to_use: '{{ matrix_postgres_backup_docker_image_latest }}' From 35c5c23a3e4c0f0d9981f18895a0da300a7ea8ac Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 27 Oct 2022 06:50:41 +0300 Subject: [PATCH 077/118] Remove useless tasks from matrix-postgres-backup role `matrix-postgres-backup` reuses the `matrix-postgres` role for these tasks, not its own definitions. --- .../util/detect_existing_postgres_version.yml | 61 ------------------- 1 file changed, 61 deletions(-) delete mode 100644 roles/matrix-postgres-backup/tasks/util/detect_existing_postgres_version.yml diff --git a/roles/matrix-postgres-backup/tasks/util/detect_existing_postgres_version.yml b/roles/matrix-postgres-backup/tasks/util/detect_existing_postgres_version.yml deleted file mode 100644 index 877e5934..00000000 --- a/roles/matrix-postgres-backup/tasks/util/detect_existing_postgres_version.yml +++ /dev/null @@ -1,61 +0,0 @@ ---- - -# This utility aims to determine if there is some existing Postgres version in use or not. -# If there is, it also tries to detect the Docker image that corresponds to that version. - -- name: Initialize Postgres version determination variables (default to empty) - ansible.builtin.set_fact: - matrix_postgres_backup_detection_pg_version_path: "{{ matrix_postgres_data_path }}/PG_VERSION" - matrix_postgres_backup_detected_existing: false - matrix_postgres_backup_detected_version: "" - matrix_postgres_backup_detected_version_corresponding_docker_image: "" - -- name: Determine existing Postgres version (check PG_VERSION file) - ansible.builtin.stat: - path: "{{ matrix_postgres_backup_detection_pg_version_path }}" - register: result_pg_version_stat - -- ansible.builtin.set_fact: - matrix_postgres_backup_detected_existing: true - when: "result_pg_version_stat.stat.exists" - -- name: Determine existing Postgres version (read PG_VERSION file) - ansible.builtin.slurp: - src: "{{ matrix_postgres_backup_detection_pg_version_path }}" - register: result_pg_version - when: matrix_postgres_backup_detected_existing | bool - -- name: Determine existing Postgres version (make sense of PG_VERSION file) - ansible.builtin.set_fact: - matrix_postgres_backup_detected_version: "{{ result_pg_version['content'] | b64decode | replace('\n', '') }}" - when: matrix_postgres_backup_detected_existing | bool - -- name: Determine corresponding Docker image to detected version (assume default of latest) - ansible.builtin.set_fact: - matrix_postgres_backup_detected_version_corresponding_docker_image: "{{ matrix_postgres_backup_docker_image_latest }}" - when: "matrix_postgres_backup_detected_version != ''" - -- name: Determine corresponding Docker image to detected version (use 9.x, if detected) - ansible.builtin.set_fact: - matrix_postgres_backup_detected_version_corresponding_docker_image: "{{ matrix_postgres_backup_docker_image_v9 }}" - when: "matrix_postgres_backup_detected_version.startswith('9.')" - -- name: Determine corresponding Docker image to detected version (use 10.x, if detected) - ansible.builtin.set_fact: - matrix_postgres_backup_detected_version_corresponding_docker_image: "{{ matrix_postgres_backup_docker_image_v10 }}" - when: "matrix_postgres_backup_detected_version == '10' or matrix_postgres_backup_detected_version.startswith('10.')" - -- name: Determine corresponding Docker image to detected version (use 11.x, if detected) - ansible.builtin.set_fact: - matrix_postgres_backup_detected_version_corresponding_docker_image: "{{ matrix_postgres_backup_docker_image_v11 }}" - when: "matrix_postgres_backup_detected_version == '11' or matrix_postgres_backup_detected_version.startswith('11.')" - -- name: Determine corresponding Docker image to detected version (use 12.x, if detected) - ansible.builtin.set_fact: - matrix_postgres_backup_detected_version_corresponding_docker_image: "{{ matrix_postgres_backup_docker_image_v12 }}" - when: "matrix_postgres_backup_detected_version == '12' or matrix_postgres_backup_detected_version.startswith('12.')" - -- name: Determine corresponding Docker image to detected version (use 13.x, if detected) - ansible.builtin.set_fact: - matrix_postgres_backup_detected_version_corresponding_docker_image: "{{ matrix_postgres_backup_docker_image_v13 }}" - when: "matrix_postgres_backup_detected_version == '13' or matrix_postgres_backup_detected_version.startswith('13.')" From ce8e9be6a857c2541a00b22a3ec14ca01cd41562 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 27 Oct 2022 09:48:49 +0300 Subject: [PATCH 078/118] Document Conduit installation Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/2076 --- docs/configuring-playbook-conduit.md | 58 ++++++++++++++++++++++++++++ docs/configuring-playbook.md | 5 ++- examples/vars.yml | 4 +- 3 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 docs/configuring-playbook-conduit.md diff --git a/docs/configuring-playbook-conduit.md b/docs/configuring-playbook-conduit.md new file mode 100644 index 00000000..8739a567 --- /dev/null +++ b/docs/configuring-playbook-conduit.md @@ -0,0 +1,58 @@ +# Configuring Conduit (optional) + +By default, this playbook configures the [Synapse](https://github.com/matrix-org/synapse) Matrix server, but you can also use [Conduit](https://conduit.rs). + +**NOTES**: + +- **You can't switch an existing Matrix server's implementation** (e.g. Synapse -> Conduit). Proceed below only if you're OK with losing data or you're dealing with a server on a new domain name, which hasn't participated in the Matrix federation yet. + +- **homeserver implementations other than Synapse may not be fully functional**. The playbook may also not assist you in an optimal way (like it does with Synapse). Make yourself familiar with the downsides before proceeding + + +## Installation + +To use Conduit, you **generally** need the following additional `vars.yml` configuration: + +```yaml +matrix_homeserver_implementation: conduit +``` + +However, since Conduit is difficult (see [famedly/conduit#276](https://gitlab.com/famedly/conduit/-/issues/276) and [famedly/conduit#354](https://gitlab.com/famedly/conduit/-/merge_requests/354)) when it comes to creating the first user account and does not support [registering users](registering-users.md) (via the command line or via the playbook) like Synapse and Dendrite do, we recommend the following flow: + +1. Add `matrix_conduit_allow_registration: true` to your `vars.yml` the first time around, temporarily +2. Run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start` - see [Installing](installing.md)) +3. Create your first user via Element or any other client which supports creating users +4. Get rid of `matrix_conduit_allow_registration: true` from your `vars.yml` +5. Run the playbook again (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-conduit,start` would be enough this time) +6. You can now use your server safely. Additional users can be created by messaging the internal Conduit bot + + +## Configuring bridges / appservices + +Automatic appservice setup is currently unsupported when using conduit. After setting up the service as usual you may notice that it is unable to start. + +You will have to manually register appservices using the the [register-appservice](https://gitlab.com/famedly/conduit/-/blob/next/APPSERVICES.md) command. + +Find the `registration.yaml` in the `/matrix` directory, for example `/matrix/mautrix-signal/bridge/registration.yaml`, then pass the content to conduit: + + + @conduit:your.server.name: register-appservice + ``` + as_token: + de.sorunome.msc2409.push_ephemeral: true + hs_token: + id: signal + namespaces: + aliases: + - exclusive: true + regex: ^#signal_.+:example\.org$ + users: + - exclusive: true + regex: ^@signal_.+:example\.org$ + - exclusive: true + regex: ^@signalbot:example\.org$ + rate_limited: false + sender_localpart: _bot_signalbot + url: http://matrix-mautrix-signal:29328 + ``` + diff --git a/docs/configuring-playbook.md b/docs/configuring-playbook.md index efb0f7e3..127ab47d 100644 --- a/docs/configuring-playbook.md +++ b/docs/configuring-playbook.md @@ -42,7 +42,10 @@ When you're done with all the configuration you'd like to do, continue with [Ins ### Core service adjustments -- [Configuring Synapse](configuring-playbook-synapse.md) (optional) +- Homeserver configuration: + - [Configuring Synapse](configuring-playbook-synapse.md), if you're going with the default/recommended homeserver implementation (optional) + + - [Configuring Conduit](configuring-playbook-conduit.md), if you've switched to the [Conduit](https://conduit.rs) homeserver implementation (optional) - [Configuring Element](configuring-playbook-client-element.md) (optional) diff --git a/examples/vars.yml b/examples/vars.yml index 3ca8f460..248f906b 100644 --- a/examples/vars.yml +++ b/examples/vars.yml @@ -12,7 +12,9 @@ matrix_domain: YOUR_BARE_DOMAIN_NAME_HERE # The Matrix homeserver software to install. -# See `roles/matrix-base/defaults/main.yml` for valid options. +# See: +# - `roles/matrix-base/defaults/main.yml` for valid options +# - the `docs/configuring-playbook-IMPLEMENTATION_NAME.md` documentation page, if one is available for your implementation choice matrix_homeserver_implementation: synapse # A secret used as a base, for generating various other secrets. From 7303f9241c583ce17a9bc224092c89c73d0233e1 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 27 Oct 2022 09:59:54 +0300 Subject: [PATCH 079/118] Fix yamllint error --- roles/matrix-postgres-backup/tasks/validate_config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/roles/matrix-postgres-backup/tasks/validate_config.yml b/roles/matrix-postgres-backup/tasks/validate_config.yml index 2ba64374..aab68eaf 100644 --- a/roles/matrix-postgres-backup/tasks/validate_config.yml +++ b/roles/matrix-postgres-backup/tasks/validate_config.yml @@ -25,5 +25,3 @@ when: "item.old in vars" with_items: - {'old': 'matrix_postgres_backup_docker_image_suffix', 'new': 'matrix_postgres_backup_docker_image_distro'} - - From 45226f31410e89ab9a116aa061c293dda4aa2985 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Thu, 27 Oct 2022 08:14:31 +0000 Subject: [PATCH 080/118] Update signald 0.22.2 -> 0.23.0 ref: https://gitlab.com/signald/signald/-/issues/343 --- roles/matrix-bridge-mautrix-signal/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-mautrix-signal/defaults/main.yml b/roles/matrix-bridge-mautrix-signal/defaults/main.yml index 9a06e423..83b0a5ce 100644 --- a/roles/matrix-bridge-mautrix-signal/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-signal/defaults/main.yml @@ -10,7 +10,7 @@ matrix_mautrix_signal_docker_repo_version: "{{ 'master' if matrix_mautrix_signal matrix_mautrix_signal_docker_src_files_path: "{{ matrix_base_data_path }}/mautrix-signal/docker-src" matrix_mautrix_signal_version: v0.4.0 -matrix_mautrix_signal_daemon_version: 0.22.2 +matrix_mautrix_signal_daemon_version: 0.23.0 # See: https://mau.dev/mautrix/signal/container_registry matrix_mautrix_signal_docker_image: "dock.mau.dev/mautrix/signal:{{ matrix_mautrix_signal_version }}" matrix_mautrix_signal_docker_image_force_pull: "{{ matrix_mautrix_signal_docker_image.endswith(':latest') }}" From 3c31638de969e5b1a7fd7fe3e6ced64125afc156 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Thu, 27 Oct 2022 10:45:08 +0000 Subject: [PATCH 081/118] fix buscarron ban list --- roles/matrix-bot-buscarron/templates/env.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bot-buscarron/templates/env.j2 b/roles/matrix-bot-buscarron/templates/env.j2 index 90ae5f7e..80ddd38c 100644 --- a/roles/matrix-bot-buscarron/templates/env.j2 +++ b/roles/matrix-bot-buscarron/templates/env.j2 @@ -10,7 +10,7 @@ BUSCARRON_SPAM_LOCALPARTS={{ matrix_bot_buscarron_spam_localparts|join(" ") }} BUSCARRON_SENTRY={{ matrix_bot_buscarron_sentry }} BUSCARRON_LOGLEVEL={{ matrix_bot_buscarron_loglevel }} BUSCARRON_BAN_SIZE={{ matrix_bot_buscarron_ban_size }} -BUSCARRON_BAN_LIST={{ matrix_bot_buscarron_ban_list }} +BUSCARRON_BAN_LIST={{ matrix_bot_buscarron_ban_list|default('')|join(' ') }} BUSCARRON_PM_TOKEN={{ matrix_bot_buscarron_pm_token }} BUSCARRON_PM_FROM={{ matrix_bot_buscarron_pm_from }} BUSCARRON_PM_REPLYTO={{ matrix_bot_buscarron_pm_replyto }} From 48388a3d96218fe61a7501c131f67ccf61718254 Mon Sep 17 00:00:00 2001 From: Sebastian Gumprich Date: Fri, 28 Oct 2022 13:20:17 +0200 Subject: [PATCH 082/118] use fqcns for some task Signed-off-by: Sebastian Gumprich --- roles/matrix-backup-borg/tasks/setup_install.yml | 4 ++-- roles/matrix-backup-borg/tasks/setup_uninstall.yml | 2 +- roles/matrix-base/tasks/server_base/setup_archlinux.yml | 4 ++-- roles/matrix-base/tasks/setup_matrix_base.yml | 2 +- roles/matrix-base/tasks/util/ensure_fuse_installed.yml | 2 +- roles/matrix-base/tasks/util/ensure_openssl_installed.yml | 2 +- roles/matrix-bot-buscarron/tasks/setup_install.yml | 4 ++-- roles/matrix-bot-buscarron/tasks/setup_uninstall.yml | 2 +- roles/matrix-bot-go-neb/tasks/setup_install.yml | 2 +- roles/matrix-bot-go-neb/tasks/setup_uninstall.yml | 2 +- roles/matrix-bot-honoroit/tasks/setup_install.yml | 4 ++-- roles/matrix-bot-honoroit/tasks/setup_uninstall.yml | 2 +- .../tasks/setup_install.yml | 4 ++-- .../tasks/setup_uninstall.yml | 2 +- .../tasks/setup_install.yml | 4 ++-- .../tasks/setup_uninstall.yml | 2 +- roles/matrix-bot-maubot/tasks/setup_install.yml | 4 ++-- roles/matrix-bot-maubot/tasks/setup_uninstall.yml | 2 +- roles/matrix-bot-mjolnir/tasks/setup_install.yml | 4 ++-- roles/matrix-bot-mjolnir/tasks/setup_uninstall.yml | 2 +- roles/matrix-bot-postmoogle/tasks/setup_install.yml | 4 ++-- roles/matrix-bot-postmoogle/tasks/setup_uninstall.yml | 2 +- .../tasks/setup_install.yml | 2 +- .../matrix-bridge-appservice-irc/tasks/setup_install.yml | 4 ++-- .../tasks/setup_install.yml | 8 ++++---- .../tasks/setup_install.yml | 4 ++-- .../tasks/setup_install.yml | 4 ++-- .../matrix-bridge-beeper-linkedin/tasks/setup_install.yml | 4 ++-- .../matrix-bridge-go-skype-bridge/tasks/setup_install.yml | 4 ++-- roles/matrix-bridge-heisenbridge/tasks/setup_install.yml | 2 +- roles/matrix-bridge-hookshot/tasks/setup_install.yml | 4 ++-- .../matrix-bridge-mautrix-discord/tasks/setup_install.yml | 4 ++-- .../tasks/setup_install.yml | 4 ++-- .../tasks/setup_install.yml | 4 ++-- .../tasks/setup_install.yml | 4 ++-- .../tasks/setup_install.yml | 4 ++-- .../matrix-bridge-mautrix-signal/tasks/setup_install.yml | 8 ++++---- .../tasks/setup_install.yml | 6 +++--- .../matrix-bridge-mautrix-twitter/tasks/setup_install.yml | 4 ++-- .../tasks/setup_install.yml | 4 ++-- .../tasks/setup_install.yml | 4 ++-- .../tasks/setup_install.yml | 4 ++-- .../tasks/setup_install.yml | 4 ++-- .../matrix-bridge-mx-puppet-slack/tasks/setup_install.yml | 4 ++-- .../matrix-bridge-mx-puppet-steam/tasks/setup_install.yml | 4 ++-- .../tasks/setup_install.yml | 4 ++-- roles/matrix-bridge-sms/tasks/setup_install.yml | 2 +- roles/matrix-cactus-comments/tasks/setup_install.yml | 4 ++-- roles/matrix-cactus-comments/tasks/setup_uninstall.yml | 2 +- roles/matrix-client-cinny/tasks/setup_install.yml | 4 ++-- roles/matrix-client-cinny/tasks/setup_uninstall.yml | 2 +- roles/matrix-client-element/tasks/setup_install.yml | 4 ++-- roles/matrix-client-element/tasks/setup_uninstall.yml | 2 +- roles/matrix-client-hydrogen/tasks/setup_install.yml | 4 ++-- roles/matrix-client-hydrogen/tasks/setup_uninstall.yml | 2 +- roles/matrix-conduit/tasks/conduit/setup_install.yml | 2 +- roles/matrix-conduit/tasks/conduit/setup_uninstall.yml | 2 +- roles/matrix-corporal/tasks/setup_corporal.yml | 6 +++--- roles/matrix-coturn/tasks/setup_install.yml | 6 +++--- roles/matrix-dendrite/tasks/dendrite/setup_install.yml | 2 +- roles/matrix-dendrite/tasks/dendrite/setup_uninstall.yml | 2 +- roles/matrix-dimension/tasks/setup_install.yml | 4 ++-- roles/matrix-dimension/tasks/setup_uninstall.yml | 2 +- roles/matrix-dynamic-dns/tasks/install.yml | 4 ++-- roles/matrix-email2matrix/tasks/setup_install.yml | 4 ++-- roles/matrix-email2matrix/tasks/setup_uninstall.yml | 2 +- roles/matrix-etherpad/tasks/setup_install.yml | 2 +- roles/matrix-etherpad/tasks/setup_uninstall.yml | 2 +- roles/matrix-grafana/tasks/setup.yml | 2 +- roles/matrix-jitsi/tasks/setup_jitsi_jicofo.yml | 2 +- roles/matrix-jitsi/tasks/setup_jitsi_jvb.yml | 2 +- roles/matrix-jitsi/tasks/setup_jitsi_prosody.yml | 2 +- roles/matrix-jitsi/tasks/setup_jitsi_web.yml | 2 +- .../tasks/setup_install.yml | 2 +- .../tasks/setup_uninstall.yml | 2 +- roles/matrix-ma1sd/tasks/setup_install.yml | 6 +++--- roles/matrix-ma1sd/tasks/setup_uninstall.yml | 2 +- roles/matrix-mailer/tasks/setup_mailer.yml | 6 +++--- .../tasks/nginx-proxy/setup_metrics_auth.yml | 2 +- roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml | 2 +- .../tasks/ssl/setup_ssl_lets_encrypt.yml | 2 +- roles/matrix-ntfy/tasks/setup_install.yml | 2 +- roles/matrix-ntfy/tasks/setup_uninstall.yml | 2 +- .../tasks/setup_postgres_backup.yml | 2 +- roles/matrix-postgres/tasks/migrate_db_to_postgres.yml | 4 ++-- roles/matrix-postgres/tasks/setup_postgres.yml | 2 +- roles/matrix-prometheus-node-exporter/tasks/setup.yml | 2 +- roles/matrix-prometheus-postgres-exporter/tasks/setup.yml | 2 +- roles/matrix-prometheus/tasks/setup_install.yml | 2 +- roles/matrix-redis/tasks/setup_redis.yml | 2 +- roles/matrix-registration/tasks/setup_install.yml | 4 ++-- roles/matrix-registration/tasks/setup_uninstall.yml | 2 +- roles/matrix-sygnal/tasks/setup_install.yml | 2 +- roles/matrix-sygnal/tasks/setup_uninstall.yml | 2 +- roles/matrix-synapse-admin/tasks/setup.yml | 6 +++--- .../tasks/ext/mjolnir-antispam/setup_install.yml | 2 +- .../tasks/ext/synapse-simple-antispam/setup_install.yml | 2 +- roles/matrix-synapse/tasks/goofys/setup_install.yml | 2 +- roles/matrix-synapse/tasks/goofys/setup_uninstall.yml | 2 +- roles/matrix-synapse/tasks/import_media_store.yml | 2 +- .../tasks/rust-synapse-compress-state/main.yml | 2 +- roles/matrix-synapse/tasks/synapse/setup_install.yml | 4 ++-- roles/matrix-synapse/tasks/synapse/setup_uninstall.yml | 2 +- 103 files changed, 159 insertions(+), 159 deletions(-) diff --git a/roles/matrix-backup-borg/tasks/setup_install.yml b/roles/matrix-backup-borg/tasks/setup_install.yml index b44a8fa1..f99051e3 100644 --- a/roles/matrix-backup-borg/tasks/setup_install.yml +++ b/roles/matrix-backup-borg/tasks/setup_install.yml @@ -59,7 +59,7 @@ mode: 0600 - name: Ensure borg image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_backup_borg_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_backup_borg_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -82,7 +82,7 @@ when: "matrix_backup_borg_container_image_self_build | bool" - name: Ensure borg image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_backup_borg_docker_image }}" source: build force_source: "{{ matrix_backup_borg_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-backup-borg/tasks/setup_uninstall.yml b/roles/matrix-backup-borg/tasks/setup_uninstall.yml index fb583f57..37832b2b 100644 --- a/roles/matrix-backup-borg/tasks/setup_uninstall.yml +++ b/roles/matrix-backup-borg/tasks/setup_uninstall.yml @@ -36,6 +36,6 @@ state: absent - name: Ensure borg Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_backup_borg_docker_image }}" state: absent diff --git a/roles/matrix-base/tasks/server_base/setup_archlinux.yml b/roles/matrix-base/tasks/server_base/setup_archlinux.yml index c912e58f..a9313614 100644 --- a/roles/matrix-base/tasks/server_base/setup_archlinux.yml +++ b/roles/matrix-base/tasks/server_base/setup_archlinux.yml @@ -1,7 +1,7 @@ --- - name: Install host dependencies - pacman: + community.general.pacman: name: - python-docker - python-dnspython @@ -9,7 +9,7 @@ update_cache: true - name: Ensure Docker is installed - pacman: + community.general.pacman: name: - docker state: present diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index 2e860987..6eebe3c0 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -20,7 +20,7 @@ when: "matrix_vars_yml_snapshotting_enabled | bool" - name: Ensure Matrix network is created in Docker - docker_network: + community.docker.docker_network: name: "{{ matrix_docker_network }}" driver: bridge diff --git a/roles/matrix-base/tasks/util/ensure_fuse_installed.yml b/roles/matrix-base/tasks/util/ensure_fuse_installed.yml index 47d2d9e8..240a5c62 100644 --- a/roles/matrix-base/tasks/util/ensure_fuse_installed.yml +++ b/roles/matrix-base/tasks/util/ensure_fuse_installed.yml @@ -16,7 +16,7 @@ when: ansible_os_family == 'Debian' - name: Ensure fuse installed (Archlinux) - pacman: + community.general.pacman: name: - fuse3 state: present diff --git a/roles/matrix-base/tasks/util/ensure_openssl_installed.yml b/roles/matrix-base/tasks/util/ensure_openssl_installed.yml index ae22fb49..a5bdf21a 100644 --- a/roles/matrix-base/tasks/util/ensure_openssl_installed.yml +++ b/roles/matrix-base/tasks/util/ensure_openssl_installed.yml @@ -16,7 +16,7 @@ when: ansible_os_family == 'Debian' - name: Ensure openssl installed (Archlinux) - pacman: + community.general.pacman: name: - openssl state: present diff --git a/roles/matrix-bot-buscarron/tasks/setup_install.yml b/roles/matrix-bot-buscarron/tasks/setup_install.yml index 564ff7e0..4a23d7e7 100644 --- a/roles/matrix-bot-buscarron/tasks/setup_install.yml +++ b/roles/matrix-bot-buscarron/tasks/setup_install.yml @@ -50,7 +50,7 @@ mode: 0640 - name: Ensure buscarron image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_buscarron_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_bot_buscarron_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -73,7 +73,7 @@ when: "matrix_bot_buscarron_container_image_self_build | bool" - name: Ensure buscarron image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_buscarron_docker_image }}" source: build force_source: "{{ matrix_bot_buscarron_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bot-buscarron/tasks/setup_uninstall.yml b/roles/matrix-bot-buscarron/tasks/setup_uninstall.yml index ad9e78cd..848d24d2 100644 --- a/roles/matrix-bot-buscarron/tasks/setup_uninstall.yml +++ b/roles/matrix-bot-buscarron/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure buscarron Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_buscarron_docker_image }}" state: absent diff --git a/roles/matrix-bot-go-neb/tasks/setup_install.yml b/roles/matrix-bot-go-neb/tasks/setup_install.yml index a651c160..52215597 100644 --- a/roles/matrix-bot-go-neb/tasks/setup_install.yml +++ b/roles/matrix-bot-go-neb/tasks/setup_install.yml @@ -17,7 +17,7 @@ when: "item.when | bool" - name: Ensure go-neb image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_go_neb_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_bot_go_neb_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bot-go-neb/tasks/setup_uninstall.yml b/roles/matrix-bot-go-neb/tasks/setup_uninstall.yml index 9794a90a..83391094 100644 --- a/roles/matrix-bot-go-neb/tasks/setup_uninstall.yml +++ b/roles/matrix-bot-go-neb/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure go-neb Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_go_neb_docker_image }}" state: absent diff --git a/roles/matrix-bot-honoroit/tasks/setup_install.yml b/roles/matrix-bot-honoroit/tasks/setup_install.yml index 8a440484..3c974557 100644 --- a/roles/matrix-bot-honoroit/tasks/setup_install.yml +++ b/roles/matrix-bot-honoroit/tasks/setup_install.yml @@ -50,7 +50,7 @@ mode: 0640 - name: Ensure honoroit image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_honoroit_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_bot_honoroit_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -73,7 +73,7 @@ when: "matrix_bot_honoroit_container_image_self_build | bool" - name: Ensure honoroit image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_honoroit_docker_image }}" source: build force_source: "{{ matrix_bot_honoroit_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bot-honoroit/tasks/setup_uninstall.yml b/roles/matrix-bot-honoroit/tasks/setup_uninstall.yml index 6ede0d1d..54869e31 100644 --- a/roles/matrix-bot-honoroit/tasks/setup_uninstall.yml +++ b/roles/matrix-bot-honoroit/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure honoroit Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_honoroit_docker_image }}" state: absent diff --git a/roles/matrix-bot-matrix-registration-bot/tasks/setup_install.yml b/roles/matrix-bot-matrix-registration-bot/tasks/setup_install.yml index d4522321..5896ac62 100644 --- a/roles/matrix-bot-matrix-registration-bot/tasks/setup_install.yml +++ b/roles/matrix-bot-matrix-registration-bot/tasks/setup_install.yml @@ -22,7 +22,7 @@ mode: 0640 - name: Ensure matrix-registration-bot image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_matrix_registration_bot_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_bot_matrix_registration_bot_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -45,7 +45,7 @@ when: "matrix_bot_matrix_registration_bot_container_image_self_build | bool" - name: Ensure matrix-registration-bot image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_matrix_registration_bot_docker_image }}" source: build force_source: "{{ matrix_bot_matrix_registration_bot_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bot-matrix-registration-bot/tasks/setup_uninstall.yml b/roles/matrix-bot-matrix-registration-bot/tasks/setup_uninstall.yml index 426eefc1..63bc53ad 100644 --- a/roles/matrix-bot-matrix-registration-bot/tasks/setup_uninstall.yml +++ b/roles/matrix-bot-matrix-registration-bot/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure matrix-registration-bot Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_matrix_registration_bot_docker_image }}" state: absent diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml index 93285bf6..5d784ef3 100644 --- a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml +++ b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml @@ -43,7 +43,7 @@ when: "item.when | bool" - name: Ensure matrix-reminder-bot image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_bot_matrix_reminder_bot_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -66,7 +66,7 @@ when: "matrix_bot_matrix_reminder_bot_container_image_self_build | bool" - name: Ensure matrix-reminder-bot image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}" source: build force_source: "{{ matrix_bot_matrix_reminder_bot_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml index d8926df7..de9e0427 100644 --- a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml +++ b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure matrix-reminder-bot Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}" state: absent diff --git a/roles/matrix-bot-maubot/tasks/setup_install.yml b/roles/matrix-bot-maubot/tasks/setup_install.yml index 185a2988..50e48254 100644 --- a/roles/matrix-bot-maubot/tasks/setup_install.yml +++ b/roles/matrix-bot-maubot/tasks/setup_install.yml @@ -26,7 +26,7 @@ mode: "u=rwx" - name: Ensure maubot image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_maubot_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_bot_maubot_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -49,7 +49,7 @@ when: "matrix_bot_maubot_container_image_self_build|bool" - name: Ensure maubot image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_maubot_docker_image }}" source: build force_source: "{{ matrix_bot_maubot_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bot-maubot/tasks/setup_uninstall.yml b/roles/matrix-bot-maubot/tasks/setup_uninstall.yml index dd0fc1f6..6a5e7fdc 100644 --- a/roles/matrix-bot-maubot/tasks/setup_uninstall.yml +++ b/roles/matrix-bot-maubot/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure maubot Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_maubot_docker_image }}" state: absent diff --git a/roles/matrix-bot-mjolnir/tasks/setup_install.yml b/roles/matrix-bot-mjolnir/tasks/setup_install.yml index 08ac9d03..b9bcf37d 100644 --- a/roles/matrix-bot-mjolnir/tasks/setup_install.yml +++ b/roles/matrix-bot-mjolnir/tasks/setup_install.yml @@ -18,7 +18,7 @@ when: "item.when | bool" - name: Ensure mjolnir Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_mjolnir_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_bot_mjolnir_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -41,7 +41,7 @@ when: "matrix_bot_mjolnir_container_image_self_build | bool" - name: Ensure mjolnir Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_mjolnir_docker_image }}" source: build force_source: "{{ matrix_bot_mjolnir_git_pull_results.changed }}" diff --git a/roles/matrix-bot-mjolnir/tasks/setup_uninstall.yml b/roles/matrix-bot-mjolnir/tasks/setup_uninstall.yml index 5c7f4c89..afefcc48 100644 --- a/roles/matrix-bot-mjolnir/tasks/setup_uninstall.yml +++ b/roles/matrix-bot-mjolnir/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure mjolnir Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_mjolnir_docker_image }}" state: absent diff --git a/roles/matrix-bot-postmoogle/tasks/setup_install.yml b/roles/matrix-bot-postmoogle/tasks/setup_install.yml index 9c9e59ee..5edfd4a9 100644 --- a/roles/matrix-bot-postmoogle/tasks/setup_install.yml +++ b/roles/matrix-bot-postmoogle/tasks/setup_install.yml @@ -46,7 +46,7 @@ mode: 0640 - name: Ensure postmoogle image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_postmoogle_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_bot_postmoogle_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -69,7 +69,7 @@ when: "matrix_bot_postmoogle_container_image_self_build | bool" - name: Ensure postmoogle image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_postmoogle_docker_image }}" source: build force_source: "{{ matrix_bot_postmoogle_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bot-postmoogle/tasks/setup_uninstall.yml b/roles/matrix-bot-postmoogle/tasks/setup_uninstall.yml index 64164a86..5502298c 100644 --- a/roles/matrix-bot-postmoogle/tasks/setup_uninstall.yml +++ b/roles/matrix-bot-postmoogle/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure postmoogle Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_postmoogle_docker_image }}" state: absent diff --git a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml index e12c1572..3ef48c3d 100644 --- a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml @@ -29,7 +29,7 @@ matrix_appservice_discord_requires_restart: true - name: Ensure Appservice Discord image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_appservice_discord_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_appservice_discord_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml b/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml index 6794e814..4b4614b1 100644 --- a/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml @@ -64,7 +64,7 @@ matrix_appservice_irc_requires_restart: true - name: Ensure Appservice IRC image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_appservice_irc_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_appservice_irc_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -87,7 +87,7 @@ when: "matrix_appservice_irc_enabled | bool and matrix_appservice_irc_container_image_self_build | bool" - name: Ensure matrix-appservice-irc Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_appservice_irc_docker_image }}" source: build force_source: "{{ matrix_appservice_irc_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-appservice-kakaotalk/tasks/setup_install.yml b/roles/matrix-bridge-appservice-kakaotalk/tasks/setup_install.yml index def73c59..2dd334cb 100644 --- a/roles/matrix-bridge-appservice-kakaotalk/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-kakaotalk/tasks/setup_install.yml @@ -9,7 +9,7 @@ when: "matrix_synapse_role_executed | default(False)" - name: Ensure matrix-appservice-kakaotalk image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_appservice_kakaotalk_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_appservice_kakaotalk_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -21,7 +21,7 @@ until: result is not failed - name: Ensure matrix-appservice-kakaotalk-node image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_appservice_kakaotalk_node_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_appservice_kakaotalk_node_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -58,7 +58,7 @@ when: "matrix_appservice_kakaotalk_container_image_self_build | bool" - name: Ensure matrix-appservice-kakaotalk-node Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_appservice_kakaotalk_node_docker_image }}" source: build force_source: "{{ matrix_appservice_kakaotalk_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -70,7 +70,7 @@ when: "matrix_appservice_kakaotalk_container_image_self_build | bool" - name: Ensure matrix-appservice-kakaotalk Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_appservice_kakaotalk_docker_image }}" source: build force_source: "{{ matrix_appservice_kakaotalk_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml b/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml index 9b741d69..d882d27d 100644 --- a/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml @@ -32,7 +32,7 @@ matrix_appservice_slack_requires_restart: true - name: Ensure Appservice Slack image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_appservice_slack_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_appservice_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -55,7 +55,7 @@ when: "matrix_appservice_slack_container_image_self_build | bool" - name: Ensure matrix-appservice-slack Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_appservice_slack_docker_image }}" source: build force_source: "{{ matrix_appservice_slack_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-appservice-webhooks/tasks/setup_install.yml b/roles/matrix-bridge-appservice-webhooks/tasks/setup_install.yml index 5cd8da88..824b5b78 100644 --- a/roles/matrix-bridge-appservice-webhooks/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-webhooks/tasks/setup_install.yml @@ -15,7 +15,7 @@ when: "item.when | bool" - name: Ensure Appservice webhooks image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_appservice_webhooks_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_appservice_webhooks_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -39,7 +39,7 @@ register: matrix_appservice_webhooks_git_pull_results - name: Ensure Appservice webhooks Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_appservice_webhooks_docker_image }}" source: build force_source: "{{ matrix_appservice_webhooks_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-beeper-linkedin/tasks/setup_install.yml b/roles/matrix-bridge-beeper-linkedin/tasks/setup_install.yml index c1b19df9..97464adc 100644 --- a/roles/matrix-bridge-beeper-linkedin/tasks/setup_install.yml +++ b/roles/matrix-bridge-beeper-linkedin/tasks/setup_install.yml @@ -23,7 +23,7 @@ - name: Ensure Beeper LinkedIn image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_beeper_linkedin_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_beeper_linkedin_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -62,7 +62,7 @@ changed_when: matrix_beeper_linkedin_generate_docker_requirements_result.rc == 0 - name: Ensure Beeper LinkedIn Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_beeper_linkedin_docker_image }}" source: build force_source: "{{ matrix_beeper_linkedin_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-go-skype-bridge/tasks/setup_install.yml b/roles/matrix-bridge-go-skype-bridge/tasks/setup_install.yml index 32019686..82ccc72c 100644 --- a/roles/matrix-bridge-go-skype-bridge/tasks/setup_install.yml +++ b/roles/matrix-bridge-go-skype-bridge/tasks/setup_install.yml @@ -52,7 +52,7 @@ when: item.when | bool - name: Ensure Go Skype Bridge image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_go_skype_bridge_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_go_skype_bridge_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -75,7 +75,7 @@ when: "matrix_go_skype_bridge_container_image_self_build | bool" - name: Ensure Go Skype Bridge Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_go_skype_bridge_docker_image }}" source: build force_source: "{{ matrix_go_skype_bridge_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-heisenbridge/tasks/setup_install.yml b/roles/matrix-bridge-heisenbridge/tasks/setup_install.yml index ffcc1c8b..f8e1259b 100644 --- a/roles/matrix-bridge-heisenbridge/tasks/setup_install.yml +++ b/roles/matrix-bridge-heisenbridge/tasks/setup_install.yml @@ -1,7 +1,7 @@ --- - name: Ensure heisenbridge image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_heisenbridge_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_heisenbridge_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-hookshot/tasks/setup_install.yml b/roles/matrix-bridge-hookshot/tasks/setup_install.yml index 0c6bfc34..7c1cdf95 100644 --- a/roles/matrix-bridge-hookshot/tasks/setup_install.yml +++ b/roles/matrix-bridge-hookshot/tasks/setup_install.yml @@ -15,7 +15,7 @@ when: item.when | bool - name: Ensure hookshot image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_hookshot_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_hookshot_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -38,7 +38,7 @@ when: "matrix_hookshot_container_image_self_build | bool" - name: Ensure hookshot Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_hookshot_docker_image }}" source: build force_source: "{{ matrix_hookshot_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mautrix-discord/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-discord/tasks/setup_install.yml index 4b05765a..ae3862d4 100644 --- a/roles/matrix-bridge-mautrix-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-discord/tasks/setup_install.yml @@ -52,7 +52,7 @@ when: item.when | bool - name: Ensure Mautrix Discord image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_discord_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mautrix_discord_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -75,7 +75,7 @@ when: "matrix_mautrix_discord_container_image_self_build | bool" - name: Ensure Mautrix discord Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_discord_docker_image }}" source: build force_source: "{{ matrix_mautrix_discord_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml index ca882fb0..651e7794 100644 --- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml @@ -37,7 +37,7 @@ matrix_mautrix_facebook_requires_restart: true - name: Ensure Mautrix Facebook image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_facebook_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mautrix_facebook_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -74,7 +74,7 @@ when: "matrix_mautrix_facebook_container_image_self_build | bool" - name: Ensure Mautrix Facebook Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_facebook_docker_image }}" source: build force_source: "{{ matrix_mautrix_facebook_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mautrix-googlechat/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-googlechat/tasks/setup_install.yml index f2fccb29..2e043def 100644 --- a/roles/matrix-bridge-mautrix-googlechat/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-googlechat/tasks/setup_install.yml @@ -37,7 +37,7 @@ matrix_mautrix_googlechat_requires_restart: true - name: Ensure Mautrix googlechat image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_googlechat_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mautrix_googlechat_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -74,7 +74,7 @@ when: "matrix_mautrix_googlechat_container_image_self_build | bool" - name: Ensure Mautrix googlechat Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_googlechat_docker_image }}" source: build force_source: "{{ matrix_mautrix_googlechat_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml index 48c83584..e62ef21c 100644 --- a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml @@ -37,7 +37,7 @@ matrix_mautrix_hangouts_requires_restart: true - name: Ensure Mautrix Hangouts image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_hangouts_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mautrix_hangouts_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -74,7 +74,7 @@ when: "matrix_mautrix_hangouts_container_image_self_build | bool" - name: Ensure Mautrix Hangouts Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_hangouts_docker_image }}" source: build force_source: "{{ matrix_mautrix_hangouts_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mautrix-instagram/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-instagram/tasks/setup_install.yml index 88b0286e..47076eb7 100644 --- a/roles/matrix-bridge-mautrix-instagram/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-instagram/tasks/setup_install.yml @@ -8,7 +8,7 @@ when: "matrix_synapse_role_executed | default(False)" - name: Ensure Mautrix instagram image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_instagram_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mautrix_instagram_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -45,7 +45,7 @@ when: "matrix_mautrix_instagram_container_image_self_build | bool" - name: Ensure Mautrix instagram Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_instagram_docker_image }}" source: build force_source: "{{ matrix_mautrix_instagram_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml index cfc704a8..577e80a9 100644 --- a/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml @@ -9,7 +9,7 @@ when: "matrix_synapse_role_executed | default(False)" - name: Ensure Mautrix Signal image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_signal_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mautrix_signal_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -33,7 +33,7 @@ when: "matrix_mautrix_signal_container_image_self_build | bool" - name: Ensure Mautrix Signal image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_signal_docker_image }}" source: build force_source: "{{ matrix_mautrix_signal_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -46,7 +46,7 @@ - name: Ensure Mautrix Signal Daemon image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_signal_daemon_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mautrix_signal_daemon_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -66,7 +66,7 @@ when: "matrix_mautrix_signal_daemon_container_image_self_build | bool" - name: Ensure Mautrix Signal Daemon image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_signal_daemon_docker_image }}" source: build force_source: "{{ matrix_mautrix_signal_daemon_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml index 7a50b709..3d6e66f6 100644 --- a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml @@ -51,7 +51,7 @@ when: item.when | bool - name: Ensure Mautrix Telegram image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_telegram_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mautrix_telegram_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -74,7 +74,7 @@ when: "matrix_telegram_lottieconverter_container_image_self_build | bool and matrix_mautrix_telegram_container_image_self_build | bool" - name: Ensure lottieconverter Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_telegram_lottieconverter_docker_image }}" source: build force_source: "{{ matrix_telegram_lottieconverter_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -97,7 +97,7 @@ when: "matrix_mautrix_telegram_container_image_self_build | bool" - name: Ensure matrix-mautrix-telegram Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_telegram_docker_image }}" source: build force_source: "{{ matrix_mautrix_telegram_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mautrix-twitter/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-twitter/tasks/setup_install.yml index 05887c6d..485e8be4 100644 --- a/roles/matrix-bridge-mautrix-twitter/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-twitter/tasks/setup_install.yml @@ -12,7 +12,7 @@ matrix_mautrix_twitter_requires_restart: false - name: Ensure Mautrix Twitter image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_twitter_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mautrix_twitter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -49,7 +49,7 @@ when: "matrix_mautrix_twitter_enabled | bool and matrix_mautrix_twitter_container_image_self_build" - name: Ensure Mautrix Twitter Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_twitter_docker_image }}" source: build force_source: "{{ matrix_mautrix_twitter_git_pull_results.changed }}" diff --git a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml index b58542f7..20dd2cc1 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml @@ -52,7 +52,7 @@ when: item.when | bool - name: Ensure Mautrix Whatsapp image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_whatsapp_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mautrix_whatsapp_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -75,7 +75,7 @@ when: "matrix_mautrix_whatsapp_container_image_self_build | bool" - name: Ensure Mautrix Whatsapp Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_whatsapp_docker_image }}" source: build force_source: "{{ matrix_mautrix_whatsapp_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml index b863b444..715c9e42 100644 --- a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml @@ -71,7 +71,7 @@ matrix_mx_puppet_discord_requires_restart: true - name: Ensure MX Puppet Discord image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_discord_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mx_puppet_discord_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -94,7 +94,7 @@ when: "matrix_mx_puppet_discord_enabled | bool and matrix_mx_puppet_discord_container_image_self_build" - name: Ensure MX Puppet Discord Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_discord_docker_image }}" source: build force_source: "{{ matrix_mx_puppet_discord_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mx-puppet-groupme/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-groupme/tasks/setup_install.yml index f81ae4a0..4e33961b 100644 --- a/roles/matrix-bridge-mx-puppet-groupme/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-groupme/tasks/setup_install.yml @@ -69,7 +69,7 @@ matrix_mx_puppet_groupme_requires_restart: true - name: Ensure MX Puppet Groupme image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_groupme_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mx_puppet_groupme_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -92,7 +92,7 @@ when: "matrix_mx_puppet_groupme_enabled | bool and matrix_mx_puppet_groupme_container_image_self_build" - name: Ensure MX Puppet Groupme Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_groupme_docker_image }}" source: build force_source: "{{ matrix_mx_puppet_groupme_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml index 600ed63c..7b1a26ee 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml @@ -38,7 +38,7 @@ matrix_mx_puppet_instagram_requires_restart: true - name: Ensure mx-puppet-instagram image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_instagram_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mx_puppet_instagram_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -75,7 +75,7 @@ when: "matrix_mx_puppet_instagram_enabled | bool and matrix_mx_puppet_instagram_container_image_self_build | bool" - name: Ensure mx-puppet-instagram Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_instagram_docker_image }}" source: build force_source: "{{ matrix_mx_puppet_instagram_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml index 9c5ae4fc..db584124 100644 --- a/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml @@ -65,7 +65,7 @@ matrix_mx_puppet_slack_requires_restart: true - name: Ensure MX Puppet Slack image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_slack_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mx_puppet_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -88,7 +88,7 @@ when: "matrix_mx_puppet_slack_enabled | bool and matrix_mx_puppet_slack_container_image_self_build" - name: Ensure MX Puppet Slack Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_slack_docker_image }}" source: build force_source: "{{ matrix_mx_puppet_slack_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml index c75566f3..9875dcce 100644 --- a/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml @@ -69,7 +69,7 @@ matrix_mx_puppet_steam_requires_restart: true - name: Ensure MX Puppet Steam image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_steam_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mx_puppet_steam_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -92,7 +92,7 @@ when: "matrix_mx_puppet_steam_enabled | bool and matrix_mx_puppet_steam_container_image_self_build" - name: Ensure MX Puppet Steam Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_steam_docker_image }}" source: build force_source: "{{ matrix_mx_puppet_steam_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml index ed94eae5..87da97b9 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml @@ -69,7 +69,7 @@ matrix_mx_puppet_twitter_requires_restart: true - name: Ensure MX Puppet Twitter image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_twitter_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mx_puppet_twitter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -92,7 +92,7 @@ when: "matrix_mx_puppet_twitter_enabled | bool and matrix_mx_puppet_twitter_container_image_self_build" - name: Ensure MX Puppet Twitter Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_twitter_docker_image }}" source: build force_source: "{{ matrix_mx_puppet_twitter_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-sms/tasks/setup_install.yml b/roles/matrix-bridge-sms/tasks/setup_install.yml index dcc317ba..b4125e51 100644 --- a/roles/matrix-bridge-sms/tasks/setup_install.yml +++ b/roles/matrix-bridge-sms/tasks/setup_install.yml @@ -1,7 +1,7 @@ --- - name: Ensure matrix-sms-bridge image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_sms_bridge_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" register: result diff --git a/roles/matrix-cactus-comments/tasks/setup_install.yml b/roles/matrix-cactus-comments/tasks/setup_install.yml index ec5311e8..7085290f 100644 --- a/roles/matrix-cactus-comments/tasks/setup_install.yml +++ b/roles/matrix-cactus-comments/tasks/setup_install.yml @@ -31,7 +31,7 @@ mode: 0640 - name: Ensure cactus comments image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_cactus_comments_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_cactus_comments_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -54,7 +54,7 @@ when: "matrix_cactus_comments_container_image_self_build | bool" - name: Ensure cactus comments image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_cactus_comments_docker_image }}" source: build force_source: "{{ matrix_cactus_comments_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-cactus-comments/tasks/setup_uninstall.yml b/roles/matrix-cactus-comments/tasks/setup_uninstall.yml index 011c04b8..3491d912 100644 --- a/roles/matrix-cactus-comments/tasks/setup_uninstall.yml +++ b/roles/matrix-cactus-comments/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure cactus comments Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_cactus_comments_docker_image }}" state: absent diff --git a/roles/matrix-client-cinny/tasks/setup_install.yml b/roles/matrix-client-cinny/tasks/setup_install.yml index 755b872f..a39c7f9d 100644 --- a/roles/matrix-client-cinny/tasks/setup_install.yml +++ b/roles/matrix-client-cinny/tasks/setup_install.yml @@ -12,7 +12,7 @@ when: "item.when | bool" - name: Ensure Cinny Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_client_cinny_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_client_cinny_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -54,7 +54,7 @@ when: "item.src is not none" - name: Ensure Cinny Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_client_cinny_docker_image }}" source: build force_source: "{{ matrix_client_cinny_git_pull_results.changed }}" diff --git a/roles/matrix-client-cinny/tasks/setup_uninstall.yml b/roles/matrix-client-cinny/tasks/setup_uninstall.yml index e6f71b0b..6cc93e1b 100644 --- a/roles/matrix-client-cinny/tasks/setup_uninstall.yml +++ b/roles/matrix-client-cinny/tasks/setup_uninstall.yml @@ -30,6 +30,6 @@ state: absent - name: Ensure Cinny Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_client_cinny_docker_image }}" state: absent diff --git a/roles/matrix-client-element/tasks/setup_install.yml b/roles/matrix-client-element/tasks/setup_install.yml index b21da064..044ed611 100644 --- a/roles/matrix-client-element/tasks/setup_install.yml +++ b/roles/matrix-client-element/tasks/setup_install.yml @@ -13,7 +13,7 @@ when: "item.when | bool" - name: Ensure Element Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_client_element_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_client_element_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -50,7 +50,7 @@ when: "matrix_client_element_container_image_self_build | bool and matrix_client_element_container_image_self_build_low_memory_system_patch_enabled | bool" - name: Ensure Element Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_client_element_docker_image }}" source: build force_source: "{{ matrix_client_element_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-client-element/tasks/setup_uninstall.yml b/roles/matrix-client-element/tasks/setup_uninstall.yml index b3cdd05e..c40a4fc6 100644 --- a/roles/matrix-client-element/tasks/setup_uninstall.yml +++ b/roles/matrix-client-element/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure Element Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_client_element_docker_image }}" state: absent diff --git a/roles/matrix-client-hydrogen/tasks/setup_install.yml b/roles/matrix-client-hydrogen/tasks/setup_install.yml index 4cd445d0..dfd0607b 100644 --- a/roles/matrix-client-hydrogen/tasks/setup_install.yml +++ b/roles/matrix-client-hydrogen/tasks/setup_install.yml @@ -13,7 +13,7 @@ when: "item.when | bool" - name: Ensure Hydrogen Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_client_hydrogen_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_client_hydrogen_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -58,7 +58,7 @@ # This step MUST come after the steps to install the configuration files because the config files # are currently only read at build time, not at run time like most other components in the playbook - name: Ensure Hydrogen Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_client_hydrogen_docker_image }}" source: build force_source: "{{ matrix_client_hydrogen_git_pull_results.changed }}" diff --git a/roles/matrix-client-hydrogen/tasks/setup_uninstall.yml b/roles/matrix-client-hydrogen/tasks/setup_uninstall.yml index d543cbb3..49d2f4ca 100644 --- a/roles/matrix-client-hydrogen/tasks/setup_uninstall.yml +++ b/roles/matrix-client-hydrogen/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure Hydrogen Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_client_hydrogen_docker_image }}" state: absent diff --git a/roles/matrix-conduit/tasks/conduit/setup_install.yml b/roles/matrix-conduit/tasks/conduit/setup_install.yml index ac5be14d..cf8c6657 100644 --- a/roles/matrix-conduit/tasks/conduit/setup_install.yml +++ b/roles/matrix-conduit/tasks/conduit/setup_install.yml @@ -1,6 +1,6 @@ --- - name: Ensure Conduit Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_conduit_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_conduit_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-conduit/tasks/conduit/setup_uninstall.yml b/roles/matrix-conduit/tasks/conduit/setup_uninstall.yml index 3bbbc3a7..1bba9a9e 100644 --- a/roles/matrix-conduit/tasks/conduit/setup_uninstall.yml +++ b/roles/matrix-conduit/tasks/conduit/setup_uninstall.yml @@ -25,6 +25,6 @@ when: "matrix_conduit_service_stat.stat.exists" - name: Ensure Conduit Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_conduit_docker_image }}" state: absent diff --git a/roles/matrix-corporal/tasks/setup_corporal.yml b/roles/matrix-corporal/tasks/setup_corporal.yml index 583c27eb..e4fce897 100644 --- a/roles/matrix-corporal/tasks/setup_corporal.yml +++ b/roles/matrix-corporal/tasks/setup_corporal.yml @@ -29,7 +29,7 @@ when: "matrix_corporal_enabled | bool and matrix_corporal_container_image_self_build | bool" - name: Ensure Matrix Corporal Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_corporal_docker_image }}" source: build force_source: "{{ matrix_corporal_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -41,7 +41,7 @@ when: "matrix_corporal_enabled | bool and matrix_corporal_container_image_self_build | bool" - name: Ensure Matrix Corporal Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_corporal_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_corporal_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -115,7 +115,7 @@ when: "not matrix_corporal_enabled | bool" - name: Ensure Matrix Corporal Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_corporal_docker_image }}" state: absent when: "not matrix_corporal_enabled | bool" diff --git a/roles/matrix-coturn/tasks/setup_install.yml b/roles/matrix-coturn/tasks/setup_install.yml index be9d8574..ef44c073 100644 --- a/roles/matrix-coturn/tasks/setup_install.yml +++ b/roles/matrix-coturn/tasks/setup_install.yml @@ -18,7 +18,7 @@ when: "item.when | bool" - name: Ensure Coturn image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_coturn_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_coturn_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -42,7 +42,7 @@ register: matrix_coturn_git_pull_results - name: Ensure Coturn Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_coturn_docker_image }}" source: build force_source: "{{ matrix_coturn_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -69,7 +69,7 @@ group: "{{ matrix_user_groupname }}" - name: Ensure Coturn network is created in Docker - docker_network: + community.docker.docker_network: name: "{{ matrix_coturn_docker_network }}" driver: bridge diff --git a/roles/matrix-dendrite/tasks/dendrite/setup_install.yml b/roles/matrix-dendrite/tasks/dendrite/setup_install.yml index 3052e101..98090e15 100644 --- a/roles/matrix-dendrite/tasks/dendrite/setup_install.yml +++ b/roles/matrix-dendrite/tasks/dendrite/setup_install.yml @@ -18,7 +18,7 @@ when: "not local_path_media_store_stat.failed and not local_path_media_store_stat.stat.exists" - name: Ensure Dendrite Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_dendrite_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_dendrite_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-dendrite/tasks/dendrite/setup_uninstall.yml b/roles/matrix-dendrite/tasks/dendrite/setup_uninstall.yml index b6d8cfac..6a2ea5b1 100644 --- a/roles/matrix-dendrite/tasks/dendrite/setup_uninstall.yml +++ b/roles/matrix-dendrite/tasks/dendrite/setup_uninstall.yml @@ -25,6 +25,6 @@ when: "matrix_dendrite_service_stat.stat.exists" - name: Ensure Dendrite Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_dendrite_docker_image }}" state: absent diff --git a/roles/matrix-dimension/tasks/setup_install.yml b/roles/matrix-dimension/tasks/setup_install.yml index c5570836..fc476397 100644 --- a/roles/matrix-dimension/tasks/setup_install.yml +++ b/roles/matrix-dimension/tasks/setup_install.yml @@ -87,7 +87,7 @@ group: "{{ matrix_dimension_user_gid }}" - name: Ensure Dimension image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_dimension_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_dimension_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -110,7 +110,7 @@ register: matrix_dimension_git_pull_results - name: Ensure Dimension Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_dimension_docker_image }}" source: build force_source: "{{ matrix_dimension_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-dimension/tasks/setup_uninstall.yml b/roles/matrix-dimension/tasks/setup_uninstall.yml index 3e2026a1..c939e66a 100644 --- a/roles/matrix-dimension/tasks/setup_uninstall.yml +++ b/roles/matrix-dimension/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure Dimension Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_dimension_docker_image }}" state: absent diff --git a/roles/matrix-dynamic-dns/tasks/install.yml b/roles/matrix-dynamic-dns/tasks/install.yml index e83637bf..4be6d9f0 100644 --- a/roles/matrix-dynamic-dns/tasks/install.yml +++ b/roles/matrix-dynamic-dns/tasks/install.yml @@ -1,7 +1,7 @@ --- - name: Ensure Dynamic DNS image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_dynamic_dns_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_dynamic_dns_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -37,7 +37,7 @@ when: "matrix_dynamic_dns_enabled | bool and matrix_dynamic_dns_container_image_self_build | bool" - name: Ensure Dynamic DNS Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_dynamic_dns_docker_image }}" source: build force_source: "{{ matrix_dynamic_dns_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-email2matrix/tasks/setup_install.yml b/roles/matrix-email2matrix/tasks/setup_install.yml index 2a782a1a..a6399a4e 100644 --- a/roles/matrix-email2matrix/tasks/setup_install.yml +++ b/roles/matrix-email2matrix/tasks/setup_install.yml @@ -22,7 +22,7 @@ mode: 0640 - name: Ensure Email2Matrix image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_email2matrix_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_email2matrix_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -45,7 +45,7 @@ when: "matrix_email2matrix_container_image_self_build | bool" - name: Ensure Email2Matrix Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_email2matrix_docker_image }}" source: build force_source: "{{ matrix_email2matrix_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-email2matrix/tasks/setup_uninstall.yml b/roles/matrix-email2matrix/tasks/setup_uninstall.yml index a713a65a..6aec40d2 100644 --- a/roles/matrix-email2matrix/tasks/setup_uninstall.yml +++ b/roles/matrix-email2matrix/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure Email2Matrix Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_email2matrix_docker_image }}" state: absent diff --git a/roles/matrix-etherpad/tasks/setup_install.yml b/roles/matrix-etherpad/tasks/setup_install.yml index 0243e9d7..4974bd47 100644 --- a/roles/matrix-etherpad/tasks/setup_install.yml +++ b/roles/matrix-etherpad/tasks/setup_install.yml @@ -17,7 +17,7 @@ group: "{{ matrix_etherpad_user_gid }}" - name: Ensure Etherpad image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_etherpad_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_etherpad_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-etherpad/tasks/setup_uninstall.yml b/roles/matrix-etherpad/tasks/setup_uninstall.yml index 38697366..1a5d003e 100644 --- a/roles/matrix-etherpad/tasks/setup_uninstall.yml +++ b/roles/matrix-etherpad/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure Etherpad Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_etherpad_docker_image }}" state: absent diff --git a/roles/matrix-grafana/tasks/setup.yml b/roles/matrix-grafana/tasks/setup.yml index 25378ce7..9198ffd8 100644 --- a/roles/matrix-grafana/tasks/setup.yml +++ b/roles/matrix-grafana/tasks/setup.yml @@ -5,7 +5,7 @@ # - name: Ensure matrix-grafana image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_grafana_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_grafana_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-jitsi/tasks/setup_jitsi_jicofo.yml b/roles/matrix-jitsi/tasks/setup_jitsi_jicofo.yml index 8b2ec6a7..5654fe3d 100644 --- a/roles/matrix-jitsi/tasks/setup_jitsi_jicofo.yml +++ b/roles/matrix-jitsi/tasks/setup_jitsi_jicofo.yml @@ -17,7 +17,7 @@ when: matrix_jitsi_enabled | bool and item.when - name: Ensure jitsi-jicofo Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_jitsi_jicofo_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_jitsi_jicofo_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-jitsi/tasks/setup_jitsi_jvb.yml b/roles/matrix-jitsi/tasks/setup_jitsi_jvb.yml index cdb94ebd..9a50f8c6 100644 --- a/roles/matrix-jitsi/tasks/setup_jitsi_jvb.yml +++ b/roles/matrix-jitsi/tasks/setup_jitsi_jvb.yml @@ -17,7 +17,7 @@ when: matrix_jitsi_enabled | bool and item.when - name: Ensure jitsi-jvb Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_jitsi_jvb_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_jitsi_jvb_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-jitsi/tasks/setup_jitsi_prosody.yml b/roles/matrix-jitsi/tasks/setup_jitsi_prosody.yml index 9383b48b..8ba99f91 100644 --- a/roles/matrix-jitsi/tasks/setup_jitsi_prosody.yml +++ b/roles/matrix-jitsi/tasks/setup_jitsi_prosody.yml @@ -18,7 +18,7 @@ when: matrix_jitsi_enabled | bool and item.when - name: Ensure jitsi-prosody Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_jitsi_prosody_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_jitsi_prosody_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-jitsi/tasks/setup_jitsi_web.yml b/roles/matrix-jitsi/tasks/setup_jitsi_web.yml index 1c7daa4b..9326ee61 100644 --- a/roles/matrix-jitsi/tasks/setup_jitsi_web.yml +++ b/roles/matrix-jitsi/tasks/setup_jitsi_web.yml @@ -19,7 +19,7 @@ when: matrix_jitsi_enabled | bool and item.when - name: Ensure jitsi-web Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_jitsi_web_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_jitsi_web_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-ldap-registration-proxy/tasks/setup_install.yml b/roles/matrix-ldap-registration-proxy/tasks/setup_install.yml index 87037337..97b7e8eb 100644 --- a/roles/matrix-ldap-registration-proxy/tasks/setup_install.yml +++ b/roles/matrix-ldap-registration-proxy/tasks/setup_install.yml @@ -26,7 +26,7 @@ register: matrix_ldap_registration_proxy_git_pull_results - name: Ensure matrix_ldap_registration_proxy Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_ldap_registration_proxy_docker_image }}" source: build force_source: "{{ matrix_ldap_registration_proxy_git_pull_results.changed }}" diff --git a/roles/matrix-ldap-registration-proxy/tasks/setup_uninstall.yml b/roles/matrix-ldap-registration-proxy/tasks/setup_uninstall.yml index 3225a3ae..96ab0b67 100644 --- a/roles/matrix-ldap-registration-proxy/tasks/setup_uninstall.yml +++ b/roles/matrix-ldap-registration-proxy/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure matrix_ldap_registration_proxy Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_ldap_registration_proxy_docker_image }}" state: absent diff --git a/roles/matrix-ma1sd/tasks/setup_install.yml b/roles/matrix-ma1sd/tasks/setup_install.yml index 5f4b2957..38ff3036 100644 --- a/roles/matrix-ma1sd/tasks/setup_install.yml +++ b/roles/matrix-ma1sd/tasks/setup_install.yml @@ -48,7 +48,7 @@ matrix_ma1sd_requires_restart: true - name: Ensure ma1sd image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_ma1sd_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_ma1sd_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -75,7 +75,7 @@ when: ansible_os_family == 'RedHat' - name: Ensure gradle is installed for self-building (Archlinux) - pacman: + community.general.pacman: name: - gradle state: present @@ -101,7 +101,7 @@ when: matrix_ma1sd_git_pull_results.changed - name: Ensure ma1sd Docker image is tagged correctly - docker_image: + community.docker.docker_image: # The build script always tags the image with 2 tags: # - based on the branch/version: e.g. `ma1uta/ma1sd:2.4.0` (when on `2.4.0`) # or `ma1uta/ma1sd:2.4.0-19-ga71d32b` (when on a given commit for a pre-release) diff --git a/roles/matrix-ma1sd/tasks/setup_uninstall.yml b/roles/matrix-ma1sd/tasks/setup_uninstall.yml index 2bc505b0..0349ec32 100644 --- a/roles/matrix-ma1sd/tasks/setup_uninstall.yml +++ b/roles/matrix-ma1sd/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure ma1sd Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_ma1sd_docker_image }}" state: absent diff --git a/roles/matrix-mailer/tasks/setup_mailer.yml b/roles/matrix-mailer/tasks/setup_mailer.yml index 2ab39df5..36ec8016 100644 --- a/roles/matrix-mailer/tasks/setup_mailer.yml +++ b/roles/matrix-mailer/tasks/setup_mailer.yml @@ -35,7 +35,7 @@ when: "matrix_mailer_enabled | bool and matrix_mailer_container_image_self_build | bool" - name: Ensure exim-relay Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mailer_docker_image }}" source: build force_source: "{{ matrix_mailer_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -47,7 +47,7 @@ when: "matrix_mailer_enabled | bool and matrix_mailer_container_image_self_build | bool" - name: Ensure exim-relay image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mailer_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mailer_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -108,7 +108,7 @@ when: "not matrix_mailer_enabled | bool" - name: Ensure mailer Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_mailer_docker_image }}" state: absent when: "not matrix_mailer_enabled | bool" diff --git a/roles/matrix-nginx-proxy/tasks/nginx-proxy/setup_metrics_auth.yml b/roles/matrix-nginx-proxy/tasks/nginx-proxy/setup_metrics_auth.yml index c2215eba..6129a49f 100644 --- a/roles/matrix-nginx-proxy/tasks/nginx-proxy/setup_metrics_auth.yml +++ b/roles/matrix-nginx-proxy/tasks/nginx-proxy/setup_metrics_auth.yml @@ -18,7 +18,7 @@ - when: matrix_nginx_proxy_proxy_matrix_metrics_basic_auth_username != '' block: - name: Ensure Apache Docker image is pulled for generating matrix-metrics-htpasswd from username/password (protecting /metrics/* URIs) - docker_image: + community.docker.docker_image: name: "{{ matrix_nginx_proxy_proxy_matrix_metrics_basic_auth_apache_container_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_nginx_proxy_proxy_matrix_metrics_basic_auth_apache_container_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml b/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml index 4d93e769..11a1cc06 100644 --- a/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml +++ b/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml @@ -193,7 +193,7 @@ # Tasks related to setting up matrix-nginx-proxy # - name: Ensure nginx Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_nginx_proxy_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_nginx_proxy_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml b/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml index 62430f4b..b212752c 100644 --- a/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml +++ b/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml @@ -22,7 +22,7 @@ ansible.builtin.import_tasks: "{{ role_path }}/tasks/ssl/purge_ssl_lets_encrypt_orphaned_configs.yml" - name: Ensure certbot Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_ssl_lets_encrypt_certbot_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_ssl_lets_encrypt_certbot_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-ntfy/tasks/setup_install.yml b/roles/matrix-ntfy/tasks/setup_install.yml index 9afabc4c..ef50c42a 100644 --- a/roles/matrix-ntfy/tasks/setup_install.yml +++ b/roles/matrix-ntfy/tasks/setup_install.yml @@ -1,7 +1,7 @@ --- - name: Ensure matrix-ntfy image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_ntfy_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_ntfy_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-ntfy/tasks/setup_uninstall.yml b/roles/matrix-ntfy/tasks/setup_uninstall.yml index f6d9cc8a..d5da1d8e 100644 --- a/roles/matrix-ntfy/tasks/setup_uninstall.yml +++ b/roles/matrix-ntfy/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure ntfy Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_ntfy_docker_image }}" state: absent diff --git a/roles/matrix-postgres-backup/tasks/setup_postgres_backup.yml b/roles/matrix-postgres-backup/tasks/setup_postgres_backup.yml index a6246757..2518326a 100644 --- a/roles/matrix-postgres-backup/tasks/setup_postgres_backup.yml +++ b/roles/matrix-postgres-backup/tasks/setup_postgres_backup.yml @@ -17,7 +17,7 @@ when: matrix_postgres_backup_enabled | bool - name: Ensure postgres backup Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_postgres_backup_docker_image_to_use }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_postgres_backup_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-postgres/tasks/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/migrate_db_to_postgres.yml index dd75fb17..215d36c1 100644 --- a/roles/matrix-postgres/tasks/migrate_db_to_postgres.yml +++ b/roles/matrix-postgres/tasks/migrate_db_to_postgres.yml @@ -61,7 +61,7 @@ replace: 'FROM debian:bullseye-slim' - name: Ensure pgloader Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_postgres_pgloader_docker_image }}" source: build force_source: "{{ matrix_postgres_pgloader_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -72,7 +72,7 @@ pull: true - name: Ensure pgloader Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_postgres_pgloader_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_postgres_pgloader_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-postgres/tasks/setup_postgres.yml b/roles/matrix-postgres/tasks/setup_postgres.yml index 49eb3249..9efc73a6 100644 --- a/roles/matrix-postgres/tasks/setup_postgres.yml +++ b/roles/matrix-postgres/tasks/setup_postgres.yml @@ -37,7 +37,7 @@ # Even if we don't run the internal server, we still need this for running the CLI - name: Ensure postgres Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_postgres_docker_image_to_use }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_postgres_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-prometheus-node-exporter/tasks/setup.yml b/roles/matrix-prometheus-node-exporter/tasks/setup.yml index 0c6e77fc..370b23d1 100644 --- a/roles/matrix-prometheus-node-exporter/tasks/setup.yml +++ b/roles/matrix-prometheus-node-exporter/tasks/setup.yml @@ -5,7 +5,7 @@ # - name: Ensure matrix-prometheus-node-exporter image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_prometheus_node_exporter_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_prometheus_node_exporter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-prometheus-postgres-exporter/tasks/setup.yml b/roles/matrix-prometheus-postgres-exporter/tasks/setup.yml index 00a61df6..dda614da 100644 --- a/roles/matrix-prometheus-postgres-exporter/tasks/setup.yml +++ b/roles/matrix-prometheus-postgres-exporter/tasks/setup.yml @@ -5,7 +5,7 @@ # - name: Ensure matrix-prometheus-postgres-exporter image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_prometheus_postgres_exporter_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_prometheus_postgres_exporter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-prometheus/tasks/setup_install.yml b/roles/matrix-prometheus/tasks/setup_install.yml index c3aeaa7a..928b1ab5 100644 --- a/roles/matrix-prometheus/tasks/setup_install.yml +++ b/roles/matrix-prometheus/tasks/setup_install.yml @@ -1,7 +1,7 @@ --- - name: Ensure matrix-prometheus image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_prometheus_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_prometheus_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-redis/tasks/setup_redis.yml b/roles/matrix-redis/tasks/setup_redis.yml index 7dd7ea9f..f3b047c0 100644 --- a/roles/matrix-redis/tasks/setup_redis.yml +++ b/roles/matrix-redis/tasks/setup_redis.yml @@ -5,7 +5,7 @@ # - name: Ensure redis Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_redis_docker_image_to_use }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_redis_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-registration/tasks/setup_install.yml b/roles/matrix-registration/tasks/setup_install.yml index 6b895d69..2f630ef0 100644 --- a/roles/matrix-registration/tasks/setup_install.yml +++ b/roles/matrix-registration/tasks/setup_install.yml @@ -48,7 +48,7 @@ when: "item.when | bool" - name: Ensure matrix-registration image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_registration_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_registration_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -79,7 +79,7 @@ when: "matrix_registration_container_image_self_build | bool and matrix_registration_container_image_self_build_python_dependencies_patch_enabled | bool" - name: Ensure matrix-registration Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_registration_docker_image }}" source: build force_source: "{{ matrix_registration_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-registration/tasks/setup_uninstall.yml b/roles/matrix-registration/tasks/setup_uninstall.yml index e3d713dc..70e4fe97 100644 --- a/roles/matrix-registration/tasks/setup_uninstall.yml +++ b/roles/matrix-registration/tasks/setup_uninstall.yml @@ -26,6 +26,6 @@ when: "matrix_registration_service_stat.stat.exists | bool" - name: Ensure matrix-registration Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_registration_docker_image }}" state: absent diff --git a/roles/matrix-sygnal/tasks/setup_install.yml b/roles/matrix-sygnal/tasks/setup_install.yml index 8f5f6937..e5c41bf0 100644 --- a/roles/matrix-sygnal/tasks/setup_install.yml +++ b/roles/matrix-sygnal/tasks/setup_install.yml @@ -1,7 +1,7 @@ --- - name: Ensure Sygnal image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_sygnal_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_sygnal_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-sygnal/tasks/setup_uninstall.yml b/roles/matrix-sygnal/tasks/setup_uninstall.yml index eff4a74b..dc752def 100644 --- a/roles/matrix-sygnal/tasks/setup_uninstall.yml +++ b/roles/matrix-sygnal/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure Sygnal Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_sygnal_docker_image }}" state: absent diff --git a/roles/matrix-synapse-admin/tasks/setup.yml b/roles/matrix-synapse-admin/tasks/setup.yml index 66021272..1dfa68a2 100644 --- a/roles/matrix-synapse-admin/tasks/setup.yml +++ b/roles/matrix-synapse-admin/tasks/setup.yml @@ -5,7 +5,7 @@ # - name: Ensure matrix-synapse-admin image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_synapse_admin_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_synapse_admin_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -28,7 +28,7 @@ when: "matrix_synapse_admin_enabled | bool and matrix_synapse_admin_container_image_self_build | bool" - name: Ensure matrix-synapse-admin Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_synapse_admin_docker_image }}" source: build force_source: "{{ matrix_synapse_admin_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -82,7 +82,7 @@ when: "not matrix_synapse_admin_enabled | bool and matrix_synapse_admin_service_stat.stat.exists" - name: Ensure matrix-synapse-admin Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_synapse_admin_docker_image }}" state: absent when: "not matrix_synapse_admin_enabled | bool" diff --git a/roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup_install.yml b/roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup_install.yml index 3869f1aa..117fb757 100644 --- a/roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup_install.yml +++ b/roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup_install.yml @@ -17,7 +17,7 @@ when: "ansible_os_family == 'Debian'" - name: Ensure git installed (Archlinux) - pacman: + community.general.pacman: name: - git state: present diff --git a/roles/matrix-synapse/tasks/ext/synapse-simple-antispam/setup_install.yml b/roles/matrix-synapse/tasks/ext/synapse-simple-antispam/setup_install.yml index 34a7a7e0..dd3ff2d3 100644 --- a/roles/matrix-synapse/tasks/ext/synapse-simple-antispam/setup_install.yml +++ b/roles/matrix-synapse/tasks/ext/synapse-simple-antispam/setup_install.yml @@ -22,7 +22,7 @@ when: "ansible_os_family == 'Debian'" - name: Ensure git installed (Archlinux) - pacman: + community.general.pacman: name: - git state: present diff --git a/roles/matrix-synapse/tasks/goofys/setup_install.yml b/roles/matrix-synapse/tasks/goofys/setup_install.yml index e3c34150..01eee8b8 100644 --- a/roles/matrix-synapse/tasks/goofys/setup_install.yml +++ b/roles/matrix-synapse/tasks/goofys/setup_install.yml @@ -3,7 +3,7 @@ - ansible.builtin.import_tasks: "{{ role_path }}/../matrix-base/tasks/util/ensure_fuse_installed.yml" - name: Ensure Goofys Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_s3_goofys_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_s3_goofys_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-synapse/tasks/goofys/setup_uninstall.yml b/roles/matrix-synapse/tasks/goofys/setup_uninstall.yml index da78003f..1e9f166d 100644 --- a/roles/matrix-synapse/tasks/goofys/setup_uninstall.yml +++ b/roles/matrix-synapse/tasks/goofys/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure Goofys Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_s3_goofys_docker_image }}" state: absent diff --git a/roles/matrix-synapse/tasks/import_media_store.yml b/roles/matrix-synapse/tasks/import_media_store.yml index 36ab9779..740eb510 100644 --- a/roles/matrix-synapse/tasks/import_media_store.yml +++ b/roles/matrix-synapse/tasks/import_media_store.yml @@ -51,7 +51,7 @@ # This can only work with local files, not if the media store is on Amazon S3, # as it won't be accessible in such a case. - name: Ensure provided media store directory is synchronized - synchronize: + ansible.builtin.synchronize: src: "{{ server_path_media_store }}/" dest: "{{ matrix_synapse_media_store_path }}" delete: true diff --git a/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml b/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml index 80c25f7f..c8138c0b 100644 --- a/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml +++ b/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml @@ -43,7 +43,7 @@ group: "{{ matrix_user_groupname }}" - name: Ensure rust-synapse-compress-state image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_synapse_rust_synapse_compress_state_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_synapse_rust_synapse_compress_state_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-synapse/tasks/synapse/setup_install.yml b/roles/matrix-synapse/tasks/synapse/setup_install.yml index 4d0e749f..86395db2 100644 --- a/roles/matrix-synapse/tasks/synapse/setup_install.yml +++ b/roles/matrix-synapse/tasks/synapse/setup_install.yml @@ -51,7 +51,7 @@ when: "matrix_synapse_git_pull_results.changed | bool or matrix_synapse_docker_image_check_result.stdout == ''" - name: Ensure Synapse Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_synapse_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_synapse_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -73,7 +73,7 @@ mode: 0640 - name: Ensure customized Docker image for Synapse is built - docker_image: + community.docker.docker_image: name: "{{ matrix_synapse_docker_image_customized }}" source: build build: diff --git a/roles/matrix-synapse/tasks/synapse/setup_uninstall.yml b/roles/matrix-synapse/tasks/synapse/setup_uninstall.yml index 06e55014..ac79f370 100644 --- a/roles/matrix-synapse/tasks/synapse/setup_uninstall.yml +++ b/roles/matrix-synapse/tasks/synapse/setup_uninstall.yml @@ -26,7 +26,7 @@ when: "matrix_synapse_service_stat.stat.exists" - name: Ensure Synapse Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ item }}" state: absent with_items: From 7fb45a507d2abf0d34670dd72a342f3fd9bcfa75 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 28 Oct 2022 17:39:53 +0300 Subject: [PATCH 083/118] Make --tags=run-postgres-vacuum and --tags=upgrade-postgres not assume Synapse Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/2211 --- group_vars/matrix_servers | 5 +++++ roles/matrix-postgres/defaults/main.yml | 4 ++++ roles/matrix-postgres/tasks/run_vacuum.yml | 11 ++++++----- roles/matrix-postgres/tasks/upgrade_postgres.yml | 11 +++++++---- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index d4464df9..18aa553c 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -2112,6 +2112,11 @@ matrix_postgres_additional_databases: | }} +matrix_postgres_systemd_services_to_stop_for_maintenance_list: | + {{ + ['matrix-' + matrix_homeserver_implementation + '.service'] + }} + ###################################################################### # # /matrix-postgres diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index 7b0660bb..fe469f16 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -15,6 +15,10 @@ matrix_postgres_db_name: "matrix" matrix_postgres_base_path: "{{ matrix_base_data_path }}/postgres" matrix_postgres_data_path: "{{ matrix_postgres_base_path }}/data" +# matrix_postgres_systemd_services_to_stop_for_maintenance_list specifies the list of systemd services to stop before vacuuming or upgrading. +# These services will be restarted after the operation completes. +matrix_postgres_systemd_services_to_stop_for_maintenance_list: [] + matrix_postgres_architecture: amd64 # matrix_postgres_docker_image_suffix controls whether we use Alpine-based images (`-alpine`) or the normal Debian-based images. diff --git a/roles/matrix-postgres/tasks/run_vacuum.yml b/roles/matrix-postgres/tasks/run_vacuum.yml index 43959982..97dec022 100644 --- a/roles/matrix-postgres/tasks/run_vacuum.yml +++ b/roles/matrix-postgres/tasks/run_vacuum.yml @@ -67,11 +67,12 @@ - ansible.builtin.set_fact: matrix_postgres_synapse_was_running: "{{ ansible_facts.services['matrix-synapse.service'] | default(none) is not none and ansible_facts.services['matrix-synapse.service'].state == 'running' }}" -- name: Ensure matrix-synapse is stopped +- name: Ensure services are stopped ansible.builtin.service: - name: matrix-synapse + name: "{{ item }}" state: stopped daemon_reload: true + with_items: "{{ matrix_postgres_systemd_services_to_stop_for_maintenance_list }}" - name: Run Postgres vacuum command ansible.builtin.command: "{{ matrix_postgres_vacuum_command }}" @@ -85,9 +86,9 @@ - ansible.builtin.debug: var: "matrix_postgres_synapse_vacuum_result" -- name: Ensure matrix-synapse is started, if it previously was +- name: Ensure services are started ansible.builtin.service: - name: matrix-synapse + name: "{{ item }}" state: started daemon_reload: true - when: "matrix_postgres_synapse_was_running | bool" + with_items: "{{ matrix_postgres_systemd_services_to_stop_for_maintenance_list }}" diff --git a/roles/matrix-postgres/tasks/upgrade_postgres.yml b/roles/matrix-postgres/tasks/upgrade_postgres.yml index 2f228a4c..3d22407c 100644 --- a/roles/matrix-postgres/tasks/upgrade_postgres.yml +++ b/roles/matrix-postgres/tasks/upgrade_postgres.yml @@ -55,10 +55,12 @@ - ansible.builtin.debug: msg: "Upgrading database from {{ matrix_postgres_detected_version_corresponding_docker_image }} to {{ matrix_postgres_docker_image_latest }}" -- name: Ensure matrix-synapse is stopped +- name: Ensure services are stopped ansible.builtin.service: - name: matrix-synapse + name: "{{ item }}" state: stopped + daemon_reload: true + with_items: "{{ matrix_postgres_systemd_services_to_stop_for_maintenance_list }}" - name: Ensure matrix-postgres is started ansible.builtin.service: @@ -175,11 +177,12 @@ path: "{{ postgres_dump_dir }}/{{ postgres_dump_name }}" state: absent -- name: Ensure matrix-synapse is started +- name: Ensure services are started ansible.builtin.service: - name: matrix-synapse + name: "{{ item }}" state: started daemon_reload: true + with_items: "{{ matrix_postgres_systemd_services_to_stop_for_maintenance_list }}" - ansible.builtin.debug: msg: "NOTE: Your old Postgres data directory is preserved at `{{ postgres_auto_upgrade_backup_data_path }}`. You might want to get rid of it once you've confirmed that all is well." From 30180048c3b6f797617ddf7f12c20b00db14e5b1 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 28 Oct 2022 19:43:44 +0300 Subject: [PATCH 084/118] Upgrade Synapse (v1.69.0 -> v1.70.1) --- roles/matrix-synapse/defaults/main.yml | 2 +- roles/matrix-synapse/vars/workers.yml | 38 +++++++++++++++++--------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 5e5867eb..2028d8b7 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -36,7 +36,7 @@ matrix_synapse_container_image_customizations_dockerfile_body_custom: '' matrix_synapse_docker_image: "{{ matrix_synapse_docker_image_name_prefix }}matrixdotorg/synapse:{{ matrix_synapse_docker_image_tag }}" matrix_synapse_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_container_image_self_build else matrix_container_global_registry_prefix }}" -matrix_synapse_version: v1.69.0 +matrix_synapse_version: v1.70.1 matrix_synapse_docker_image_tag: "{{ matrix_synapse_version }}" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" diff --git a/roles/matrix-synapse/vars/workers.yml b/roles/matrix-synapse/vars/workers.yml index bc49e835..2598caa6 100644 --- a/roles/matrix-synapse/vars/workers.yml +++ b/roles/matrix-synapse/vars/workers.yml @@ -43,6 +43,8 @@ matrix_synapse_workers_generic_worker_endpoints: - ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/members$ - ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/state$ - ^/_matrix/client/v1/rooms/.*/hierarchy$ + - ^/_matrix/client/(v1|unstable)/rooms/.*/relations/ + - ^/_matrix/client/v1/rooms/.*/threads$ - ^/_matrix/client/unstable/org.matrix.msc2716/rooms/.*/batch_send$ - ^/_matrix/client/unstable/im.nheko.summary/rooms/.*/summary$ - ^/_matrix/client/(r0|v3|unstable)/account/3pid$ @@ -136,8 +138,9 @@ matrix_synapse_workers_generic_worker_endpoints: # [#7530](https://github.com/matrix-org/synapse/issues/7530) and # [#9427](https://github.com/matrix-org/synapse/issues/9427). - # Note that a HTTP listener with `client` and `federation` resources must be - # configured in the `worker_listeners` option in the worker config. + # Note that a [HTTP listener](usage/configuration/config_documentation.md#listeners) + # with `client` and `federation` `resources` must be configured in the `worker_listeners` + # option in the worker config. # #### Load balancing @@ -177,7 +180,8 @@ matrix_synapse_workers_generic_worker_endpoints: # Additionally, the writing of specific streams (such as events) can be moved off # of the main process to a particular worker. - # To enable this, the worker must have a HTTP replication listener configured, + # To enable this, the worker must have a + # [HTTP `replication` listener](usage/configuration/config_documentation.md#listeners) configured, # have a `worker_name` and be listed in the `instance_map` config. The same worker # can handle multiple streams, but unless otherwise documented, each stream can only # have a single writer. @@ -266,7 +270,7 @@ matrix_synapse_workers_generic_worker_endpoints: # There is also support for moving background tasks to a separate # worker. Background tasks are run periodically or started via replication. Exactly # which tasks are configured to run depends on your Synapse configuration (e.g. if - # stats is enabled). + # stats is enabled). This worker doesn't handle any REST endpoints itself. # To enable this, the worker must have a `worker_name` and can be configured to run # background tasks. For example, to move background tasks to a dedicated worker, @@ -314,8 +318,8 @@ matrix_synapse_workers_generic_worker_endpoints: # #### Notifying Application Services # You can designate one generic worker to send output traffic to Application Services. - - # Specify its name in the shared configuration as follows: + # Doesn't handle any REST endpoints itself, but you should specify its name in the + # shared configuration as follows: # ```yaml # notify_appservices_from_worker: worker_name @@ -342,6 +346,12 @@ matrix_synapse_workers_generic_worker_endpoints: # - pusher_worker2 # ``` + # An example for a pusher instance: + + # ```yaml + # {{#include systemd-with-workers/workers/pusher_worker.yaml}} + # ``` + # ] # appservice worker (no API endpoints) [ @@ -371,6 +381,12 @@ matrix_synapse_workers_generic_worker_endpoints: # - federation_sender1 # - federation_sender2 # ``` + + # An example for a federation sender instance: + + # ```yaml + # {{#include systemd-with-workers/workers/federation_sender.yaml}} + # ``` # ] matrix_synapse_workers_media_repository_endpoints: @@ -392,16 +408,12 @@ matrix_synapse_workers_media_repository_endpoints: # media repository. Note that doing so will prevent the main process from being # able to handle the above endpoints. - # In the `media_repository` worker configuration file, configure the http listener to + # In the `media_repository` worker configuration file, configure the + # [HTTP listener](usage/configuration/config_documentation.md#listeners) to # expose the `media` resource. For example: # ```yaml - # worker_listeners: - # - type: http - # port: 8085 - # resources: - # - names: - # - media + # {{#include systemd-with-workers/workers/media_worker.yaml}} # ``` # Note that if running multiple media repositories they must be on the same server From e25464409dbc59e53d181ebcb41c37e3bf473e4e Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Fri, 28 Oct 2022 20:53:48 +0000 Subject: [PATCH 085/118] Update mautrix-signal 0.4.0 -> 0.4.1 ref: https://github.com/mautrix/signal/releases/tag/v0.4.1 --- roles/matrix-bridge-mautrix-signal/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-mautrix-signal/defaults/main.yml b/roles/matrix-bridge-mautrix-signal/defaults/main.yml index 83b0a5ce..24ba9b39 100644 --- a/roles/matrix-bridge-mautrix-signal/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-signal/defaults/main.yml @@ -9,7 +9,7 @@ matrix_mautrix_signal_docker_repo: "https://mau.dev/mautrix/signal.git" matrix_mautrix_signal_docker_repo_version: "{{ 'master' if matrix_mautrix_signal_version == 'latest' else matrix_mautrix_signal_version }}" matrix_mautrix_signal_docker_src_files_path: "{{ matrix_base_data_path }}/mautrix-signal/docker-src" -matrix_mautrix_signal_version: v0.4.0 +matrix_mautrix_signal_version: v0.4.1 matrix_mautrix_signal_daemon_version: 0.23.0 # See: https://mau.dev/mautrix/signal/container_registry matrix_mautrix_signal_docker_image: "dock.mau.dev/mautrix/signal:{{ matrix_mautrix_signal_version }}" From 80e57417f497c3c37d53daa04f8ef6b010b96e19 Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Sun, 19 Jun 2022 11:30:28 +0200 Subject: [PATCH 086/118] To recover from a failure or allow to quickly reset to known state we need to know with what playbook-commit the sever is installed. This commit saves the GIT hash/status when we run the playbook in a file called git_hash.yml. It also backs up that file by copying it to the target machine. --- roles/matrix-base/defaults/main.yml | 4 ++ roles/matrix-base/tasks/setup_matrix_base.yml | 53 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index eccda626..9213c661 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -298,3 +298,7 @@ run_setup: true run_self_check: true run_start: true run_stop: true + +# Saves the git hash in a file called git_hash.yml +# Set this to false if GIT is not installed on the local system (the system where the ansible command is run on) +git_save_hash: true \ No newline at end of file diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index 6eebe3c0..5d3c5820 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -19,6 +19,59 @@ mode: '0660' when: "matrix_vars_yml_snapshotting_enabled | bool" +- name: Save current git-repo status on the target to aid with restoring in case of problems + block: + - name: Get local git hash + delegate_to: 127.0.0.1 + become: false + register: git_describe + shell: + git describe + --always + --tags + --dirty + --long + --all + + - set_fact: + git_hash: "{{ git_describe.stdout }}" + + - name: Git hash + debug: + msg: "Git hash: {{ git_hash }}" + + - name: Save git hash in git_hash.yml + become: false + local_action: + copy + content="git_hash_last_run{{ ":" }} {{ git_hash }}\n" + dest="{{ matrix_vars_yml_snapshotting_src }}/git_hash.yml" + + - name: Copy git_hash.yml file to target + copy: + src: "{{ matrix_vars_yml_snapshotting_src }}/git_hash.yml" + dest: "{{ matrix_base_data_path }}/git_hash.yml" + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + mode: '0660' + + rescue: + - name: GIT not found error + ansible.builtin.debug: + msg: >- + Couldn't find GIT on the local machine. Continuing without saving the GIT hash. + You can disable saving the GIT hash by setting 'git_save_hash: false' in vars.yml + when: "git_describe.stderr.find('not found') != -1" + + - name: GIT hash error + ansible.builtin.fail: + msg: >- + Error when trying to get the GIT hash. + You can disable saving the GIT hash by setting 'git_save_hash: false' in vars.yml + when: "git_describe.stderr.find('not found') == -1" + + when: "matrix_vars_yml_snapshotting_enabled|bool and git_save_hash|bool" + - name: Ensure Matrix network is created in Docker community.docker.docker_network: name: "{{ matrix_docker_network }}" From ab398276af1d1b82c685a2cbd3f017f4cac2683a Mon Sep 17 00:00:00 2001 From: Stefan Warnat Date: Sun, 30 Oct 2022 00:55:36 +0200 Subject: [PATCH 087/118] Always use resolver variable in nginx conf.d files --- .../nginx/conf.d/matrix-base-domain.conf.j2 | 2 +- .../nginx/conf.d/matrix-bot-buscarron.conf.j2 | 4 ++-- .../nginx/conf.d/matrix-bot-go-neb.conf.j2 | 4 ++-- .../nginx/conf.d/matrix-client-cinny.conf.j2 | 4 ++-- .../nginx/conf.d/matrix-client-element.conf.j2 | 4 ++-- .../nginx/conf.d/matrix-client-hydrogen.conf.j2 | 4 ++-- .../templates/nginx/conf.d/matrix-conduit.conf.j2 | 4 ++-- .../templates/nginx/conf.d/matrix-dendrite.conf.j2 | 4 ++-- .../nginx/conf.d/matrix-dimension.conf.j2 | 4 ++-- .../templates/nginx/conf.d/matrix-domain.conf.j2 | 14 +++++++------- .../templates/nginx/conf.d/matrix-grafana.conf.j2 | 4 ++-- .../templates/nginx/conf.d/matrix-jitsi.conf.j2 | 8 ++++---- .../templates/nginx/conf.d/matrix-ntfy.conf.j2 | 4 ++-- .../templates/nginx/conf.d/matrix-riot-web.conf.j2 | 2 +- .../templates/nginx/conf.d/matrix-sygnal.conf.j2 | 4 ++-- .../templates/nginx/conf.d/matrix-synapse.conf.j2 | 4 ++-- 16 files changed, 37 insertions(+), 37 deletions(-) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-base-domain.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-base-domain.conf.j2 index 3aff997d..44978dc4 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-base-domain.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-base-domain.conf.j2 @@ -45,7 +45,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-bot-buscarron.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-bot-buscarron.conf.j2 index 0ce1473b..4f0fd4a8 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-bot-buscarron.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-bot-buscarron.conf.j2 @@ -24,7 +24,7 @@ location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-bot-buscarron:8080"; proxy_pass http://$backend; {% else %} @@ -51,7 +51,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-bot-go-neb.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-bot-go-neb.conf.j2 index e5589f55..a62ddfc8 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-bot-go-neb.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-bot-go-neb.conf.j2 @@ -18,7 +18,7 @@ location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-bot-go-neb:4050"; proxy_pass http://$backend; {% else %} @@ -44,7 +44,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-cinny.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-cinny.conf.j2 index df66349f..2ec6eb1b 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-cinny.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-cinny.conf.j2 @@ -24,7 +24,7 @@ location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-client-cinny:8080"; proxy_pass http://$backend; {% else %} @@ -51,7 +51,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-element.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-element.conf.j2 index dea91b21..0beeae52 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-element.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-element.conf.j2 @@ -26,7 +26,7 @@ location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-client-element:8080"; proxy_pass http://$backend; {% else %} @@ -53,7 +53,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-hydrogen.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-hydrogen.conf.j2 index e9428c55..7a2e9dfa 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-hydrogen.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-hydrogen.conf.j2 @@ -24,7 +24,7 @@ location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-client-hydrogen:8080"; proxy_pass http://$backend; {% else %} @@ -51,7 +51,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-conduit.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-conduit.conf.j2 index 2106acc4..6e7aca79 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-conduit.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-conduit.conf.j2 @@ -28,7 +28,7 @@ server { location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_conduit_client_api_addr_with_container }}"; proxy_pass http://$backend; {% else %} @@ -59,7 +59,7 @@ server { location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_conduit_federation_api_addr_with_container }}"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-dendrite.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-dendrite.conf.j2 index 939156a3..10eccf37 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-dendrite.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-dendrite.conf.j2 @@ -28,7 +28,7 @@ server { location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_dendrite_client_api_addr_with_container }}"; proxy_pass http://$backend; {% else %} @@ -59,7 +59,7 @@ server { location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_dendrite_federation_api_addr_with_container }}"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-dimension.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-dimension.conf.j2 index 07347be6..730fc4c1 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-dimension.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-dimension.conf.j2 @@ -21,7 +21,7 @@ location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-dimension:8184"; proxy_pass http://$backend; {% else %} @@ -47,7 +47,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-domain.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-domain.conf.j2 index 2895ba14..63d45bc6 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-domain.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-domain.conf.j2 @@ -62,7 +62,7 @@ location ^~ /_matrix/corporal { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_matrix_corporal_api_addr_with_container }}"; proxy_pass http://$backend; {% else %} @@ -80,7 +80,7 @@ location ^~ /_matrix/identity { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_matrix_identity_api_addr_with_container }}"; proxy_pass http://$backend; {% else %} @@ -98,7 +98,7 @@ location ^~ /_matrix/client/r0/user_directory/search { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_matrix_user_directory_search_addr_with_container }}"; proxy_pass http://$backend; {% else %} @@ -115,7 +115,7 @@ location ~ ^/_matrix/client/r0/register/(email|msisdn)/requestToken$ { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_matrix_3pid_registration_addr_with_container }}"; proxy_pass http://$backend; {% else %} @@ -140,7 +140,7 @@ location ~* ^({{ matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_prefix_regexes|join('|') }}) { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_matrix_client_api_addr_with_container }}"; proxy_pass http://$backend; {% else %} @@ -185,7 +185,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} @@ -288,7 +288,7 @@ server { location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_matrix_federation_api_addr_with_container }}"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-grafana.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-grafana.conf.j2 index def67f66..09418044 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-grafana.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-grafana.conf.j2 @@ -28,7 +28,7 @@ location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-grafana:3000"; proxy_pass http://$backend; {% else %} @@ -55,7 +55,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-jitsi.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-jitsi.conf.j2 index 54b8ea43..aa4b6b44 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-jitsi.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-jitsi.conf.j2 @@ -21,7 +21,7 @@ location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-jitsi-web:80"; proxy_pass http://$backend; {% else %} @@ -36,7 +36,7 @@ # colibri (JVB) websockets location ~ ^/colibri-ws/([a-zA-Z0-9-\.]+)/(.*) { {% if matrix_nginx_proxy_enabled %} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-jitsi-jvb:9090"; proxy_pass http://$backend; {% else %} @@ -57,7 +57,7 @@ # XMPP websocket location = /xmpp-websocket { {% if matrix_nginx_proxy_enabled %} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend {{ matrix_jitsi_xmpp_bosh_url_base }}; proxy_pass $backend/xmpp-websocket; {% else %} @@ -89,7 +89,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-ntfy.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-ntfy.conf.j2 index ae100eda..fbae47e1 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-ntfy.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-ntfy.conf.j2 @@ -20,7 +20,7 @@ location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-ntfy:8080"; proxy_pass http://$backend; {% else %} @@ -49,7 +49,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-riot-web.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-riot-web.conf.j2 index 5bcbeba5..99026913 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-riot-web.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-riot-web.conf.j2 @@ -36,7 +36,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-sygnal.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-sygnal.conf.j2 index 0f33c0a7..e3c6a461 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-sygnal.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-sygnal.conf.j2 @@ -19,7 +19,7 @@ location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-sygnal:6000"; proxy_pass http://$backend; {% else %} @@ -46,7 +46,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 1d6f2106..da189329 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -150,7 +150,7 @@ server { location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_synapse_client_api_addr_with_container }}"; proxy_pass http://$backend; {% else %} @@ -211,7 +211,7 @@ server { location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_synapse_federation_api_addr_with_container }}"; proxy_pass http://$backend; {% else %} From 678dfc6dc3afe990fc71decf65260115458371bc Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Sun, 30 Oct 2022 14:00:04 +0100 Subject: [PATCH 088/118] Rename var --- roles/matrix-base/defaults/main.yml | 2 +- roles/matrix-base/tasks/setup_matrix_base.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index 9213c661..22d89c23 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -301,4 +301,4 @@ run_stop: true # Saves the git hash in a file called git_hash.yml # Set this to false if GIT is not installed on the local system (the system where the ansible command is run on) -git_save_hash: true \ No newline at end of file +matrix_playbook_commit_hash_preservation_enabled: true \ No newline at end of file diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index 5d3c5820..d443fee8 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -60,17 +60,17 @@ ansible.builtin.debug: msg: >- Couldn't find GIT on the local machine. Continuing without saving the GIT hash. - You can disable saving the GIT hash by setting 'git_save_hash: false' in vars.yml + You can disable saving the GIT hash by setting 'matrix_playbook_commit_hash_preservation_enabled: false' in vars.yml when: "git_describe.stderr.find('not found') != -1" - name: GIT hash error ansible.builtin.fail: msg: >- Error when trying to get the GIT hash. - You can disable saving the GIT hash by setting 'git_save_hash: false' in vars.yml + You can disable saving the GIT hash by setting 'matrix_playbook_commit_hash_preservation_enabled: false' in vars.yml when: "git_describe.stderr.find('not found') == -1" - when: "matrix_vars_yml_snapshotting_enabled|bool and git_save_hash|bool" + when: "matrix_vars_yml_snapshotting_enabled|bool and matrix_playbook_commit_hash_preservation_enabled|bool" - name: Ensure Matrix network is created in Docker community.docker.docker_network: From 230eea678471e9b46b3a9069d5505c4499dc6f63 Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Sun, 30 Oct 2022 14:04:33 +0100 Subject: [PATCH 089/118] Save git_hash.yml only on the target --- roles/matrix-base/tasks/setup_matrix_base.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index d443fee8..a9773855 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -40,16 +40,9 @@ debug: msg: "Git hash: {{ git_hash }}" - - name: Save git hash in git_hash.yml - become: false - local_action: - copy - content="git_hash_last_run{{ ":" }} {{ git_hash }}\n" - dest="{{ matrix_vars_yml_snapshotting_src }}/git_hash.yml" - - - name: Copy git_hash.yml file to target + - name: Save git_hash.yml on target copy: - src: "{{ matrix_vars_yml_snapshotting_src }}/git_hash.yml" + content: "{{ git_hash }}" dest: "{{ matrix_base_data_path }}/git_hash.yml" owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" From 930867d50d7bf399c843c13211ed78473a430943 Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Sun, 30 Oct 2022 14:07:39 +0100 Subject: [PATCH 090/118] Remove matrix_vars_yml_snapshotting_enabled condition --- roles/matrix-base/tasks/setup_matrix_base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index a9773855..6fe7e47d 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -63,7 +63,7 @@ You can disable saving the GIT hash by setting 'matrix_playbook_commit_hash_preservation_enabled: false' in vars.yml when: "git_describe.stderr.find('not found') == -1" - when: "matrix_vars_yml_snapshotting_enabled|bool and matrix_playbook_commit_hash_preservation_enabled|bool" + when: "matrix_playbook_commit_hash_preservation_enabled|bool" - name: Ensure Matrix network is created in Docker community.docker.docker_network: From 3d902b7fe733957e966593775979ec22e349899e Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Sun, 30 Oct 2022 14:33:09 +0100 Subject: [PATCH 091/118] be more specific on GIT not found error --- roles/matrix-base/tasks/setup_matrix_base.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index 6fe7e47d..59dbe087 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -54,14 +54,14 @@ msg: >- Couldn't find GIT on the local machine. Continuing without saving the GIT hash. You can disable saving the GIT hash by setting 'matrix_playbook_commit_hash_preservation_enabled: false' in vars.yml - when: "git_describe.stderr.find('not found') != -1" + when: "git_describe.stderr.find('git: not found') != -1" - - name: GIT hash error + - name: Get GIT hash error ansible.builtin.fail: msg: >- Error when trying to get the GIT hash. You can disable saving the GIT hash by setting 'matrix_playbook_commit_hash_preservation_enabled: false' in vars.yml - when: "git_describe.stderr.find('not found') == -1" + when: "git_describe.stderr.find('git: not found') == -1" when: "matrix_playbook_commit_hash_preservation_enabled|bool" From fd25cf8f54e9341b005004f1dd991b9458c0be2e Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Sun, 30 Oct 2022 14:37:45 +0100 Subject: [PATCH 092/118] Update wording --- roles/matrix-base/defaults/main.yml | 11 ++++++----- roles/matrix-base/tasks/setup_matrix_base.yml | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index 22d89c23..4ad5fe55 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -284,6 +284,11 @@ matrix_docker_installation_enabled: true # Possible values are "docker-ce" (default) and "docker.io" (Debian). matrix_docker_package_name: docker-ce +# Controls whether the current playbook's commit hash is saved in `git_hash.yml` on the target +# Set this to false if GIT is not installed on the local system (the system where the ansible command is run on) +# to suppres the warning message. +matrix_playbook_commit_hash_preservation_enabled: true + # Variables to Control which parts of our roles run. run_postgres_import: true run_postgres_upgrade: true @@ -297,8 +302,4 @@ run_dendrite_register_user: true run_setup: true run_self_check: true run_start: true -run_stop: true - -# Saves the git hash in a file called git_hash.yml -# Set this to false if GIT is not installed on the local system (the system where the ansible command is run on) -matrix_playbook_commit_hash_preservation_enabled: true \ No newline at end of file +run_stop: true \ No newline at end of file diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index 59dbe087..44e4137c 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -59,7 +59,7 @@ - name: Get GIT hash error ansible.builtin.fail: msg: >- - Error when trying to get the GIT hash. + Error when trying to get the GIT hash. Please consult the error message above. You can disable saving the GIT hash by setting 'matrix_playbook_commit_hash_preservation_enabled: false' in vars.yml when: "git_describe.stderr.find('git: not found') == -1" From 15fbc525cf4f70e8e1f9a0814e4b9fb671d18dd3 Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Sun, 30 Oct 2022 14:53:27 +0100 Subject: [PATCH 093/118] Typo --- roles/matrix-base/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index 4ad5fe55..bcc821a4 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -286,7 +286,7 @@ matrix_docker_package_name: docker-ce # Controls whether the current playbook's commit hash is saved in `git_hash.yml` on the target # Set this to false if GIT is not installed on the local system (the system where the ansible command is run on) -# to suppres the warning message. +# to suppress the warning message. matrix_playbook_commit_hash_preservation_enabled: true # Variables to Control which parts of our roles run. From 54e71f2012afa055f6a4f119b00ffd76d756cc7a Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 31 Oct 2022 12:38:43 +0200 Subject: [PATCH 094/118] Mention "matrix_hookshot_enabled: true" on the Hookshot documentation page --- docs/configuring-playbook-bridge-hookshot.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/configuring-playbook-bridge-hookshot.md b/docs/configuring-playbook-bridge-hookshot.md index 51d7a335..5505d0ad 100644 --- a/docs/configuring-playbook-bridge-hookshot.md +++ b/docs/configuring-playbook-bridge-hookshot.md @@ -13,13 +13,16 @@ Note: the playbook also supports [matrix-appservice-webhooks](configuring-playbo Refer to the [official instructions](https://matrix-org.github.io/matrix-hookshot/latest/setup.html) to learn what the individual options do. -1. For each of the services (GitHub, GitLab, Jira, Figma, generic webhooks) fill in the respective variables `matrix_hookshot_service_*` listed in [main.yml](/roles/matrix-bridge-hookshot/defaults/main.yml) as required. -2. Take special note of the `matrix_hookshot_*_enabled` variables. Services that need no further configuration are enabled by default (GitLab, Generic), while you must first add the required configuration and enable the others (GitHub, Jira, Figma). -3. If you're setting up the GitHub bridge, you'll need to generate and download a private key file after you created your GitHub app. Copy the contents of that file to the variable `matrix_hookshot_github_private_key` so the playbook can install it for you, or use one of the [other methods](#manage-github-private-key-with-matrix-aux-role) explained below. -4. If you've already installed Matrix services using the playbook before, you'll need to re-run it (`--tags=setup-all,start`). If not, proceed with [configuring other playbook services](configuring-playbook.md) and then with [Installing](installing.md). Get back to this guide once ready. Hookshot can be set up individually using the tag `setup-hookshot`. +1. Enable the bridge by adding `matrix_hookshot_enabled: true` to your `vars.yml` file +2. For each of the services (GitHub, GitLab, Jira, Figma, generic webhooks) fill in the respective variables `matrix_hookshot_service_*` listed in [main.yml](/roles/matrix-bridge-hookshot/defaults/main.yml) as required. +3. Take special note of the `matrix_hookshot_*_enabled` variables. Services that need no further configuration are enabled by default (GitLab, Generic), while you must first add the required configuration and enable the others (GitHub, Jira, Figma). +4. If you're setting up the GitHub bridge, you'll need to generate and download a private key file after you created your GitHub app. Copy the contents of that file to the variable `matrix_hookshot_github_private_key` so the playbook can install it for you, or use one of the [other methods](#manage-github-private-key-with-matrix-aux-role) explained below. +5. If you've already installed Matrix services using the playbook before, you'll need to re-run it (`--tags=setup-all,start`). If not, proceed with [configuring other playbook services](configuring-playbook.md) and then with [Installing](installing.md). Get back to this guide once ready. Hookshot can be set up individually using the tag `setup-hookshot`. Other configuration options are available via the `matrix_hookshot_configuration_extension_yaml` and `matrix_hookshot_registration_extension_yaml` variables, see the comments in [main.yml](/roles/matrix-bridge-hookshot/defaults/main.yml) for how to use them. +Finally, run the playbook (see [installing](installing.md)). + ## Usage From cada3ef48b05b35da11b714d8505eeb785173a19 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 31 Oct 2022 12:47:10 +0200 Subject: [PATCH 095/118] Use pre-built image for Hookshot on arm64 --- group_vars/matrix_servers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 18aa553c..e5de094f 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -765,7 +765,7 @@ matrix_heisenbridge_systemd_wanted_services_list: | # We don't enable bridges by default. matrix_hookshot_enabled: false -matrix_hookshot_container_image_self_build: "{{ matrix_architecture not in ['amd64'] }}" +matrix_hookshot_container_image_self_build: "{{ matrix_architecture not in ['arm64', 'amd64'] }}" matrix_hookshot_appservice_token: "{{ '%s' | format(matrix_homeserver_generic_secret_key) | password_hash('sha512', 'hookshot.as.tok') | to_uuid }}" From 975dda54c9db16f7158f33d1263318626c9cf735 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 31 Oct 2022 21:34:14 +0200 Subject: [PATCH 096/118] Upgrade Dendrite (0.10.4 -> 0.10.5) --- roles/matrix-dendrite/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-dendrite/defaults/main.yml b/roles/matrix-dendrite/defaults/main.yml index dc0af2dc..945c1619 100644 --- a/roles/matrix-dendrite/defaults/main.yml +++ b/roles/matrix-dendrite/defaults/main.yml @@ -6,7 +6,7 @@ matrix_dendrite_enabled: true matrix_dendrite_docker_image: "{{ matrix_dendrite_docker_image_name_prefix }}matrixdotorg/dendrite-monolith:{{ matrix_dendrite_docker_image_tag }}" matrix_dendrite_docker_image_name_prefix: "docker.io/" -matrix_dendrite_docker_image_tag: "v0.10.4" +matrix_dendrite_docker_image_tag: "v0.10.5" matrix_dendrite_docker_image_force_pull: "{{ matrix_dendrite_docker_image.endswith(':latest') }}" matrix_dendrite_base_path: "{{ matrix_base_data_path }}/dendrite" From ec45b873762826044c558e91764b8abfe3651368 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 31 Oct 2022 21:50:39 +0200 Subject: [PATCH 097/118] Add matrix_hookshot_feeds_pollTimeoutSeconds --- roles/matrix-bridge-hookshot/defaults/main.yml | 1 + roles/matrix-bridge-hookshot/templates/config.yml.j2 | 1 + 2 files changed, 2 insertions(+) diff --git a/roles/matrix-bridge-hookshot/defaults/main.yml b/roles/matrix-bridge-hookshot/defaults/main.yml index 1c390f34..775ef087 100644 --- a/roles/matrix-bridge-hookshot/defaults/main.yml +++ b/roles/matrix-bridge-hookshot/defaults/main.yml @@ -131,6 +131,7 @@ matrix_hookshot_generic_user_id_prefix: '_webhooks_' matrix_hookshot_feeds_enabled: true # polling interval in seconds matrix_hookshot_feeds_interval: 600 +matrix_hookshot_feeds_pollTimeoutSeconds: 10 # There is no need to edit ports. use matrix_hookshot_container_http_host_bind_ports below to expose ports instead. diff --git a/roles/matrix-bridge-hookshot/templates/config.yml.j2 b/roles/matrix-bridge-hookshot/templates/config.yml.j2 index 5cd11a87..fb029efd 100644 --- a/roles/matrix-bridge-hookshot/templates/config.yml.j2 +++ b/roles/matrix-bridge-hookshot/templates/config.yml.j2 @@ -84,6 +84,7 @@ feeds: # enabled: {{ matrix_hookshot_feeds_enabled }} pollIntervalSeconds: {{ matrix_hookshot_feeds_interval }} + pollTimeoutSeconds: {{ matrix_hookshot_feeds_pollTimeoutSeconds | to_json }} {% endif %} {% if matrix_hookshot_provisioning_enabled %} provisioning: From e1274a6e56ea3e06284b2867a76d951276c160c8 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 31 Oct 2022 21:52:01 +0200 Subject: [PATCH 098/118] Rename variable (matrix_hookshot_feeds_interval -> matrix_hookshot_feeds_pollIntervalSeconds) This is more consistent with how we name variables. It's also less confusing, especially given that we have `matrix_hookshot_feeds_pollTimeoutSeconds` as well. --- roles/matrix-bridge-hookshot/defaults/main.yml | 3 +-- roles/matrix-bridge-hookshot/tasks/validate_config.yml | 9 +++++++++ roles/matrix-bridge-hookshot/templates/config.yml.j2 | 4 ++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/roles/matrix-bridge-hookshot/defaults/main.yml b/roles/matrix-bridge-hookshot/defaults/main.yml index 775ef087..96781f9a 100644 --- a/roles/matrix-bridge-hookshot/defaults/main.yml +++ b/roles/matrix-bridge-hookshot/defaults/main.yml @@ -129,8 +129,7 @@ matrix_hookshot_generic_user_id_prefix: '_webhooks_' matrix_hookshot_feeds_enabled: true -# polling interval in seconds -matrix_hookshot_feeds_interval: 600 +matrix_hookshot_feeds_pollIntervalSeconds: 600 matrix_hookshot_feeds_pollTimeoutSeconds: 10 diff --git a/roles/matrix-bridge-hookshot/tasks/validate_config.yml b/roles/matrix-bridge-hookshot/tasks/validate_config.yml index 0fbcf53c..3392f1b6 100644 --- a/roles/matrix-bridge-hookshot/tasks/validate_config.yml +++ b/roles/matrix-bridge-hookshot/tasks/validate_config.yml @@ -58,6 +58,15 @@ with_items: - "matrix_hookshot_provisioning_secret" +- name: (Deprecation) Catch and report renamed Hookshot variables + ansible.builtin.fail: + msg: >- + Your configuration contains a variable, which now has a different name. + Please change your configuration to rename the variable (`{{ item.old }}` -> `{{ item.new }}`). + when: "item.old in vars" + with_items: + - {'old': 'matrix_hookshot_feeds_interval', 'new': 'matrix_hookshot_feeds_pollIntervalSeconds'} + - name: (Deprecation) Catch and report old metrics usage ansible.builtin.fail: msg: >- diff --git a/roles/matrix-bridge-hookshot/templates/config.yml.j2 b/roles/matrix-bridge-hookshot/templates/config.yml.j2 index fb029efd..527afafa 100644 --- a/roles/matrix-bridge-hookshot/templates/config.yml.j2 +++ b/roles/matrix-bridge-hookshot/templates/config.yml.j2 @@ -82,8 +82,8 @@ generic: feeds: # (Optional) Configure this to enable RSS/Atom feed support # - enabled: {{ matrix_hookshot_feeds_enabled }} - pollIntervalSeconds: {{ matrix_hookshot_feeds_interval }} + enabled: {{ matrix_hookshot_feeds_enabled | to_json }} + pollIntervalSeconds: {{ matrix_hookshot_feeds_pollIntervalSeconds | to_json }} pollTimeoutSeconds: {{ matrix_hookshot_feeds_pollTimeoutSeconds | to_json }} {% endif %} {% if matrix_hookshot_provisioning_enabled %} From 39e6484956671080c1d7e2fbd94c6d7427669b66 Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Mon, 31 Oct 2022 22:57:16 +0100 Subject: [PATCH 099/118] fix lints --- roles/matrix-base/tasks/setup_matrix_base.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index 44e4137c..5c45323e 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -21,11 +21,11 @@ - name: Save current git-repo status on the target to aid with restoring in case of problems block: - - name: Get local git hash + - name: Get local git hash delegate_to: 127.0.0.1 become: false register: git_describe - shell: + ansible.builtin.shell: git describe --always --tags @@ -33,15 +33,15 @@ --long --all - - set_fact: + - ansible.builtin.set_fact: git_hash: "{{ git_describe.stdout }}" - name: Git hash - debug: + ansible.builtin.debug: msg: "Git hash: {{ git_hash }}" - + - name: Save git_hash.yml on target - copy: + ansible.builtin.copy: content: "{{ git_hash }}" dest: "{{ matrix_base_data_path }}/git_hash.yml" owner: "{{ matrix_user_username }}" From 2b049da963ddab978e6be9fdc014d23f819ed272 Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Mon, 31 Oct 2022 23:00:04 +0100 Subject: [PATCH 100/118] move when clause per linting suggestion --- roles/matrix-base/tasks/setup_matrix_base.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index 5c45323e..cdde09d2 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -20,6 +20,7 @@ when: "matrix_vars_yml_snapshotting_enabled | bool" - name: Save current git-repo status on the target to aid with restoring in case of problems + when: "matrix_playbook_commit_hash_preservation_enabled|bool" block: - name: Get local git hash delegate_to: 127.0.0.1 @@ -63,8 +64,6 @@ You can disable saving the GIT hash by setting 'matrix_playbook_commit_hash_preservation_enabled: false' in vars.yml when: "git_describe.stderr.find('git: not found') == -1" - when: "matrix_playbook_commit_hash_preservation_enabled|bool" - - name: Ensure Matrix network is created in Docker community.docker.docker_network: name: "{{ matrix_docker_network }}" From 45c9c2b17a6e60f828b0d4a1723c4eb8a5aae1cb Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Mon, 31 Oct 2022 23:08:45 +0100 Subject: [PATCH 101/118] convert shell to command usage --- roles/matrix-base/tasks/setup_matrix_base.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index cdde09d2..0aafe183 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -26,13 +26,14 @@ delegate_to: 127.0.0.1 become: false register: git_describe - ansible.builtin.shell: - git describe - --always - --tags - --dirty - --long - --all + ansible.builtin.command: + cmd: >- + git describe + --always + --tags + --dirty + --long + --all - ansible.builtin.set_fact: git_hash: "{{ git_describe.stdout }}" From 93615b6ad988edb4a7972e32f0595d70fb2fd1e0 Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Mon, 31 Oct 2022 23:20:56 +0100 Subject: [PATCH 102/118] Revert "convert shell to command usage" to allow for a catching the specific "git: not found" error. This reverts commit 45c9c2b17a6e60f828b0d4a1723c4eb8a5aae1cb. --- roles/matrix-base/tasks/setup_matrix_base.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index 0aafe183..cdde09d2 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -26,14 +26,13 @@ delegate_to: 127.0.0.1 become: false register: git_describe - ansible.builtin.command: - cmd: >- - git describe - --always - --tags - --dirty - --long - --all + ansible.builtin.shell: + git describe + --always + --tags + --dirty + --long + --all - ansible.builtin.set_fact: git_hash: "{{ git_describe.stdout }}" From 8b425b995ec446109b6b13f711c85d11b57e8136 Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Mon, 31 Oct 2022 23:44:47 +0100 Subject: [PATCH 103/118] add newline char --- roles/matrix-base/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index bcc821a4..52049ed5 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -302,4 +302,4 @@ run_dendrite_register_user: true run_setup: true run_self_check: true run_start: true -run_stop: true \ No newline at end of file +run_stop: true From 3fc19295e2373a2149b0ebe67b48d63915c12faf Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Nov 2022 07:05:26 +0200 Subject: [PATCH 104/118] Fix ansible-lint error (ansible.builtin.synchronize -> ansible.posix.synchronize) --- roles/matrix-synapse/tasks/import_media_store.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse/tasks/import_media_store.yml b/roles/matrix-synapse/tasks/import_media_store.yml index 740eb510..bdd99ca4 100644 --- a/roles/matrix-synapse/tasks/import_media_store.yml +++ b/roles/matrix-synapse/tasks/import_media_store.yml @@ -51,7 +51,7 @@ # This can only work with local files, not if the media store is on Amazon S3, # as it won't be accessible in such a case. - name: Ensure provided media store directory is synchronized - ansible.builtin.synchronize: + ansible.posix.synchronize: src: "{{ server_path_media_store }}/" dest: "{{ matrix_synapse_media_store_path }}" delete: true From 3a49b0077c8b8704c6f7b1cf00b8c19d2e24270b Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Tue, 1 Nov 2022 09:23:55 +0100 Subject: [PATCH 105/118] fix ansible-lint errors --- roles/matrix-base/tasks/setup_matrix_base.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index cdde09d2..0bf197b6 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -22,10 +22,11 @@ - name: Save current git-repo status on the target to aid with restoring in case of problems when: "matrix_playbook_commit_hash_preservation_enabled|bool" block: - - name: Get local git hash + - name: Get local git hash # noqa command-instead-of-module delegate_to: 127.0.0.1 become: false register: git_describe + changed_when: false ansible.builtin.shell: git describe --always From f7b45fd4402a25db979da3086d053fd61816dcab Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Nov 2022 12:58:22 +0200 Subject: [PATCH 106/118] Upgrade ddclient (v3.10.0-ls102 -> v3.10.0-ls103) --- roles/matrix-dynamic-dns/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-dynamic-dns/defaults/main.yml b/roles/matrix-dynamic-dns/defaults/main.yml index af4d9592..056eecba 100644 --- a/roles/matrix-dynamic-dns/defaults/main.yml +++ b/roles/matrix-dynamic-dns/defaults/main.yml @@ -7,7 +7,7 @@ matrix_dynamic_dns_enabled: true # The dynamic dns daemon interval matrix_dynamic_dns_daemon_interval: '300' -matrix_dynamic_dns_version: v3.10.0-ls102 +matrix_dynamic_dns_version: v3.10.0-ls103 # The docker container to use when in mode matrix_dynamic_dns_docker_image: "{{ matrix_dynamic_dns_docker_image_name_prefix }}linuxserver/ddclient:{{ matrix_dynamic_dns_version }}" From 17caa13a71a36365343ba417c7d42d5d44d68dda Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Nov 2022 12:59:15 +0200 Subject: [PATCH 107/118] Upgrade Grafana (9.2.2 -> 9.2.3) --- roles/matrix-grafana/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-grafana/defaults/main.yml b/roles/matrix-grafana/defaults/main.yml index 5bccb60d..43b29c94 100644 --- a/roles/matrix-grafana/defaults/main.yml +++ b/roles/matrix-grafana/defaults/main.yml @@ -5,7 +5,7 @@ matrix_grafana_enabled: true -matrix_grafana_version: 9.2.2 +matrix_grafana_version: 9.2.3 matrix_grafana_docker_image: "{{ matrix_container_global_registry_prefix }}grafana/grafana:{{ matrix_grafana_version }}" matrix_grafana_docker_image_force_pull: "{{ matrix_grafana_docker_image.endswith(':latest') }}" From 9fb21b89a8a7be51d014ad40fb6bb4cccbbb8dec Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Nov 2022 14:57:29 +0200 Subject: [PATCH 108/118] Fix import-postgres/run-postgres-vacuum failures being suppressed We were only reporting failures for when the async task didn't finish. We also need to report a failure for when the task finished, but returned a non-zero exit code. --- roles/matrix-postgres/tasks/import_postgres.yml | 2 +- roles/matrix-postgres/tasks/run_vacuum.yml | 2 +- .../tasks/rust-synapse-compress-state/compress_room.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/matrix-postgres/tasks/import_postgres.yml b/roles/matrix-postgres/tasks/import_postgres.yml index 40877e0b..302a8f41 100644 --- a/roles/matrix-postgres/tasks/import_postgres.yml +++ b/roles/matrix-postgres/tasks/import_postgres.yml @@ -108,5 +108,5 @@ async: "{{ postgres_import_wait_time }}" poll: 10 register: matrix_postgres_import_postgres_command_result - failed_when: not matrix_postgres_import_postgres_command_result.finished + failed_when: not matrix_postgres_import_postgres_command_result.finished or matrix_postgres_import_postgres_command_result.rc != 0 changed_when: matrix_postgres_import_postgres_command_result.finished and matrix_postgres_import_postgres_command_result.rc == 0 diff --git a/roles/matrix-postgres/tasks/run_vacuum.yml b/roles/matrix-postgres/tasks/run_vacuum.yml index 97dec022..9801ab2c 100644 --- a/roles/matrix-postgres/tasks/run_vacuum.yml +++ b/roles/matrix-postgres/tasks/run_vacuum.yml @@ -79,7 +79,7 @@ async: "{{ postgres_vacuum_wait_time }}" poll: 10 register: matrix_postgres_synapse_vacuum_result - failed_when: not matrix_postgres_synapse_vacuum_result.finished + failed_when: not matrix_postgres_synapse_vacuum_result.finished or matrix_postgres_synapse_vacuum_result.rc != 0 changed_when: matrix_postgres_synapse_vacuum_result.finished and matrix_postgres_synapse_vacuum_result.rc == 0 # Intentionally show the results diff --git a/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml b/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml index 9834e256..88db265d 100644 --- a/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml +++ b/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml @@ -21,7 +21,7 @@ async: "{{ matrix_synapse_rust_synapse_compress_state_compress_room_time }}" poll: 10 register: matrix_synapse_rust_synapse_compress_state_compress_room_command_result - failed_when: not matrix_synapse_rust_synapse_compress_state_compress_room_command_result.finished + failed_when: not matrix_synapse_rust_synapse_compress_state_compress_room_command_result.finished or matrix_synapse_rust_synapse_compress_state_compress_room_command_result.rc != 0 changed_when: matrix_synapse_rust_synapse_compress_state_compress_room_command_result.finished and matrix_synapse_rust_synapse_compress_state_compress_room_command_result.rc == 0 - ansible.builtin.debug: From 74b186a4543cd7c59ffcdf6f13c20da4590ea2dc Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Nov 2022 15:05:41 +0200 Subject: [PATCH 109/118] Fix "too many spaces after colon" ansible-lint error --- roles/matrix-dynamic-dns/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-dynamic-dns/defaults/main.yml b/roles/matrix-dynamic-dns/defaults/main.yml index 056eecba..77e01d0e 100644 --- a/roles/matrix-dynamic-dns/defaults/main.yml +++ b/roles/matrix-dynamic-dns/defaults/main.yml @@ -7,7 +7,7 @@ matrix_dynamic_dns_enabled: true # The dynamic dns daemon interval matrix_dynamic_dns_daemon_interval: '300' -matrix_dynamic_dns_version: v3.10.0-ls103 +matrix_dynamic_dns_version: v3.10.0-ls103 # The docker container to use when in mode matrix_dynamic_dns_docker_image: "{{ matrix_dynamic_dns_docker_image_name_prefix }}linuxserver/ddclient:{{ matrix_dynamic_dns_version }}" From a4662660d2ca70cdadf496e1047fc50204937faf Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Nov 2022 15:06:10 +0200 Subject: [PATCH 110/118] Do not wait needlessly when vacuuming or importing Postgres --- roles/matrix-postgres/tasks/import_postgres.yml | 4 +++- roles/matrix-postgres/tasks/run_vacuum.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/roles/matrix-postgres/tasks/import_postgres.yml b/roles/matrix-postgres/tasks/import_postgres.yml index 302a8f41..107a1f88 100644 --- a/roles/matrix-postgres/tasks/import_postgres.yml +++ b/roles/matrix-postgres/tasks/import_postgres.yml @@ -49,8 +49,10 @@ name: matrix-postgres state: started daemon_reload: true + register: matrix_postgres_import_start_result -- name: Wait a bit, so that Postgres can start +- when: matrix_postgres_import_start_result.changed | bool + name: Wait a bit, so that Postgres can start ansible.builtin.wait_for: timeout: "{{ postgres_start_wait_time }}" delegate_to: 127.0.0.1 diff --git a/roles/matrix-postgres/tasks/run_vacuum.yml b/roles/matrix-postgres/tasks/run_vacuum.yml index 9801ab2c..1b769f01 100644 --- a/roles/matrix-postgres/tasks/run_vacuum.yml +++ b/roles/matrix-postgres/tasks/run_vacuum.yml @@ -28,8 +28,10 @@ name: matrix-postgres state: started daemon_reload: true + register: matrix_postgres_vacuum_start_result -- name: Wait a bit, so that Postgres can start +- when: matrix_postgres_vacuum_start_result.changed | bool + name: Wait a bit, so that Postgres can start ansible.builtin.wait_for: timeout: "{{ postgres_start_wait_time }}" delegate_to: 127.0.0.1 From 125ca5569d4cc81d8b0d9f07a9b299dd3852436c Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Nov 2022 15:47:00 +0200 Subject: [PATCH 111/118] Use unique filter for matrix_postgres_import_roles_to_ignore and matrix_postgres_import_databases_to_ignore Dendrite uses a lot of databases, but a single (`dendrite`) role, which leads to `matrix_postgres_import_roles_to_ignore` being something like `['dendrite', 'dendrite', 'dendrite', ...]` needlessly. This leads to weird regexes being generated for `matrix_postgres_import_roles_ignore_regex`. It's not that it hurts, but it just looks odd. --- roles/matrix-postgres/defaults/main.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index fe469f16..5a0cdb6c 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -74,9 +74,11 @@ matrix_postgres_additional_databases: [] # We either need to not create them or to ignore the `CREATE ROLE` statements in the dump. matrix_postgres_import_roles_to_ignore: | {{ - [matrix_postgres_connection_username] - + - matrix_postgres_additional_databases|map(attribute='username') | list + ( + [matrix_postgres_connection_username] + + + matrix_postgres_additional_databases|map(attribute='username') | list + ) | unique }} # When importing an existing Postgres database (when restoring a backup) or when doing a Postgres upgrade (which dumps & restores), we'd like to avoid: @@ -98,9 +100,11 @@ matrix_postgres_import_roles_ignore_regex: "^(CREATE|ALTER) ROLE \\\"?({{ matrix # We either need to not create them or to ignore the `CREATE DATABASE` statements in the dump. matrix_postgres_import_databases_to_ignore: | {{ - [matrix_postgres_db_name] - + - matrix_postgres_additional_databases|map(attribute='name') | list + ( + [matrix_postgres_db_name] + + + matrix_postgres_additional_databases|map(attribute='name') | list + ) | unique }} # We also allow for the database name to be quoted, which is rare, but might happen for database names which are special keywords (e.g. `default`). From 501420f3cc09d2fc071ff620fd949c10b3fa2109 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 1 Nov 2022 14:18:41 +0000 Subject: [PATCH 112/118] Update element 1.11.12 -> 1.11.13 --- roles/matrix-client-element/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index 2d5c99d2..b2ffd05d 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -10,7 +10,7 @@ matrix_client_element_container_image_self_build_repo: "https://github.com/vecto # - https://github.com/vector-im/element-web/issues/19544 matrix_client_element_container_image_self_build_low_memory_system_patch_enabled: "{{ ansible_memtotal_mb < 4096 }}" -matrix_client_element_version: v1.11.12 +matrix_client_element_version: v1.11.13 matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:{{ matrix_client_element_version }}" matrix_client_element_docker_image_name_prefix: "{{ 'localhost/' if matrix_client_element_container_image_self_build else matrix_container_global_registry_prefix }}" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" From c3dc64b1d5e06fb356fce37dcab27664ef335ef7 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Nov 2022 16:22:58 +0200 Subject: [PATCH 113/118] Add matrix-user-creator role - automatic user account creation support We no longer ask users to create Matrix user accounts for these bots: - Postmoogle - Honoroit - Reminder Bot Other bots and services (matrix-registration-bot, maubot, mjolnir, Dimension, etc.) require an Access Token to run (not a password), so this new role doesn't help for them. It does help for the above bots though, and for defining your own "initial user accounts" in the `matrix_user_creator_users_additional` variable. --- docs/configuring-playbook-bot-honoroit.md | 30 +++---- ...ng-playbook-bot-matrix-registration-bot.md | 2 +- ...guring-playbook-bot-matrix-reminder-bot.md | 30 +++---- docs/configuring-playbook-bot-postmoogle.md | 30 +++---- docs/installing.md | 80 ++++++++++++++----- group_vars/matrix_servers | 34 ++++++++ roles/matrix-user-creator/defaults/main.yml | 22 +++++ roles/matrix-user-creator/tasks/main.yml | 9 +++ roles/matrix-user-creator/tasks/setup.yml | 33 ++++++++ .../util/ensure_user_registered_conduit.yml | 5 ++ .../util/ensure_user_registered_dendrite.yml | 17 ++++ .../util/ensure_user_registered_synapse.yml | 22 +++++ .../tasks/util/validate_user.yml | 16 ++++ roles/matrix-user-creator/vars/main.yml | 34 ++++++++ setup.yml | 1 + 15 files changed, 292 insertions(+), 73 deletions(-) create mode 100644 roles/matrix-user-creator/defaults/main.yml create mode 100644 roles/matrix-user-creator/tasks/main.yml create mode 100644 roles/matrix-user-creator/tasks/setup.yml create mode 100644 roles/matrix-user-creator/tasks/util/ensure_user_registered_conduit.yml create mode 100644 roles/matrix-user-creator/tasks/util/ensure_user_registered_dendrite.yml create mode 100644 roles/matrix-user-creator/tasks/util/ensure_user_registered_synapse.yml create mode 100644 roles/matrix-user-creator/tasks/util/validate_user.yml create mode 100644 roles/matrix-user-creator/vars/main.yml diff --git a/docs/configuring-playbook-bot-honoroit.md b/docs/configuring-playbook-bot-honoroit.md index 45fc033d..42f31d49 100644 --- a/docs/configuring-playbook-bot-honoroit.md +++ b/docs/configuring-playbook-bot-honoroit.md @@ -7,21 +7,6 @@ It's a bot you can use to setup **your own helpdesk on matrix** See the project's [documentation](https://gitlab.com/etke.cc/honoroit#how-it-looks-like) to learn what it does with screenshots and why it might be useful to you. -## Registering the bot user - -By default, the playbook will set up the bot with a username like this: `@honoroit:DOMAIN`. - -(to use a different username, adjust the `matrix_bot_honoroit_login` variable). - -You **need to register the bot user manually** before setting up the bot. You can use the playbook to [register a new user](registering-users.md): - -``` -ansible-playbook -i inventory/hosts setup.yml --extra-vars='username=honoroit password=PASSWORD_FOR_THE_BOT admin=no' --tags=register-user -``` - -Choose a strong password for the bot. You can generate a good password with a command like this: `pwgen -s 64 1`. - - ## Adjusting the playbook configuration Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars.yml` file: @@ -29,7 +14,10 @@ Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars. ```yaml matrix_bot_honoroit_enabled: true -# Adjust this to whatever password you chose when registering the bot user +# Uncomment and adjust this part if you'd like to use a username different than the default +# matrix_bot_honoroit_login: honoroit + +# Generate a strong password here. Consider generating it with `pwgen -s 64 1` matrix_bot_honoroit_password: PASSWORD_FOR_THE_BOT # Adjust this to your room ID @@ -41,9 +29,15 @@ matrix_bot_honoroit_roomid: "!yourRoomID:DOMAIN" After configuring the playbook, run the [installation](installing.md) command again: +```sh +ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,ensure-matrix-users-created,start ``` -ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start -``` + +**Notes**: + +- the `ensure-matrix-users-created` playbook tag makes the playbook automatically create the bot's user account + +- if you change the bot password (`matrix_bot_honoroit_password` in your `vars.yml` file) subsequently, the bot user's credentials on the homeserver won't be updated automatically. If you'd like to change the bot user's password, use a tool like [synapse-admin](configuring-playbook-synapse-admin.md) to change it, and then update `matrix_bot_honoroit_password` to let the bot know its new password ## Usage diff --git a/docs/configuring-playbook-bot-matrix-registration-bot.md b/docs/configuring-playbook-bot-matrix-registration-bot.md index a3e4bbeb..b1e3fdc6 100644 --- a/docs/configuring-playbook-bot-matrix-registration-bot.md +++ b/docs/configuring-playbook-bot-matrix-registration-bot.md @@ -16,7 +16,7 @@ By default, the playbook will set use the bot with a username like this: `@bot.m (to use a different username, adjust the `matrix_bot_matrix_registration_bot_matrix_user_id_localpart` variable). -You **need to register the bot user manually** before setting up the bot. You can use the playbook to [register a new user](registering-users.md): +For [other bots supported by the playbook](configuring-playbook.md#bots), Matrix bot user accounts are created and put to use automatically. For `matrix-registration-bot`, however, this is not the case - you **need to register the bot user manually** before setting up the bot. You can use the playbook to [register a new user](registering-users.md): ``` ansible-playbook -i inventory/hosts setup.yml --extra-vars='username=bot.matrix-registration-bot password=PASSWORD_FOR_THE_BOT admin=yes' --tags=register-user diff --git a/docs/configuring-playbook-bot-matrix-reminder-bot.md b/docs/configuring-playbook-bot-matrix-reminder-bot.md index aaf5670c..da73e5db 100644 --- a/docs/configuring-playbook-bot-matrix-reminder-bot.md +++ b/docs/configuring-playbook-bot-matrix-reminder-bot.md @@ -7,21 +7,6 @@ It's a bot you can use to **schedule one-off & recurring reminders and alarms**. See the project's [documentation](https://github.com/anoadragon453/matrix-reminder-bot#usage) to learn what it does and why it might be useful to you. -## Registering the bot user - -By default, the playbook will set up the bot with a username like this: `@bot.matrix-reminder-bot:DOMAIN`. - -(to use a different username, adjust the `matrix_bot_matrix_reminder_bot_matrix_user_id_localpart` variable). - -You **need to register the bot user manually** before setting up the bot. You can use the playbook to [register a new user](registering-users.md): - -``` -ansible-playbook -i inventory/hosts setup.yml --extra-vars='username=bot.matrix-reminder-bot password=PASSWORD_FOR_THE_BOT admin=no' --tags=register-user -``` - -Choose a strong password for the bot. You can generate a good password with a command like this: `pwgen -s 64 1`. - - ## Adjusting the playbook configuration Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars.yml` file: @@ -29,7 +14,10 @@ Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars. ```yaml matrix_bot_matrix_reminder_bot_enabled: true -# Adjust this to whatever password you chose when registering the bot user +# Uncomment and adjust this part if you'd like to use a username different than the default +# matrix_bot_matrix_reminder_bot_matrix_user_id_localpart: bot.matrix-reminder-bot + +# Generate a strong password here. Consider generating it with `pwgen -s 64 1` matrix_bot_matrix_reminder_bot_matrix_user_password: PASSWORD_FOR_THE_BOT # Adjust this to your timezone @@ -41,9 +29,15 @@ matrix_bot_matrix_reminder_bot_reminders_timezone: Europe/London After configuring the playbook, run the [installation](installing.md) command again: +```sh +ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,ensure-matrix-users-created,start ``` -ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start -``` + +**Notes**: + +- the `ensure-matrix-users-created` playbook tag makes the playbook automatically create the bot's user account + +- if you change the bot password (`matrix_bot_matrix_reminder_bot_matrix_user_password` in your `vars.yml` file) subsequently, the bot user's credentials on the homeserver won't be updated automatically. If you'd like to change the bot user's password, use a tool like [synapse-admin](configuring-playbook-synapse-admin.md) to change it, and then update `matrix_bot_matrix_reminder_bot_matrix_user_password` to let the bot know its new password ## Usage diff --git a/docs/configuring-playbook-bot-postmoogle.md b/docs/configuring-playbook-bot-postmoogle.md index 31566da9..0fa36693 100644 --- a/docs/configuring-playbook-bot-postmoogle.md +++ b/docs/configuring-playbook-bot-postmoogle.md @@ -9,21 +9,6 @@ It's a bot/bridge you can use to forward emails to Matrix rooms See the project's [documentation](https://gitlab.com/etke.cc/postmoogle) to learn what it does and why it might be useful to you. -## Registering the bot user - -By default, the playbook will set up the bot with a username like this: `@postmoogle:DOMAIN`. - -(to use a different username, adjust the `matrix_bot_postmoogle_login` variable). - -You **need to register the bot user manually** before setting up the bot. You can use the playbook to [register a new user](registering-users.md): - -``` -ansible-playbook -i inventory/hosts setup.yml --extra-vars='username=postmoogle password=PASSWORD_FOR_THE_BOT admin=no' --tags=register-user -``` - -Choose a strong password for the bot. You can generate a good password with a command like this: `pwgen -s 64 1`. - - ## Adjusting the playbook configuration Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars.yml` file: @@ -31,7 +16,10 @@ Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars. ```yaml matrix_bot_postmoogle_enabled: true -# Adjust this to whatever password you chose when registering the bot user +# Uncomment and adjust this part if you'd like to use a username different than the default +# matrix_bot_postmoogle_login: postmoogle + +# Generate a strong password here. Consider generating it with `pwgen -s 64 1` matrix_bot_postmoogle_password: PASSWORD_FOR_THE_BOT ``` @@ -43,9 +31,15 @@ See [Configuring DNS](configuring-dns.md). After configuring the playbook, run the [installation](installing.md) command again: +```sh +ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,ensure-matrix-users-created,start ``` -ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start -``` + +**Notes**: + +- the `ensure-matrix-users-created` playbook tag makes the playbook automatically create the bot's user account + +- if you change the bot password (`matrix_bot_postmoogle_password` in your `vars.yml` file) subsequently, the bot user's credentials on the homeserver won't be updated automatically. If you'd like to change the bot user's password, use a tool like [synapse-admin](configuring-playbook-synapse-admin.md) to change it, and then update `matrix_bot_postmoogle_password` to let the bot know its new password ## Usage diff --git a/docs/installing.md b/docs/installing.md index 8b1b51aa..308e6fe6 100644 --- a/docs/installing.md +++ b/docs/installing.md @@ -1,25 +1,64 @@ # Installing -## 1. Installing the Matrix services - If you've [configured your DNS](configuring-dns.md) and have [configured the playbook](configuring-playbook.md), you can start the installation procedure. -Run this command to install the Matrix services: +## Playbook tags introduction -```bash -ansible-playbook -i inventory/hosts setup.yml --tags=setup-all +The Ansible playbook's tasks are tagged, so that certain parts of the Ansible playbook can be run without running all other tasks. + +The general command syntax is: `ansible-playbook -i inventory/hosts setup.yml --tags=COMMA_SEPARATED_TAGS_GO_HERE` + +Here are some playbook tags that you should be familiar with: + +- `setup-all` - runs all setup tasks for all components, but does not start/restart services + +- `setup-SERVICE` (e.g. `setup-bot-postmoogle`) - runs the setup tasks only for a given role, but does not start/restart services. You can discover these additional tags in each role (`roles/*/main.yml`). Running per-component setup tasks is **not recommended**, as components sometimes depend on each other and running just the setup tasks for a given component may not be enough. For example, setting up the [mautrix-telegram bridge](configuring-playbook-bridge-mautrix-telegram.md), in addition to the `setup-mautrix-telegram` tag, requires database changes (the `setup-postgres` tag) as well as reverse-proxy changes (the `setup-nginx-proxy` tag). + +- `start` - starts all systemd services and makes them start automatically in the future + +- `stop` - stops all systemd services + +- `ensure-matrix-users-created` - a special tag which ensures that all special users needed by the playbook (for bots, etc.) are created + +`setup-*` tags **do not start services** automatically, because you may wish to do things before starting services, such as importing a database dump, restoring data from another server, etc. + + +## 1. Installing Matrix + +If you **don't** use SSH keys for authentication, but rather a regular password, you may need to add `--ask-pass` to the all Ansible commands + +If you **do** use SSH keys for authentication, **and** use a non-root user to *become* root (sudo), you may need to add `-K` (`--ask-become-pass`) to all Ansible commands + +There 2 ways to start the installation process - depending on whether you're [Installing a brand new server (without importing data)](#installing-a-brand-new-server-without-importing-data) or [Installing a server into which you'll import old data](#installing-a-server-into-which-youll-import-old-data). + + +### Installing a brand new server (without importing data) + +If this is **a brand new** Matrix server and you **won't be importing old data into it**, run all these tags: + +```sh +ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,ensure-matrix-users-created,start ``` -The above command **doesn't start any services just yet** (another step does this later - below). Feel free to **re-run this setup command any time** you think something is off with the server configuration. +This will do a full installation and start all Matrix services. + +Proceed to [Maintaining your setup in the future](#2-maintaining-your-setup-in-the-future) and [Finalize the installation](#3-finalize-the-installation) -**Notes**: -- if you **don't** use SSH keys for authentication, but rather a regular password, you may need to add `--ask-pass` to the above (and all other) Ansible commands. -- if you **do** use SSH keys for authentication, **and** use a non-root user to *become* root (sudo), you may need to add `-K` (`--ask-become-pass`) to the above (and all other) Ansible commands. +### Installing a server into which you'll import old data + +If you will be importing data into your newly created Matrix server, install it, but **do not** start its services just yet. +Starting its services or messing with its database now will affect your data import later on. + +To do the installation **without** starting services, run only the `setup-all` tag: + +```sh +ansible-playbook -i inventory/hosts setup.yml --tags=setup-all +``` -## 2. Things you might want to do after installing +When this command completes, services won't be running yet. -**Before starting the services**, you may want to do additional things like: +You can now: - [Importing an existing SQLite database (from another Synapse installation)](importing-synapse-sqlite.md) (optional) @@ -27,21 +66,26 @@ The above command **doesn't start any services just yet** (another step does thi - [Importing `media_store` data files from an existing Synapse installation](importing-synapse-media-store.md) (optional) +.. and then proceed to starting all services: -## 3. Starting the services - -When you're ready to start the Matrix services (and set them up to auto-start in the future), run this command: - -```bash +```sh ansible-playbook -i inventory/hosts setup.yml --tags=start ``` -## 4. Finalize the installation +Proceed to [Maintaining your setup in the future](#2-maintaining-your-setup-in-the-future) and [Finalize the installation](#3-finalize-the-installation) + + +## 2. Maintaining your setup in the future + +Feel free to **re-run the setup command any time** you think something is off with the server configuration. Ansible will take your configuration and update your server to match. + + +## 3. Finalize the installation Now that services are running, you need to **finalize the installation process** (required for federation to work!) by [Configuring Service Discovery via .well-known](configuring-well-known.md). -## 5. Things to do next +## 4. Things to do next After you have started the services and **finalized the installation process** (required for federation to work!) by [Configuring Service Discovery via .well-known](configuring-well-known.md), you can: diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index e5de094f..872e778b 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -2677,3 +2677,37 @@ matrix_conduit_systemd_required_services_list: | # /matrix-conduit # ###################################################################### + + +###################################################################### +# +# matrix-user-creator +# +###################################################################### + +matrix_user_creator_users_auto: | + {{ + [{ + 'username': matrix_bot_matrix_reminder_bot_matrix_user_id_localpart, + 'initial_password': matrix_bot_matrix_reminder_bot_matrix_user_password, + 'initial_type': 'bot', + }] if matrix_bot_matrix_reminder_bot_enabled else [] + + + [{ + 'username': matrix_bot_honoroit_login, + 'initial_password': matrix_bot_honoroit_password, + 'initial_type': 'bot', + }] if matrix_bot_honoroit_enabled else [] + + + [{ + 'username': matrix_bot_postmoogle_login, + 'initial_password': matrix_bot_postmoogle_password, + 'initial_type': 'bot', + }] if matrix_bot_postmoogle_enabled else [] + }} + +###################################################################### +# +# /matrix-user-creator +# +###################################################################### diff --git a/roles/matrix-user-creator/defaults/main.yml b/roles/matrix-user-creator/defaults/main.yml new file mode 100644 index 00000000..5c90a1e0 --- /dev/null +++ b/roles/matrix-user-creator/defaults/main.yml @@ -0,0 +1,22 @@ +--- + +# matrix-user-creator is a role that aims to automate initial Matrix user account creation. +# +# This role only supports initial user account creation and will not manage subsequent user-type changes +# or password changes. +# +# The playbook registers various bot user accounts automatically using this role by injecting +# user creation definitions into the `matrix_user_creator_users_auto` variable. +# +# To get started creating your own Matrix user accounts, use the `matrix_user_creator_users_additional` variable. + +# matrix_user_creator_users_auto holds a list of users that should be created on the Matrix homeserver. +# The playbook adds some user definitions here to have them end up in `matrix_user_creator_users` (see `vars/main.yml`) and get automatically created. +# This value is influenced by the playbook and will be overwritten elsewhere (`group_vars/`, etc.) +# To add your own user definitions, use the `matrix_user_creator_users_additional` variable. +matrix_user_creator_users_auto: [] + +# matrix_user_creator_users_additional holds a list of users that should be created on the Matrix homeserver. +# Add your own users here to have them end up in `matrix_user_creator_users` (see `vars/main.yml`) and get automatically created. +# For example syntax for this variable, see the documentation for `matrix_user_creator_users` in `vars/main.yml`. +matrix_user_creator_users_additional: [] diff --git a/roles/matrix-user-creator/tasks/main.yml b/roles/matrix-user-creator/tasks/main.yml new file mode 100644 index 00000000..ba20504f --- /dev/null +++ b/roles/matrix-user-creator/tasks/main.yml @@ -0,0 +1,9 @@ +--- + +- when: matrix_user_creator_users | length > 0 + ansible.builtin.import_tasks: "{{ role_path }}/tasks/setup.yml" + tags: + # This role intentionally doesn't do work on a `setup-all` tag. + # If it did, the initial installation (`--tags=setup-all`) would also potentially polute the database with data, + # which would make importing a database dump problematic. + - ensure-matrix-users-created diff --git a/roles/matrix-user-creator/tasks/setup.yml b/roles/matrix-user-creator/tasks/setup.yml new file mode 100644 index 00000000..73dfd454 --- /dev/null +++ b/roles/matrix-user-creator/tasks/setup.yml @@ -0,0 +1,33 @@ +--- + +- name: Validate Matrix users to create + ansible.builtin.include_tasks: "{{ role_path }}/tasks/util/validate_user.yml" + with_items: "{{ matrix_user_creator_users }}" + loop_control: + loop_var: user + # Suppress logging to avoid dumping the credentials to the shell + no_log: true + +- name: Ensure systemd is reloaded before starting the homeserver + ansible.builtin.service: + daemon_reload: true + +- name: Ensure homeserver is started before creating Matrix users + ansible.builtin.service: + name: "matrix-{{ matrix_homeserver_implementation }}.service" + state: started + daemon_reload: true + register: matrix_user_registrator_homeserver_start_result + +- name: Wait a while, so that the homeserver can manage to start before creating Matrix users + ansible.builtin.pause: + seconds: 7 + when: matrix_user_registrator_homeserver_start_result.changed | bool + +- name: Ensure Matrix users are created + ansible.builtin.include_tasks: "{{ role_path }}/tasks/util/ensure_user_registered_{{ matrix_homeserver_implementation }}.yml" + with_items: "{{ matrix_user_creator_users }}" + loop_control: + loop_var: user + # Suppress logging to avoid dumping the credentials to the shell + no_log: true diff --git a/roles/matrix-user-creator/tasks/util/ensure_user_registered_conduit.yml b/roles/matrix-user-creator/tasks/util/ensure_user_registered_conduit.yml new file mode 100644 index 00000000..8bbd147b --- /dev/null +++ b/roles/matrix-user-creator/tasks/util/ensure_user_registered_conduit.yml @@ -0,0 +1,5 @@ +--- + +- name: Ensure Conduit user registered - {{ user.username | quote }} + ansible.builtin.debug: + msg: "Not registering user. To register Conduit users, message the Conduit bot" diff --git a/roles/matrix-user-creator/tasks/util/ensure_user_registered_dendrite.yml b/roles/matrix-user-creator/tasks/util/ensure_user_registered_dendrite.yml new file mode 100644 index 00000000..5fb1636f --- /dev/null +++ b/roles/matrix-user-creator/tasks/util/ensure_user_registered_dendrite.yml @@ -0,0 +1,17 @@ +--- + +- name: Ensure Dendrite user registered - {{ user.username | quote }} + ansible.builtin.command: + cmd: |- + {{ matrix_host_command_docker }} exec matrix-dendrite + create-account + -config /data/dendrite.yaml + -username {{ user.username | quote }} + -password {{ user.initial_password | quote }} + {% if user.initial_type == 'admin' %} + -admin + {% endif %} + -url http://localhost:{{ matrix_dendrite_http_bind_port }} + register: matrix_dendrite_register_user_result + changed_when: matrix_dendrite_register_user_result.rc == 0 and 'Desired user ID is already taken' not in matrix_dendrite_register_user_result.stderr + failed_when: matrix_dendrite_register_user_result.rc != 0 and 'Desired user ID is already taken' not in matrix_dendrite_register_user_result.stderr diff --git a/roles/matrix-user-creator/tasks/util/ensure_user_registered_synapse.yml b/roles/matrix-user-creator/tasks/util/ensure_user_registered_synapse.yml new file mode 100644 index 00000000..fde82096 --- /dev/null +++ b/roles/matrix-user-creator/tasks/util/ensure_user_registered_synapse.yml @@ -0,0 +1,22 @@ +--- + +- name: Ensure Synapse user registered - {{ user.username | quote }} + ansible.builtin.command: + cmd: |- + {{ matrix_host_command_docker }} exec matrix-synapse + register_new_matrix_user + -u {{ user.username | quote }} + -p {{ user.initial_password | quote }} + -c /data/homeserver.yaml + {% if user.initial_type == 'admin' %} + --admin + {% else %} + --no-admin + {% if user.initial_type != 'user' %} + --user_type={{ user.initial_type | quote }} + {% endif %} + {% endif %} + http://localhost:{{ matrix_synapse_container_client_api_port }} + register: matrix_synapse_register_user_result + changed_when: matrix_synapse_register_user_result.rc == 0 and 'User ID already taken' not in matrix_synapse_register_user_result.stdout + failed_when: matrix_synapse_register_user_result.rc != 0 and 'User ID already taken' not in matrix_synapse_register_user_result.stdout diff --git a/roles/matrix-user-creator/tasks/util/validate_user.yml b/roles/matrix-user-creator/tasks/util/validate_user.yml new file mode 100644 index 00000000..e35475d4 --- /dev/null +++ b/roles/matrix-user-creator/tasks/util/validate_user.yml @@ -0,0 +1,16 @@ +--- + +- name: Fail if invalid username + ansible.builtin.fail: + msg: "Empty usernames values are not allowed ({{ user }})" + when: not (user.username | default('')) + +- name: Fail if invalid initial_password for user - {{ user.username }} + ansible.builtin.fail: + msg: "Empty initial_password values are not allowed" + when: not (user.initial_password | default('')) + +- name: Fail if invalid initial_type for user - {{ user.username }} + ansible.builtin.fail: + msg: "User initial_type `{{ user.initial_type | default('undefined') }}` is not supported" + when: user.initial_type | default('undefined') not in ['admin', 'user', 'bot', 'support'] diff --git a/roles/matrix-user-creator/vars/main.yml b/roles/matrix-user-creator/vars/main.yml new file mode 100644 index 00000000..7b65bb8a --- /dev/null +++ b/roles/matrix-user-creator/vars/main.yml @@ -0,0 +1,34 @@ +--- + +# matrix_user_creator_users holds a list of users that should be created on the Matrix homeserver. +# +# Removing a user from this list will not automatically delete/disable the Matrix user on the homeserver. +# +# As the `initial_password` / `initial_type` field names indicate, these are just initial values. +# Changing the password or type values subsequently will not update the already existing user's details. +# +# The known user types are: 'admin', 'user', 'bot', 'support'. +# These are inspired by Synapse's user types. +# 'admin' and 'user' types are generally recognized across homeservers. +# Other homeservers may not support 'bot' and 'support'. Such homeservers will fall back to whatever types they do support. +# +# Example: +# matrix_user_creator_users: +# - username: root +# initial_password: some-password +# initial_type: admin +# +# - username: john +# initial_password: some-password +# initial_type: user +# +# - username: bot.matrix-reminder-bot +# initial_password: some-password +# initial_type: bot +# +# - username: bot.matrix-reminder-bot +# initial_password: some-password +# initial_type: support +# +# To create you own users, use the `matrix_user_creator_users_additional` variable. +matrix_user_creator_users: "{{ matrix_user_creator_users_auto + matrix_user_creator_users_additional }}" diff --git a/setup.yml b/setup.yml index 723f87d9..b4179354 100755 --- a/setup.yml +++ b/setup.yml @@ -72,4 +72,5 @@ - matrix-aux - matrix-postgres-backup - matrix-backup-borg + - matrix-user-creator - matrix-common-after From eaa9b7cfc4a05bce65cbc0424508bec827c44fb1 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Nov 2022 17:03:53 +0200 Subject: [PATCH 114/118] Add automatic user account creation support to Buscarron Continuation of c3dc64b1d5e06f --- docs/configuring-playbook-bot-buscarron.md | 29 +++++++++------------- group_vars/matrix_servers | 6 +++++ 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/docs/configuring-playbook-bot-buscarron.md b/docs/configuring-playbook-bot-buscarron.md index b38f25c6..0b80ba40 100644 --- a/docs/configuring-playbook-bot-buscarron.md +++ b/docs/configuring-playbook-bot-buscarron.md @@ -5,20 +5,6 @@ The playbook can install and configure [buscarron](https://gitlab.com/etke.cc/bu It's a bot you can use to setup **your own helpdesk on matrix** It's a bot you can use to send any form (HTTP POST, HTML) to a (encrypted) matrix room -## Registering the bot user - -By default, the playbook will set up the bot with a username like this: `@bot.buscarron:DOMAIN`. - -(to use a different username, adjust the `matrix_bot_buscarron_login` variable). - -You **need to register the bot user manually** before setting up the bot. You can use the playbook to [register a new user](registering-users.md): - -``` -ansible-playbook -i inventory/hosts setup.yml --extra-vars='username=bot.buscarron password=PASSWORD_FOR_THE_BOT admin=no' --tags=register-user -``` - -Choose a strong password for the bot. You can generate a good password with a command like this: `pwgen -s 64 1`. - ## Adjusting the playbook configuration @@ -27,7 +13,10 @@ Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars. ```yaml matrix_bot_buscarron_enabled: true -# Adjust this to whatever password you chose when registering the bot user +# Uncomment and adjust this part if you'd like to use a username different than the default +# matrix_bot_buscarron_login: bot.buscarron + +# Generate a strong password here. Consider generating it with `pwgen -s 64 1` matrix_bot_buscarron_password: PASSWORD_FOR_THE_BOT # Adjust accepted forms @@ -57,9 +46,15 @@ matrix_server_fqn_buscarron: "form.{{ matrix_domain }}" After configuring the playbook, run the [installation](installing.md) command again: +```sh +ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,ensure-matrix-users-created,start ``` -ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start -``` + +**Notes**: + +- the `ensure-matrix-users-created` playbook tag makes the playbook automatically create the bot's user account + +- if you change the bot password (`matrix_bot_buscarron_password` in your `vars.yml` file) subsequently, the bot user's credentials on the homeserver won't be updated automatically. If you'd like to change the bot user's password, use a tool like [synapse-admin](configuring-playbook-synapse-admin.md) to change it, and then update `matrix_bot_buscarron_password` to let the bot know its new password ## Usage diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 872e778b..bdf5dcce 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -2704,6 +2704,12 @@ matrix_user_creator_users_auto: | 'initial_password': matrix_bot_postmoogle_password, 'initial_type': 'bot', }] if matrix_bot_postmoogle_enabled else [] + + + [{ + 'username': matrix_bot_buscarron_login, + 'initial_password': matrix_bot_buscarron_password, + 'initial_type': 'bot', + }] if matrix_bot_buscarron_enabled else [] }} ###################################################################### From b7bd68d3cb58e7e48e239ba66867a8b4c4f1631e Mon Sep 17 00:00:00 2001 From: Michael Hollister Date: Tue, 1 Nov 2022 13:11:11 -0500 Subject: [PATCH 115/118] Fixed filter for Synapse stream_writer HS config --- roles/matrix-synapse/tasks/synapse/workers/init.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse/tasks/synapse/workers/init.yml b/roles/matrix-synapse/tasks/synapse/workers/init.yml index 0fc4e79c..d88884d0 100644 --- a/roles/matrix-synapse/tasks/synapse/workers/init.yml +++ b/roles/matrix-synapse/tasks/synapse/workers/init.yml @@ -37,7 +37,7 @@ - name: Populate matrix_synapse_stream_writers from enabled stream writer workers list ansible.builtin.set_fact: - matrix_synapse_stream_writers: "{{ matrix_synapse_stream_writers | combine({item.ansible_facts.worker.stream_writer_stream: [item.ansible_facts.worker.name]}) }}" + matrix_synapse_stream_writers: "{{ matrix_synapse_stream_writers | combine({item.ansible_facts.worker.stream_writer_stream: [item.ansible_facts.worker.name]}, list_merge='append') }}" with_items: "{{ matrix_synapse_workers_list_results_stream_writer_workers.results }}" - name: Build federation sender workers From 28d9ee857895e257bef651df9d91e45b5f28335c Mon Sep 17 00:00:00 2001 From: Darren Rambaud Date: Tue, 1 Nov 2022 14:12:52 -0500 Subject: [PATCH 116/118] dendrite: update tag to v0.10.6 - update tag to latest dendrite release (https://github.com/matrix-org/dendrite/releases/tag/v0.10.6) --- roles/matrix-dendrite/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-dendrite/defaults/main.yml b/roles/matrix-dendrite/defaults/main.yml index 945c1619..a1d03960 100644 --- a/roles/matrix-dendrite/defaults/main.yml +++ b/roles/matrix-dendrite/defaults/main.yml @@ -6,7 +6,7 @@ matrix_dendrite_enabled: true matrix_dendrite_docker_image: "{{ matrix_dendrite_docker_image_name_prefix }}matrixdotorg/dendrite-monolith:{{ matrix_dendrite_docker_image_tag }}" matrix_dendrite_docker_image_name_prefix: "docker.io/" -matrix_dendrite_docker_image_tag: "v0.10.5" +matrix_dendrite_docker_image_tag: "v0.10.6" matrix_dendrite_docker_image_force_pull: "{{ matrix_dendrite_docker_image.endswith(':latest') }}" matrix_dendrite_base_path: "{{ matrix_base_data_path }}/dendrite" From 8a609e5cf2b0bf59fdbf36cdcaafdfc9cf750d1d Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 2 Nov 2022 07:00:45 +0200 Subject: [PATCH 117/118] Fix some ansible-lint-reported errors --- roles/matrix-base/tasks/setup_matrix_base.yml | 2 +- roles/matrix-bridge-hookshot/defaults/main.yml | 4 ++-- roles/matrix-postgres/tasks/import_postgres.yml | 4 ++-- roles/matrix-postgres/tasks/run_vacuum.yml | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index 0bf197b6..2439fdea 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -22,7 +22,7 @@ - name: Save current git-repo status on the target to aid with restoring in case of problems when: "matrix_playbook_commit_hash_preservation_enabled|bool" block: - - name: Get local git hash # noqa command-instead-of-module + - name: Get local git hash # noqa command-instead-of-module delegate_to: 127.0.0.1 become: false register: git_describe diff --git a/roles/matrix-bridge-hookshot/defaults/main.yml b/roles/matrix-bridge-hookshot/defaults/main.yml index 96781f9a..4c39876a 100644 --- a/roles/matrix-bridge-hookshot/defaults/main.yml +++ b/roles/matrix-bridge-hookshot/defaults/main.yml @@ -129,8 +129,8 @@ matrix_hookshot_generic_user_id_prefix: '_webhooks_' matrix_hookshot_feeds_enabled: true -matrix_hookshot_feeds_pollIntervalSeconds: 600 -matrix_hookshot_feeds_pollTimeoutSeconds: 10 +matrix_hookshot_feeds_pollIntervalSeconds: 600 # no-qa var-naming +matrix_hookshot_feeds_pollTimeoutSeconds: 10 # no-qa var-naming # There is no need to edit ports. use matrix_hookshot_container_http_host_bind_ports below to expose ports instead. diff --git a/roles/matrix-postgres/tasks/import_postgres.yml b/roles/matrix-postgres/tasks/import_postgres.yml index 107a1f88..d5bfaa86 100644 --- a/roles/matrix-postgres/tasks/import_postgres.yml +++ b/roles/matrix-postgres/tasks/import_postgres.yml @@ -51,8 +51,8 @@ daemon_reload: true register: matrix_postgres_import_start_result -- when: matrix_postgres_import_start_result.changed | bool - name: Wait a bit, so that Postgres can start +- name: Wait a bit, so that Postgres can start + when: matrix_postgres_import_start_result.changed | bool ansible.builtin.wait_for: timeout: "{{ postgres_start_wait_time }}" delegate_to: 127.0.0.1 diff --git a/roles/matrix-postgres/tasks/run_vacuum.yml b/roles/matrix-postgres/tasks/run_vacuum.yml index 1b769f01..c1bf8d76 100644 --- a/roles/matrix-postgres/tasks/run_vacuum.yml +++ b/roles/matrix-postgres/tasks/run_vacuum.yml @@ -30,8 +30,8 @@ daemon_reload: true register: matrix_postgres_vacuum_start_result -- when: matrix_postgres_vacuum_start_result.changed | bool - name: Wait a bit, so that Postgres can start +- name: Wait a bit, so that Postgres can start + when: matrix_postgres_vacuum_start_result.changed | bool ansible.builtin.wait_for: timeout: "{{ postgres_start_wait_time }}" delegate_to: 127.0.0.1 From 0fd9f3d4f99076fab4bc85a8b16dfda324aa4091 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Wed, 2 Nov 2022 07:21:09 +0000 Subject: [PATCH 118/118] Pin mautrix-instagram to v0.2.2 --- roles/matrix-bridge-mautrix-instagram/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-mautrix-instagram/defaults/main.yml b/roles/matrix-bridge-mautrix-instagram/defaults/main.yml index 7305d0fb..dcdf6723 100644 --- a/roles/matrix-bridge-mautrix-instagram/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-instagram/defaults/main.yml @@ -8,7 +8,7 @@ matrix_mautrix_instagram_container_image_self_build: false matrix_mautrix_instagram_container_image_self_build_repo: "https://github.com/mautrix/instagram.git" matrix_mautrix_instagram_container_image_self_build_repo_version: "{{ 'master' if matrix_mautrix_instagram_version == 'latest' else matrix_mautrix_instagram_version }}" -matrix_mautrix_instagram_version: latest +matrix_mautrix_instagram_version: v0.2.2 # See: https://mau.dev/tulir/mautrix-instagram/container_registry matrix_mautrix_instagram_docker_image: "{{ matrix_mautrix_instagram_docker_image_name_prefix }}mautrix/instagram:{{ matrix_mautrix_instagram_version }}" matrix_mautrix_instagram_docker_image_name_prefix: "{{ 'localhost/' if matrix_mautrix_instagram_container_image_self_build else 'dock.mau.dev/' }}"