From 69efcb5c21ad55bf597a298a3f6ba499c729920a Mon Sep 17 00:00:00 2001
From: Sabine <sabinbox@laszakovits.net>
Date: Thu, 22 Oct 2020 23:29:34 +0200
Subject: [PATCH 001/179] added mautrix-signal role

---
 .../defaults/main.yml                         | 102 +++++++++
 .../tasks/init.yml                            |  63 ++++++
 .../tasks/main.yml                            |  21 ++
 .../tasks/setup_install.yml                   | 102 +++++++++
 .../tasks/setup_uninstall.yml                 |  64 ++++++
 .../tasks/validate_config.yml                 |  22 ++
 .../templates/config.yaml.j2                  | 207 ++++++++++++++++++
 .../matrix-mautrix-signal-daemon.service.j2   |  29 +++
 .../matrix-mautrix-signal-db.service.j2       |  32 +++
 .../systemd/matrix-mautrix-signal.service.j2  |  44 ++++
 setup.yml                                     |   1 +
 11 files changed, 687 insertions(+)
 create mode 100644 roles/matrix-bridge-mautrix-signal/defaults/main.yml
 create mode 100644 roles/matrix-bridge-mautrix-signal/tasks/init.yml
 create mode 100644 roles/matrix-bridge-mautrix-signal/tasks/main.yml
 create mode 100644 roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml
 create mode 100644 roles/matrix-bridge-mautrix-signal/tasks/setup_uninstall.yml
 create mode 100644 roles/matrix-bridge-mautrix-signal/tasks/validate_config.yml
 create mode 100644 roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2
 create mode 100644 roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal-daemon.service.j2
 create mode 100644 roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal-db.service.j2
 create mode 100644 roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal.service.j2

diff --git a/roles/matrix-bridge-mautrix-signal/defaults/main.yml b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
new file mode 100644
index 00000000..d86bdb59
--- /dev/null
+++ b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
@@ -0,0 +1,102 @@
+# mautrix-telegram is a Matrix <-> Telegram bridge
+# See: https://github.com/tulir/mautrix-telegram
+
+matrix_mautrix_signal_enabled: false
+
+# See: https://mau.dev/tulir/mautrix-signal/container_registry
+matrix_mautrix_signal_docker_image: "dock.mau.dev/tulir/mautrix-signal:latest"
+matrix_mautrix_signal_docker_image_force_pull: "{{ matrix_mautrix_signal_docker_image.endswith(':latest') }}"
+
+matrix_mautrix_signal_base_path: "{{ matrix_base_data_path }}/mautrix-signal"
+matrix_mautrix_signal_config_path: "{{ matrix_mautrix_signal_base_path }}/bridge"
+
+matrix_mautrix_signal_daemon_docker_image: "dock.mau.dev/maunium/signald:latest"
+matrix_mautrix_signal_daemon_docker_image_force_pull: "{{ matrix_mautrix_signal_daemon_docker_image.endswith(':latest') }}"
+
+matrix_mautrix_signal_daemon_path: "{{ matrix_mautrix_signal_base_path }}/signald"
+
+# # Get your own API keys at https://my.telegram.org/apps
+# matrix_mautrix_telegram_api_id: ''
+# matrix_mautrix_telegram_api_hash: ''
+# matrix_mautrix_telegram_bot_token: disabled
+
+# # Mautrix telegram public endpoint to log in to telegram
+# # Use an uuid so it's not easily discoverable.
+# # Example: /741a0483-ba17-4682-9900-30bd7269f1cc
+# matrix_mautrix_telegram_public_endpoint: ''
+
+matrix_mautrix_signal_homeserver_address: 'http://matrix-synapse:8008'
+matrix_mautrix_signal_homeserver_domain: '{{ matrix_domain }}'
+matrix_mautrix_signal_appservice_address: 'http://matrix-mautrix-signal:29328'
+# matrix_mautrix_telegram_appservice_public_external: 'https://{{ matrix_server_fqn_matrix }}{{ matrix_mautrix_telegram_public_endpoint }}'
+
+# Controls whether the matrix-mautrix-telegram container exposes its HTTP port (tcp/8080 in the container).
+#
+# Takes an "<ip>:<port>" or "<port>" value (e.g. "127.0.0.1:9006"), or empty string to not expose.
+matrix_mautrix_signal_container_http_host_bind_port: ''
+
+# A list of extra arguments to pass to the container
+matrix_mautrix_signal_container_extra_arguments: []
+
+# List of systemd services that matrix-mautrix-signal.service depends on.
+matrix_mautrix_signal_systemd_required_services_list: ['docker.service', 'matrix-mautrix-signal-daemon.service', 'matrix-mautrix-signal-db.service']
+
+# List of systemd services that matrix-mautrix-telegram.service wants
+matrix_mautrix_signal_systemd_wanted_services_list: []
+
+matrix_mautrix_signal_appservice_token: ''
+matrix_mautrix_signal_homeserver_token: ''
+
+matrix_mautrix_signal_db_docker_image: "postgres:13.0-alpine"
+matrix_mautrix_signal_db_docker_image_force_pull: "{{ matrix_mautrix_signal_db_docker_image.endswith(':latest') }}"
+matrix_mautrix_signal_db_storage_path: "{{ matrix_mautrix_signal_base_path }}/database"
+
+matrix_mautrix_signal_db_user: ''
+matrix_mautrix_signal_db_password: ''
+matrix_mautrix_signal_db_host: 'matrix-mautrix-signal-db'
+matrix_mautrix_signal_db_port: '5432'
+matrix_mautrix_signal_db_database: ''
+
+matrix_mautrix_signal_db_url: "postgres://{{ matrix_mautrix_signal_db_user }}:{{ matrix_mautrix_signal_db_password }}@{{ matrix_mautrix_signal_db_host }}:{{ matrix_mautrix_signal_db_port }}/{{ matrix_mautrix_signal_db_database }}"
+
+# Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
+matrix_mautrix_signal_login_shared_secret: ''
+
+# Default configuration template which covers the generic use case.
+# You can customize it by controlling the various variables inside it.
+#
+# For a more advanced customization, you can extend the default (see `matrix_mautrix_telegram_configuration_extension_yaml`)
+# or completely replace this variable with your own template.
+matrix_mautrix_signal_configuration_yaml: "{{ lookup('template', 'templates/config.yaml.j2') }}"
+
+matrix_mautrix_signal_configuration_extension_yaml: |
+  # Your custom YAML configuration goes here.
+  # This configuration extends the default starting configuration (`matrix_mautrix_telegram_configuration_yaml`).
+  #
+  # You can override individual variables from the default configuration, or introduce new ones.
+  #
+  # If you need something more special, you can take full control by
+  # completely redefining `matrix_mautrix_telegram_configuration_yaml`.
+
+matrix_mautrix_signal_configuration_extension: "{{ matrix_mautrix_signal_configuration_extension_yaml|from_yaml if matrix_mautrix_signal_configuration_extension_yaml|from_yaml is mapping else {} }}"
+
+# Holds the final configuration (a combination of the default and its extension).
+# You most likely don't need to touch this variable. Instead, see `matrix_mautrix_telegram_configuration_yaml`.
+matrix_mautrix_signal_configuration: "{{ matrix_mautrix_signal_configuration_yaml|from_yaml|combine(matrix_mautrix_signal_configuration_extension, recursive=True) }}"
+
+matrix_mautrix_signal_registration_yaml: |
+  id: signal
+  as_token: "{{ matrix_mautrix_signal_appservice_token }}"
+  hs_token: "{{ matrix_mautrix_signal_homeserver_token }}"
+  namespaces:
+      users:
+      - exclusive: true
+        regex: '^@signal_.+:{{ matrix_mautrix_signal_homeserver_domain|regex_escape }}$'
+      aliases:
+      - exclusive: true
+        regex: '^#signal_.+:{{ matrix_mautrix_signal_homeserver_domain|regex_escape }}$'
+  url: {{ matrix_mautrix_signal_appservice_address }}
+  sender_localpart: signalbot
+  rate_limited: false
+
+matrix_mautrix_signal_registration: "{{ matrix_mautrix_signal_registration_yaml|from_yaml }}"
diff --git a/roles/matrix-bridge-mautrix-signal/tasks/init.yml b/roles/matrix-bridge-mautrix-signal/tasks/init.yml
new file mode 100644
index 00000000..8fdc9834
--- /dev/null
+++ b/roles/matrix-bridge-mautrix-signal/tasks/init.yml
@@ -0,0 +1,63 @@
+- set_fact:
+    matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-mautrix-signal', 'matrix-mautrix-signal-daemon'] }}"
+  when: matrix_mautrix_signal_enabled|bool
+
+# If the matrix-synapse role is not used, these variables may not exist.
+- set_fact:
+    matrix_synapse_container_extra_arguments: >
+      {{ matrix_synapse_container_extra_arguments|default([]) }}
+      +
+      ["--mount type=bind,src={{ matrix_mautrix_signal_config_path }}/registration.yaml,dst=/matrix-mautrix-signal-registration.yaml,ro"]
+
+    matrix_synapse_app_service_config_files: >
+      {{ matrix_synapse_app_service_config_files|default([]) }}
+      +
+      {{ ["/matrix-mautrix-signal-registration.yaml"] }}
+  when: matrix_mautrix_signal_enabled|bool
+
+# - block:
+#   - name: Fail if matrix-nginx-proxy role already executed
+#     fail:
+#       msg: >-
+#         Trying to append Mautrix Signal's reverse-proxying configuration to matrix-nginx-proxy,
+#         but it's pointless since the matrix-nginx-proxy role had already executed.
+#         To fix this, please change the order of roles in your plabook,
+#         so that the matrix-nginx-proxy role would run after the matrix-bridge-mautrix-signal role.
+#     when: matrix_nginx_proxy_role_executed|default(False)|bool
+
+  # - name: Generate Mautrix Signal proxying configuration for matrix-nginx-proxy
+  #   set_fact:
+  #     matrix_mautrix_telegram_matrix_nginx_proxy_configuration: |
+  #       location {{ matrix_mautrix_telegram_public_endpoint }} {
+  #       {% if matrix_nginx_proxy_enabled|default(False) %}
+  #       	{# Use the embedded DNS resolver in Docker containers to discover the service #}
+  #       	resolver 127.0.0.11 valid=5s;
+  #       	set $backend "matrix-mautrix-telegram:8080";
+  #       	proxy_pass http://$backend;
+  #       {% else %}
+  #       	{# Generic configuration for use outside of our container setup #}
+  #       	proxy_pass http://127.0.0.1:9006;
+  #       {% endif %}
+  #       }
+
+  # - name: Register Mautrix Telegram proxying configuration with matrix-nginx-proxy
+  #   set_fact:
+  #     matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks: |
+  #       {{
+  #         matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks|default([])
+  #         +
+  #         [matrix_mautrix_telegram_matrix_nginx_proxy_configuration]
+  #       }}
+  # tags:
+  #  - always
+  # when: matrix_mautrix_telegram_enabled|bool
+
+# - name: Warn about reverse-proxying if matrix-nginx-proxy not used
+#   debug:
+#     msg: >-
+#       NOTE: You've enabled the Mautrix Telegram bridge but are not using the matrix-nginx-proxy
+#       reverse proxy.
+#       Please make sure that you're proxying the `{{ matrix_mautrix_telegram_public_endpoint }}`
+#       URL endpoint to the matrix-mautrix-telegram container.
+#       You can expose the container's port using the `matrix_mautrix_telegram_container_http_host_bind_port` variable.
+#   when: "matrix_mautrix_telegram_enabled|bool and matrix_nginx_proxy_enabled is not defined"
diff --git a/roles/matrix-bridge-mautrix-signal/tasks/main.yml b/roles/matrix-bridge-mautrix-signal/tasks/main.yml
new file mode 100644
index 00000000..edca20e6
--- /dev/null
+++ b/roles/matrix-bridge-mautrix-signal/tasks/main.yml
@@ -0,0 +1,21 @@
+- import_tasks: "{{ role_path }}/tasks/init.yml"
+  tags:
+    - always
+
+- import_tasks: "{{ role_path }}/tasks/validate_config.yml"
+  when: "run_setup|bool and matrix_mautrix_signal_enabled|bool"
+  tags:
+    - setup-all
+    - setup-mautrix-signal
+
+- import_tasks: "{{ role_path }}/tasks/setup_install.yml"
+  when: "run_setup|bool and matrix_mautrix_signal_enabled|bool"
+  tags:
+    - setup-all
+    - setup-mautrix-signal
+
+- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml"
+  when: "run_setup|bool and not matrix_mautrix_signal_enabled|bool"
+  tags:
+    - setup-all
+    - setup-mautrix-signal
diff --git a/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml
new file mode 100644
index 00000000..cacb4c6c
--- /dev/null
+++ b/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml
@@ -0,0 +1,102 @@
+---
+
+# If the matrix-synapse role is not used, `matrix_synapse_role_executed` won't exist.
+# We don't want to fail in such cases.
+- name: Fail if matrix-synapse role already executed
+  fail:
+    msg: >-
+      The matrix-bridge-mautrix-signal role needs to execute before the matrix-synapse role.
+  when: "matrix_synapse_role_executed|default(False)"
+
+- name: Ensure Mautrix Signal image is pulled
+  docker_image:
+    name: "{{ matrix_mautrix_signal_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_mautrix_signal_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mautrix_signal_docker_image_force_pull }}"
+
+- name: Ensure Mautrix Signal Daemon image is pulled
+  docker_image:
+    name: "{{ matrix_mautrix_signal_daemon_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_mautrix_signal_daemon_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mautrix_signal_docker_image_force_pull }}"
+
+- name: Ensure Mautrix Signal database image is pulled
+  docker_image:
+    name: "{{ matrix_mautrix_signal_db_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_mautrix_signal_db_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mautrix_signal_docker_image_force_pull }}"
+
+- name: Ensure Mautrix Signal paths exist
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: 0750
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+  with_items:
+    - "{{ matrix_mautrix_signal_base_path }}"
+    - "{{ matrix_mautrix_signal_config_path }}"
+    - "{{ matrix_mautrix_signal_daemon_path }}"
+    - "{{ matrix_mautrix_signal_db_storage_path }}"
+
+# - name: Check if an old database file already exists
+#   stat:
+#     path: "{{ matrix_mautrix_telegram_base_path }}/mautrix-telegram.db"
+#   register: matrix_mautrix_telegram_stat_database
+#
+# - name: (Data relocation) Ensure matrix-mautrix-telegram.service is stopped
+#   service:
+#     name: matrix-mautrix-telegram
+#     state: stopped
+#     daemon_reload: yes
+#   failed_when: false
+#   when: "matrix_mautrix_telegram_stat_database.stat.exists"
+#
+# - name: (Data relocation) Move mautrix-telegram database file to ./data directory
+#   command: "mv {{ matrix_mautrix_telegram_base_path }}/mautrix-telegram.db {{ matrix_mautrix_telegram_data_path }}/mautrix-telegram.db"
+#   when: "matrix_mautrix_telegram_stat_database.stat.exists"
+
+- name: Ensure mautrix-signal config.yaml installed
+  copy:
+    content: "{{ matrix_mautrix_signal_configuration|to_nice_yaml }}"
+    dest: "{{ matrix_mautrix_signal_config_path }}/config.yaml"
+    mode: 0644
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+
+- name: Ensure mautrix-signal registration.yaml installed
+  copy:
+    content: "{{ matrix_mautrix_signal_registration|to_nice_yaml }}"
+    dest: "{{ matrix_mautrix_signal_config_path }}/registration.yaml"
+    mode: 0644
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+
+- name: Ensure matrix-mautrix-signal-daemon.service installed
+  template:
+    src: "{{ role_path }}/templates/systemd/matrix-mautrix-signal-daemon.service.j2"
+    dest: "{{ matrix_systemd_path }}/matrix-mautrix-signal-daemon.service"
+    mode: 0644
+  register: matrix_mautrix_signal_daemon_systemd_service_result
+
+- name: Ensure matrix-mautrix-signal-db.service installed
+  template:
+    src: "{{ role_path }}/templates/systemd/matrix-mautrix-signal-db.service.j2"
+    dest: "{{ matrix_systemd_path }}/matrix-mautrix-signal-db.service"
+    mode: 0644
+  register: matrix_mautrix_signal_db_systemd_service_result
+
+- name: Ensure matrix-mautrix-signal.service installed
+  template:
+    src: "{{ role_path }}/templates/systemd/matrix-mautrix-signal.service.j2"
+    dest: "{{ matrix_systemd_path }}/matrix-mautrix-signal.service"
+    mode: 0644
+  register: matrix_mautrix_signal_systemd_service_result
+
+- name: Ensure systemd reloaded after matrix-mautrix-signal.service installation
+  service:
+    daemon_reload: yes
+  when: "matrix_mautrix_signal_systemd_service_result.changed or matrix_mautrix_signal_daemon_systemd_service_result.changed or matrix_mautrix_signal_db_systemd_service_result.changed"
diff --git a/roles/matrix-bridge-mautrix-signal/tasks/setup_uninstall.yml b/roles/matrix-bridge-mautrix-signal/tasks/setup_uninstall.yml
new file mode 100644
index 00000000..51581874
--- /dev/null
+++ b/roles/matrix-bridge-mautrix-signal/tasks/setup_uninstall.yml
@@ -0,0 +1,64 @@
+---
+
+# Signal database service
+- name: Check existence of matrix-mautrix-signal-db service
+  stat:
+    path: "{{ matrix_systemd_path }}/matrix-mautrix-signal-db.service"
+  register: matrix_mautrix_signal_db_service_stat
+
+- name: Ensure matrix-mautrix-signal-db is stopped
+  service:
+    name: matrix-mautrix-signal-db
+    state: stopped
+    daemon_reload: yes
+  when: "matrix_mautrix_signal_db_service_stat.stat.exists"
+
+- name: Ensure matrix-mautrix-signal-db.service doesn't exist
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-mautrix-signal-db.service"
+    state: absent
+  when: "matrix_mautrix_signal_db_service_stat.stat.exists"
+
+# Signal daemon service
+- name: Check existence of matrix-mautrix-signal-daemon service
+  stat:
+    path: "{{ matrix_systemd_path }}/matrix-mautrix-signal-daemon.service"
+  register: matrix_mautrix_signal_daemon_service_stat
+
+- name: Ensure matrix-mautrix-signal-daemon is stopped
+  service:
+    name: matrix-mautrix-signal-daemon
+    state: stopped
+    daemon_reload: yes
+  when: "matrix_mautrix_signal_daemon_service_stat.stat.exists"
+
+- name: Ensure matrix-mautrix-signal-daemon.service doesn't exist
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-mautrix-signal-daemon.service"
+    state: absent
+  when: "matrix_mautrix_signal_daemon_service_stat.stat.exists"
+
+# Bridge service
+- name: Check existence of matrix-mautrix-signal service
+  stat:
+    path: "{{ matrix_systemd_path }}/matrix-mautrix-signal.service"
+  register: matrix_mautrix_signal_service_stat
+
+- name: Ensure matrix-mautrix-signal is stopped
+  service:
+    name: matrix-mautrix-signal
+    state: stopped
+    daemon_reload: yes
+  when: "matrix_mautrix_signal_service_stat.stat.exists"
+
+- name: Ensure matrix-mautrix-signal.service doesn't exist
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-mautrix-signal.service"
+    state: absent
+  when: "matrix_mautrix_signal_service_stat.stat.exists"
+
+# All services
+- name: Ensure systemd reloaded after matrix-mautrix-signal_X.service removal
+  service:
+    daemon_reload: yes
+  when: "matrix_mautrix_signal_service_stat.stat.exists or matrix_mautrix_signal_daemon_service_stat.stat.exists or matrix_mautrix_signal_db_service_stat.stat.exists"
diff --git a/roles/matrix-bridge-mautrix-signal/tasks/validate_config.yml b/roles/matrix-bridge-mautrix-signal/tasks/validate_config.yml
new file mode 100644
index 00000000..a11e192d
--- /dev/null
+++ b/roles/matrix-bridge-mautrix-signal/tasks/validate_config.yml
@@ -0,0 +1,22 @@
+---
+
+- name: Fail if required settings not defined
+  fail:
+    msg: >-
+      You need to define a required configuration setting (`{{ item }}`).
+  when: "vars[item] == ''"
+  with_items:
+    - "matrix_mautrix_signal_appservice_token"
+    - "matrix_mautrix_signal_homeserver_token"
+    - "matrix_mautrix_signal_db_user"
+    - "matrix_mautrix_signal_db_password"
+    - "matrix_mautrix_signal_db_database"
+
+- name: (Deprecation) Catch and report renamed Signal variables
+  fail:
+    msg: >-
+      Your configuration contains a variable, which now has a different name.
+      Please change your configuration to rename the variable (`{{ item.old }}` -> `{{ item.new }}`).
+  when: "item.old in vars"
+  with_items:
+    - {'old': 'matrix_mautrix_signal_container_exposed_port_number', 'new': '<superseded by matrix_mautrix_signal_container_http_host_bind_port>'}
diff --git a/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2
new file mode 100644
index 00000000..6865d3c1
--- /dev/null
+++ b/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2
@@ -0,0 +1,207 @@
+#jinja2: lstrip_blocks: "True"
+# Homeserver details
+homeserver:
+    # The address that this appservice can use to connect to the homeserver.
+    address: {{ matrix_mautrix_signal_homeserver_address }}
+    # The domain of the homeserver (for MXIDs, etc).
+    domain: {{ matrix_mautrix_signal_homeserver_domain }}
+    # Whether or not to verify the SSL certificate of the homeserver.
+    # Only applies if address starts with https://
+    verify_ssl: true
+    asmux: false
+
+# Application service host/registration related details
+# Changing these values requires regeneration of the registration.
+appservice:
+    # The address that the homeserver can use to connect to this appservice.
+    address: {{ matrix_mautrix_signal_appservice_address }}
+    # When using https:// the TLS certificate and key files for the address.
+    tls_cert: false
+    tls_key: false
+
+    # The hostname and port where this appservice should listen.
+    hostname: 0.0.0.0
+    port: 29328
+    # The maximum body size of appservice API requests (from the homeserver) in mebibytes
+    # Usually 1 is enough, but on high-traffic bridges you might need to increase this to avoid 413s
+    max_body_size: 1
+
+    # The full URI to the database. Only Postgres is currently supported.
+    database: {{ matrix_mautrix_signal_db_url }}
+
+    # Provisioning API part of the web server for automated portal creation and fetching information.
+    # Used by things like mautrix-manager (https://github.com/tulir/mautrix-manager).
+    provisioning:
+        # Whether or not the provisioning API should be enabled.
+        enabled: true
+        # The prefix to use in the provisioning API endpoints.
+        prefix: /_matrix/provision/v1
+        # The shared secret to authorize users of the API.
+        # Set to "generate" to generate and save a new token.
+        shared_secret: generate
+
+    # The unique ID of this appservice.
+    id: signal
+    # Username of the appservice bot.
+    bot_username: signalbot
+    # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty
+    # to leave display name/avatar as-is.
+    bot_displayname: Signal bridge bot
+    bot_avatar: mxc://maunium.net/wPJgTQbZOtpBFmDNkiNEMDUp
+
+    # Community ID for bridged users (changes registration file) and rooms.
+    # Must be created manually.
+    #
+    # Example: "+signal:example.com". Set to false to disable.
+    community_id: false
+
+    # Authentication tokens for AS <-> HS communication.
+    as_token: "{{ matrix_mautrix_signal_appservice_token }}"
+    hs_token: "{{ matrix_mautrix_signal_homeserver_token }}"
+
+# Prometheus telemetry config. Requires prometheus-client to be installed.
+metrics:
+    enabled: false
+    listen_port: 8000
+
+signal:
+    # Path to signald unix socket
+    socket_path: /signald/signald.sock
+    # Directory for temp files when sending files to Signal. This should be an
+    # absolute path that signald can read. For attachments in the other direction,
+    # make sure signald is configured to use an absolute path as the data directory.
+    outgoing_attachment_dir: /signald/attachments
+    # Directory where signald stores avatars for groups.
+    avatar_dir: /signald/avatars
+    # Whether or not message attachments should be removed from disk after they're bridged.
+    remove_file_after_handling: true
+
+# Bridge config
+bridge:
+    # Localpart template of MXIDs for Signal users.
+    # {userid} is replaced with an identifier for the Signal user.
+    username_template: "signal_{userid}"
+    # Displayname template for Signal users.
+    # {displayname} is replaced with the displayname of the Signal user, which is the first
+    # available variable in displayname_preference. The variables in displayname_preference
+    # can also be used here directly.
+    displayname_template: "{displayname} (Signal)"
+    # Whether or not contact list displaynames should be used.
+    # Using this isn't recommended on multi-user instances.
+    allow_contact_list_name_updates: false
+    # Available variables: full_name, first_name, last_name, phone, uuid
+    displayname_preference:
+    - full_name
+    - phone
+
+    # Whether or not to create portals for all groups on login/connect.
+    autocreate_group_portal: true
+    # Whether or not to create portals for all contacts on login/connect.
+    autocreate_contact_portal: false
+    # Whether or not to use /sync to get read receipts and typing notifications
+    # when double puppeting is enabled
+    sync_with_custom_puppets: true
+    # Whether or not to update the m.direct account data event when double puppeting is enabled.
+    # Note that updating the m.direct event is not atomic (except with mautrix-asmux)
+    # and is therefore prone to race conditions.
+    sync_direct_chat_list: false
+    # Allow using double puppeting from any server with a valid client .well-known file.
+    double_puppet_allow_discovery: false
+    # Servers to allow double puppeting from, even if double_puppet_allow_discovery is false.
+    double_puppet_server_map: {}
+    # Shared secret for https://github.com/devture/matrix-synapse-shared-secret-auth
+    #
+    # If set, custom puppets will be enabled automatically for local users
+    # instead of users having to find an access token and run `login-matrix`
+    # manually.
+    # If using this for other servers than the bridge's server,
+    # you must also set the URL in the double_puppet_server_map.
+    login_shared_secret_map:
+        {{ matrix_mautrix_signal_homeserver_domain }}: {{ matrix_mautrix_signal_login_shared_secret|to_json }}
+    # Whether or not created rooms should have federation enabled.
+    # If false, created portal rooms will never be federated.
+    federate_rooms: true
+    # End-to-bridge encryption support options. These require matrix-nio to be installed with pip
+    # and login_shared_secret to be configured in order to get a device for the bridge bot.
+    #
+    # Additionally, https://github.com/matrix-org/synapse/pull/5758 is required if using a normal
+    # application service.
+    encryption:
+        # Allow encryption, work in group chat rooms with e2ee enabled
+        allow: false
+        # Default to encryption, force-enable encryption in all portals the bridge creates
+        # This will cause the bridge bot to be in private chats for the encryption to work properly.
+        default: false
+        # Options for automatic key sharing.
+        key_sharing:
+            # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled.
+            # You must use a client that supports requesting keys from other users to use this feature.
+            allow: false
+            # Require the requesting device to have a valid cross-signing signature?
+            # This doesn't require that the bridge has verified the device, only that the user has verified it.
+            # Not yet implemented.
+            require_cross_signing: false
+            # Require devices to be verified by the bridge?
+            # Verification by the bridge is not yet implemented.
+            require_verification: true
+    # Whether or not to explicitly set the avatar and room name for private
+    # chat portal rooms. This will be implicitly enabled if encryption.default is true.
+    private_chat_portal_meta: false
+    # Whether or not the bridge should send a read receipt from the bridge bot when a message has
+    # been sent to Signal. This let's you check manually whether the bridge is receiving your
+    # messages.
+    # Note that this is not related to Signal delivery receipts.
+    delivery_receipts: false
+    # Whether or not delivery errors should be reported as messages in the Matrix room. (not yet implemented)
+    delivery_error_reports: false
+    # Set this to true to tell the bridge to re-send m.bridge events to all rooms on the next run.
+    # This field will automatically be changed back to false after it,
+    # except if the config file is not writable.
+    resend_bridge_info: false
+
+    # The prefix for commands. Only required in non-management rooms.
+    command_prefix: "!signal"
+
+    # Permissions for using the bridge.
+    # Permitted values:
+    #       user - Use the bridge with puppeting.
+    #      admin - Use and administrate the bridge.
+    # Permitted keys:
+    #        * - All Matrix users
+    #   domain - All users on that homeserver
+    #     mxid - Specific user
+    permissions:
+        "1m.at": "user"
+        "@admin:1m.at": "admin"
+
+
+# Python logging configuration.
+#
+# See section 16.7.2 of the Python documentation for more info:
+# https://docs.python.org/3.6/library/logging.config.html#configuration-dictionary-schema
+logging:
+    version: 1
+    formatters:
+        colored:
+            (): mautrix_signal.util.ColorFormatter
+            format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s"
+        normal:
+            format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s"
+    handlers:
+        file:
+            class: logging.handlers.RotatingFileHandler
+            formatter: normal
+            filename: ./mautrix-signal.log
+            maxBytes: 10485760
+            backupCount: 10
+        console:
+            class: logging.StreamHandler
+            formatter: colored
+    loggers:
+        mau:
+            level: DEBUG
+        aiohttp:
+            level: INFO
+    root:
+        level: DEBUG
+        handlers: [file, console]
diff --git a/roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal-daemon.service.j2 b/roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal-daemon.service.j2
new file mode 100644
index 00000000..499ee309
--- /dev/null
+++ b/roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal-daemon.service.j2
@@ -0,0 +1,29 @@
+#jinja2: lstrip_blocks: "True"
+[Unit]
+Description=Matrix Mautrix Signal daemon
+
+[Service]
+Type=simple
+
+ExecStartPre=-{{ matrix_host_command_docker }} kill matrix-mautrix-signal-daemon
+ExecStartPre=-{{ matrix_host_command_docker }} rm matrix-mautrix-signal-daemon
+
+# Intentional delay, so that the homeserver (we likely depend on) can manage to start.
+ExecStartPre={{ matrix_host_command_sleep }} 5
+
+ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-mautrix-signal-daemon \
+			--log-driver=none \
+			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
+			--network={{ matrix_docker_network }} \
+			-v {{ matrix_mautrix_signal_daemon_path }}:/signald:z \
+			{{ matrix_mautrix_signal_daemon_docker_image }}
+
+ExecStop=-{{ matrix_host_command_docker }} kill matrix-mautrix-signal-daemon
+ExecStop=-{{ matrix_host_command_docker }} rm matrix-mautrix-signal-daemon
+
+Restart=always
+RestartSec=30
+SyslogIdentifier=matrix-mautrix-signal-daemon
+
+[Install]
+WantedBy=multi-user.target
diff --git a/roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal-db.service.j2 b/roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal-db.service.j2
new file mode 100644
index 00000000..653b67b8
--- /dev/null
+++ b/roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal-db.service.j2
@@ -0,0 +1,32 @@
+#jinja2: lstrip_blocks: "True"
+[Unit]
+Description=Matrix Mautrix Signal database server
+
+[Service]
+Type=simple
+
+ExecStartPre=-{{ matrix_host_command_docker }} kill {{ matrix_mautrix_signal_db_host }}
+ExecStartPre=-{{ matrix_host_command_docker }} rm {{ matrix_mautrix_signal_db_host }}
+
+# Intentional delay, so that the homeserver (we likely depend on) can manage to start.
+ExecStartPre={{ matrix_host_command_sleep }} 5
+
+ExecStart={{ matrix_host_command_docker }} run --rm --name {{ matrix_mautrix_signal_db_host }} \
+			--log-driver=none \
+			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
+			--network={{ matrix_docker_network }} \
+			--env POSTGRES_USER={{ matrix_mautrix_signal_db_user }} \
+			--env POSTGRES_PASSWORD={{ matrix_mautrix_signal_db_password }} \
+			--env POSTGRES_DB={{ matrix_mautrix_signal_db_database }} \
+			-v {{ matrix_mautrix_signal_db_storage_path }}:/var/lib/postgresql/data:z \
+			{{ matrix_mautrix_signal_db_docker_image }}
+
+ExecStop=-{{ matrix_host_command_docker }} kill {{ matrix_mautrix_signal_db_host }}
+ExecStop=-{{ matrix_host_command_docker }} rm {{ matrix_mautrix_signal_db_host }}
+
+Restart=always
+RestartSec=30
+SyslogIdentifier=matrix-mautrix-signal-db
+
+[Install]
+WantedBy=multi-user.target
diff --git a/roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal.service.j2 b/roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal.service.j2
new file mode 100644
index 00000000..c1c835a2
--- /dev/null
+++ b/roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal.service.j2
@@ -0,0 +1,44 @@
+#jinja2: lstrip_blocks: "True"
+[Unit]
+Description=Matrix Mautrix Signal server
+
+{% for service in matrix_mautrix_signal_systemd_required_services_list %}
+Requires={{ service }}
+After={{ service }}
+{% endfor %}
+
+{% for service in matrix_mautrix_signal_systemd_wanted_services_list %}
+Wants={{ service }}
+{% endfor %}
+
+[Service]
+Type=simple
+ExecStartPre=-{{ matrix_host_command_docker }} kill matrix-mautrix-signal
+ExecStartPre=-{{ matrix_host_command_docker }} rm matrix-mautrix-signal
+
+# Intentional delay, so that the homeserver (we likely depend on) can manage to start.
+ExecStartPre={{ matrix_host_command_sleep }} 5
+
+ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-mautrix-signal \
+			--log-driver=none \
+			--network={{ matrix_docker_network }} \
+			{% if matrix_mautrix_signal_container_http_host_bind_port %}
+			-p {{ matrix_mautrix_signal_container_http_host_bind_port }}:29328 \
+			{% endif %}
+			-v {{ matrix_mautrix_signal_daemon_path }}:/signald:z \
+			-v {{ matrix_mautrix_signal_config_path }}:/data:z \
+			{% for arg in matrix_mautrix_signal_container_extra_arguments %}
+			{{ arg }} \
+			{% endfor %}
+			{{ matrix_mautrix_signal_docker_image }} \
+			python3 -m mautrix_signal -c /data/config.yaml
+
+ExecStop=-{{ matrix_host_command_docker }} kill matrix-mautrix-signal
+ExecStop=-{{ matrix_host_command_docker }} rm matrix-mautrix-signal
+
+Restart=always
+RestartSec=30
+SyslogIdentifier=matrix-mautrix-signal
+
+[Install]
+WantedBy=multi-user.target
diff --git a/setup.yml b/setup.yml
index 68c4e4ee..67639f3e 100755
--- a/setup.yml
+++ b/setup.yml
@@ -14,6 +14,7 @@
     - matrix-bridge-appservice-irc
     - matrix-bridge-mautrix-facebook
     - matrix-bridge-mautrix-hangouts
+    - matrix-bridge-mautrix-signal
     - matrix-bridge-mautrix-telegram
     - matrix-bridge-mautrix-whatsapp
     - matrix-bridge-mx-puppet-discord

From 7a54e1109066ea5960c772de34ae7461be7a304d Mon Sep 17 00:00:00 2001
From: Sabine <sabinbox@laszakovits.net>
Date: Sat, 24 Oct 2020 12:15:03 +0200
Subject: [PATCH 002/179] started moving settings to group_vars

---
 group_vars/matrix_servers                     | 25 +++++++++++++++++++
 .../defaults/main.yml                         |  4 +--
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 00cdbdfb..00936700 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -241,7 +241,32 @@ matrix_mautrix_hangouts_login_shared_secret: "{{ matrix_synapse_ext_password_pro
 ######################################################################
 
 
+######################################################################
+#
+# matrix-bridge-mautrix-signal
+#
+######################################################################
+
+matrix_mautrix_signal_enabled: false
+
+matrix_mautrix_signal_systemd_required_services_list: |
+  {{
+    ['docker.service']
+    +
+    (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+  }}
+
+matrix_mautrix_signal_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'si.as.token') | to_uuid }}"
+
+matrix_mautrix_signal_homeserver_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'si.hs.token') | to_uuid }}"
+
+matrix_mautrix_signal_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
+######################################################################
+#
+# /matrix-bridge-mautrix-signal
+#
+######################################################################
 
 
 ######################################################################
diff --git a/roles/matrix-bridge-mautrix-signal/defaults/main.yml b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
index d86bdb59..d034cab1 100644
--- a/roles/matrix-bridge-mautrix-signal/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
@@ -1,7 +1,7 @@
 # mautrix-telegram is a Matrix <-> Telegram bridge
 # See: https://github.com/tulir/mautrix-telegram
 
-matrix_mautrix_signal_enabled: false
+matrix_mautrix_signal_enabled: true
 
 # See: https://mau.dev/tulir/mautrix-signal/container_registry
 matrix_mautrix_signal_docker_image: "dock.mau.dev/tulir/mautrix-signal:latest"
@@ -41,7 +41,7 @@ matrix_mautrix_signal_container_extra_arguments: []
 # List of systemd services that matrix-mautrix-signal.service depends on.
 matrix_mautrix_signal_systemd_required_services_list: ['docker.service', 'matrix-mautrix-signal-daemon.service', 'matrix-mautrix-signal-db.service']
 
-# List of systemd services that matrix-mautrix-telegram.service wants
+# List of systemd services that matrix-mautrix-signal.service wants
 matrix_mautrix_signal_systemd_wanted_services_list: []
 
 matrix_mautrix_signal_appservice_token: ''

From d6940d9f106429629c6f4b758ebbbd122f6026a4 Mon Sep 17 00:00:00 2001
From: Sabine Laszakovits <sabinbox@laszakovits.net>
Date: Sun, 25 Oct 2020 18:44:19 +0100
Subject: [PATCH 003/179] added docs

---
 ...figuring-playbook-bridge-mautrix-signal.md | 45 +++++++++++++++++++
 docs/configuring-playbook.md                  |  2 +
 2 files changed, 47 insertions(+)
 create mode 100644 docs/configuring-playbook-bridge-mautrix-signal.md

diff --git a/docs/configuring-playbook-bridge-mautrix-signal.md b/docs/configuring-playbook-bridge-mautrix-signal.md
new file mode 100644
index 00000000..7bdd9924
--- /dev/null
+++ b/docs/configuring-playbook-bridge-mautrix-signal.md
@@ -0,0 +1,45 @@
+# Setting up Mautrix Signal (optional)
+
+The playbook can install and configure [mautrix-signal](https://github.com/tulir/mautrix-signal) for you.
+
+See the project's [documentation](https://github.com/tulir/mautrix-signal/wiki) to learn what it does and why it might be useful to you.
+
+Use the following playbook configuration:
+
+```yaml
+matrix_mautrix_signal_enabled: true
+```
+
+
+## Set up Double Puppeting
+
+If you'd like to use [Double Puppeting](https://github.com/tulir/mautrix-whatsapp/wiki/Authentication#replacing-whatsapp-accounts-matrix-puppet-with-matrix-account) (hint: you most likely do), you have 2 ways of going about it.
+
+### Method 1: automatically, by enabling Shared Secret Auth
+
+The bridge will automatically perform Double Puppeting if you enable [Shared Secret Auth](configuring-playbook-shared-secret-auth.md) for this playbook.
+
+This is the recommended way of setting up Double Puppeting, as it's easier to accomplish, works for all your users automatically, and has less of a chance of breaking in the future.
+
+### Method 2: manually, by asking each user to provide a working access token
+
+**Note**: This method for enabling Double Puppeting can be configured only after you've already set up bridging (see [Usage](#usage)).
+
+When using this method, **each user** that wishes to enable Double Puppeting needs to follow the following steps:
+
+- retrieve a Matrix access token for yourself. You can use the following command:
+
+```
+curl \
+--data '{"identifier": {"type": "m.id.user", "user": "YOUR_MATRIX_USERNAME" }, "password": "YOUR_MATRIX_PASSWORD", "type": "m.login.password", "device_id": "Mautrix-Signal", "initial_device_display_name": "Mautrix-Signal"}' \
+https://matrix.DOMAIN/_matrix/client/r0/login
+```
+
+- send the access token to the bot. Example: `login-matrix MATRIX_ACCESS_TOKEN_HERE`
+
+- make sure you don't log out the `Mautrix-Signal` device some time in the future, as that would break the Double Puppeting feature
+
+
+## Usage
+
+You then need to start a chat with `@signalbot:YOUR_DOMAIN` (where `YOUR_DOMAIN` is your base domain, not the `matrix.` domain).
diff --git a/docs/configuring-playbook.md b/docs/configuring-playbook.md
index b174637e..ad45eac7 100644
--- a/docs/configuring-playbook.md
+++ b/docs/configuring-playbook.md
@@ -93,6 +93,8 @@ When you're done with all the configuration you'd like to do, continue with [Ins
 
 - [Setting up Mautrix Hangouts bridging](configuring-playbook-bridge-mautrix-hangouts.md) (optional)
 
+- [Setting up Mautrix Signal bridging](configuring-playbook-bridge-mautrix-signal.md) (optional)
+
 - [Setting up Appservice IRC bridging](configuring-playbook-bridge-appservice-irc.md) (optional)
 
 - [Setting up Appservice Discord bridging](configuring-playbook-bridge-appservice-discord.md) (optional)

From ff1d7921a0f5a64b3f418d13d096112c100990f4 Mon Sep 17 00:00:00 2001
From: Sabine Laszakovits <sabinbox@laszakovits.net>
Date: Sun, 25 Oct 2020 20:16:24 +0100
Subject: [PATCH 004/179] moved registration.yaml contents to template

---
 .../defaults/main.yml                             | 15 +--------------
 .../templates/registration.yaml.j2                | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 14 deletions(-)
 create mode 100644 roles/matrix-bridge-mautrix-signal/templates/registration.yaml.j2

diff --git a/roles/matrix-bridge-mautrix-signal/defaults/main.yml b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
index d034cab1..94428d4f 100644
--- a/roles/matrix-bridge-mautrix-signal/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
@@ -84,19 +84,6 @@ matrix_mautrix_signal_configuration_extension: "{{ matrix_mautrix_signal_configu
 # You most likely don't need to touch this variable. Instead, see `matrix_mautrix_telegram_configuration_yaml`.
 matrix_mautrix_signal_configuration: "{{ matrix_mautrix_signal_configuration_yaml|from_yaml|combine(matrix_mautrix_signal_configuration_extension, recursive=True) }}"
 
-matrix_mautrix_signal_registration_yaml: |
-  id: signal
-  as_token: "{{ matrix_mautrix_signal_appservice_token }}"
-  hs_token: "{{ matrix_mautrix_signal_homeserver_token }}"
-  namespaces:
-      users:
-      - exclusive: true
-        regex: '^@signal_.+:{{ matrix_mautrix_signal_homeserver_domain|regex_escape }}$'
-      aliases:
-      - exclusive: true
-        regex: '^#signal_.+:{{ matrix_mautrix_signal_homeserver_domain|regex_escape }}$'
-  url: {{ matrix_mautrix_signal_appservice_address }}
-  sender_localpart: signalbot
-  rate_limited: false
+matrix_mautrix_signal_registration_yaml: "{{ lookup('template', 'templates/registration.yaml.j2') }}"
 
 matrix_mautrix_signal_registration: "{{ matrix_mautrix_signal_registration_yaml|from_yaml }}"
diff --git a/roles/matrix-bridge-mautrix-signal/templates/registration.yaml.j2 b/roles/matrix-bridge-mautrix-signal/templates/registration.yaml.j2
new file mode 100644
index 00000000..45cc5a0f
--- /dev/null
+++ b/roles/matrix-bridge-mautrix-signal/templates/registration.yaml.j2
@@ -0,0 +1,14 @@
+#jinja2: lstrip_blocks: "True"
+id: signal
+as_token: "{{ matrix_mautrix_signal_appservice_token }}"
+hs_token: "{{ matrix_mautrix_signal_homeserver_token }}"
+namespaces:
+    users:
+    - exclusive: true
+      regex: '^@signal_.+:{{ matrix_mautrix_signal_homeserver_domain|regex_escape }}$'
+    aliases:
+    - exclusive: true
+      regex: '^#signal_.+:{{ matrix_mautrix_signal_homeserver_domain|regex_escape }}$'
+url: {{ matrix_mautrix_signal_appservice_address }}
+sender_localpart: signalbot
+rate_limited: false

From 9b890e9ced5efc95de2b112e12aac5989a22aeb7 Mon Sep 17 00:00:00 2001
From: Sabine Laszakovits <sabinbox@laszakovits.net>
Date: Sun, 25 Oct 2020 20:45:24 +0100
Subject: [PATCH 005/179] moved some settings from role to group_vars

---
 group_vars/matrix_servers                                  | 7 +++++++
 roles/matrix-bridge-mautrix-signal/defaults/main.yml       | 4 ++--
 .../matrix-bridge-mautrix-signal/tasks/validate_config.yml | 2 ++
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 00936700..c42054d7 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -257,11 +257,18 @@ matrix_mautrix_signal_systemd_required_services_list: |
   }}
 
 matrix_mautrix_signal_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'si.as.token') | to_uuid }}"
+matrix_mautrix_signal_homeserver_domain: '{{ matrix_domain }}'
+
+matrix_mautrix_signal_homeserver_address: "{{ 'http://matrix-synapse:8008' if matrix_synapse_enabled else '' }}"
 
 matrix_mautrix_signal_homeserver_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'si.hs.token') | to_uuid }}"
 
 matrix_mautrix_signal_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
+matrix_mautrix_signal_db_user: 'signal-db-user'
+matrix_mautrix_signal_db_password: 'signal-db-password'
+matrix_mautrix_signal_db_database: 'signal-db'
+
 ######################################################################
 #
 # /matrix-bridge-mautrix-signal
diff --git a/roles/matrix-bridge-mautrix-signal/defaults/main.yml b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
index 94428d4f..0e970591 100644
--- a/roles/matrix-bridge-mautrix-signal/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
@@ -25,8 +25,8 @@ matrix_mautrix_signal_daemon_path: "{{ matrix_mautrix_signal_base_path }}/signal
 # # Example: /741a0483-ba17-4682-9900-30bd7269f1cc
 # matrix_mautrix_telegram_public_endpoint: ''
 
-matrix_mautrix_signal_homeserver_address: 'http://matrix-synapse:8008'
-matrix_mautrix_signal_homeserver_domain: '{{ matrix_domain }}'
+matrix_mautrix_signal_homeserver_address: ''
+matrix_mautrix_signal_homeserver_domain: ''
 matrix_mautrix_signal_appservice_address: 'http://matrix-mautrix-signal:29328'
 # matrix_mautrix_telegram_appservice_public_external: 'https://{{ matrix_server_fqn_matrix }}{{ matrix_mautrix_telegram_public_endpoint }}'
 
diff --git a/roles/matrix-bridge-mautrix-signal/tasks/validate_config.yml b/roles/matrix-bridge-mautrix-signal/tasks/validate_config.yml
index a11e192d..0f3a8e35 100644
--- a/roles/matrix-bridge-mautrix-signal/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mautrix-signal/tasks/validate_config.yml
@@ -7,6 +7,8 @@
   when: "vars[item] == ''"
   with_items:
     - "matrix_mautrix_signal_appservice_token"
+    - "matrix_mautrix_signal_homeserver_domain"
+    - "matrix_mautrix_signal_homeserver_address"
     - "matrix_mautrix_signal_homeserver_token"
     - "matrix_mautrix_signal_db_user"
     - "matrix_mautrix_signal_db_password"

From 8ebc39dea0f1e7ca587c2f781279e3b16527167a Mon Sep 17 00:00:00 2001
From: Sabine Laszakovits <sabinbox@laszakovits.net>
Date: Sun, 25 Oct 2020 20:46:32 +0100
Subject: [PATCH 006/179] fixed dependencies of bridge service (not ideal, but
 correct)

---
 group_vars/matrix_servers                            | 3 +++
 roles/matrix-bridge-mautrix-signal/defaults/main.yml | 7 ++++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index c42054d7..735581a2 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -249,11 +249,14 @@ matrix_mautrix_hangouts_login_shared_secret: "{{ matrix_synapse_ext_password_pro
 
 matrix_mautrix_signal_enabled: false
 
+# TODO: unclear whether to put this into the role or keep it here
 matrix_mautrix_signal_systemd_required_services_list: |
   {{
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    ['matrix-mautrix-signal-daemon.service', 'matrix-mautrix-signal-db.service']
   }}
 
 matrix_mautrix_signal_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'si.as.token') | to_uuid }}"
diff --git a/roles/matrix-bridge-mautrix-signal/defaults/main.yml b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
index 0e970591..85ae6992 100644
--- a/roles/matrix-bridge-mautrix-signal/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
@@ -39,7 +39,12 @@ matrix_mautrix_signal_container_http_host_bind_port: ''
 matrix_mautrix_signal_container_extra_arguments: []
 
 # List of systemd services that matrix-mautrix-signal.service depends on.
-matrix_mautrix_signal_systemd_required_services_list: ['docker.service', 'matrix-mautrix-signal-daemon.service', 'matrix-mautrix-signal-db.service']
+# TODO: unclear whether to put this into group_vars or keep it here
+matrix_mautrix_signal_systemd_required_services_list:
+  - 'docker.service'
+  - "{{ 'matrix-synapse.service' if matrix_synapse_enabled else [] }}"
+  - 'matrix-mautrix-signal-daemon.service'
+  - 'matrix-mautrix-signal-db.service'
 
 # List of systemd services that matrix-mautrix-signal.service wants
 matrix_mautrix_signal_systemd_wanted_services_list: []

From c36e1355cf58d3f73d3891060cb720db9e5b9230 Mon Sep 17 00:00:00 2001
From: Sabine Laszakovits <sabinbox@laszakovits.net>
Date: Sun, 25 Oct 2020 20:46:57 +0100
Subject: [PATCH 007/179] cleanup

---
 group_vars/matrix_servers                                    | 3 ++-
 roles/matrix-bridge-mautrix-signal/defaults/main.yml         | 5 ++---
 roles/matrix-bridge-mautrix-signal/tasks/validate_config.yml | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 735581a2..5252f7ad 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -259,13 +259,14 @@ matrix_mautrix_signal_systemd_required_services_list: |
     ['matrix-mautrix-signal-daemon.service', 'matrix-mautrix-signal-db.service']
   }}
 
-matrix_mautrix_signal_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'si.as.token') | to_uuid }}"
 matrix_mautrix_signal_homeserver_domain: '{{ matrix_domain }}'
 
 matrix_mautrix_signal_homeserver_address: "{{ 'http://matrix-synapse:8008' if matrix_synapse_enabled else '' }}"
 
 matrix_mautrix_signal_homeserver_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'si.hs.token') | to_uuid }}"
 
+matrix_mautrix_signal_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'si.as.token') | to_uuid }}"
+
 matrix_mautrix_signal_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
 matrix_mautrix_signal_db_user: 'signal-db-user'
diff --git a/roles/matrix-bridge-mautrix-signal/defaults/main.yml b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
index 85ae6992..96778940 100644
--- a/roles/matrix-bridge-mautrix-signal/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
@@ -7,12 +7,11 @@ matrix_mautrix_signal_enabled: true
 matrix_mautrix_signal_docker_image: "dock.mau.dev/tulir/mautrix-signal:latest"
 matrix_mautrix_signal_docker_image_force_pull: "{{ matrix_mautrix_signal_docker_image.endswith(':latest') }}"
 
-matrix_mautrix_signal_base_path: "{{ matrix_base_data_path }}/mautrix-signal"
-matrix_mautrix_signal_config_path: "{{ matrix_mautrix_signal_base_path }}/bridge"
-
 matrix_mautrix_signal_daemon_docker_image: "dock.mau.dev/maunium/signald:latest"
 matrix_mautrix_signal_daemon_docker_image_force_pull: "{{ matrix_mautrix_signal_daemon_docker_image.endswith(':latest') }}"
 
+matrix_mautrix_signal_base_path: "{{ matrix_base_data_path }}/mautrix-signal"
+matrix_mautrix_signal_config_path: "{{ matrix_mautrix_signal_base_path }}/bridge"
 matrix_mautrix_signal_daemon_path: "{{ matrix_mautrix_signal_base_path }}/signald"
 
 # # Get your own API keys at https://my.telegram.org/apps
diff --git a/roles/matrix-bridge-mautrix-signal/tasks/validate_config.yml b/roles/matrix-bridge-mautrix-signal/tasks/validate_config.yml
index 0f3a8e35..6663c641 100644
--- a/roles/matrix-bridge-mautrix-signal/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mautrix-signal/tasks/validate_config.yml
@@ -6,10 +6,10 @@
       You need to define a required configuration setting (`{{ item }}`).
   when: "vars[item] == ''"
   with_items:
-    - "matrix_mautrix_signal_appservice_token"
     - "matrix_mautrix_signal_homeserver_domain"
     - "matrix_mautrix_signal_homeserver_address"
     - "matrix_mautrix_signal_homeserver_token"
+    - "matrix_mautrix_signal_appservice_token"
     - "matrix_mautrix_signal_db_user"
     - "matrix_mautrix_signal_db_password"
     - "matrix_mautrix_signal_db_database"

From e59aa07b8660d57e6d34e1cb730524ea7fc27dd9 Mon Sep 17 00:00:00 2001
From: Sabine Laszakovits <sabinbox@laszakovits.net>
Date: Sun, 25 Oct 2020 20:47:10 +0100
Subject: [PATCH 008/179] more cleanup

---
 roles/matrix-bridge-mautrix-signal/defaults/main.yml | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/roles/matrix-bridge-mautrix-signal/defaults/main.yml b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
index 96778940..2d61d207 100644
--- a/roles/matrix-bridge-mautrix-signal/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
@@ -14,16 +14,6 @@ matrix_mautrix_signal_base_path: "{{ matrix_base_data_path }}/mautrix-signal"
 matrix_mautrix_signal_config_path: "{{ matrix_mautrix_signal_base_path }}/bridge"
 matrix_mautrix_signal_daemon_path: "{{ matrix_mautrix_signal_base_path }}/signald"
 
-# # Get your own API keys at https://my.telegram.org/apps
-# matrix_mautrix_telegram_api_id: ''
-# matrix_mautrix_telegram_api_hash: ''
-# matrix_mautrix_telegram_bot_token: disabled
-
-# # Mautrix telegram public endpoint to log in to telegram
-# # Use an uuid so it's not easily discoverable.
-# # Example: /741a0483-ba17-4682-9900-30bd7269f1cc
-# matrix_mautrix_telegram_public_endpoint: ''
-
 matrix_mautrix_signal_homeserver_address: ''
 matrix_mautrix_signal_homeserver_domain: ''
 matrix_mautrix_signal_appservice_address: 'http://matrix-mautrix-signal:29328'

From 2211e678f39af4f6e0931bba834d090c5ce0a0f6 Mon Sep 17 00:00:00 2001
From: Sabine Laszakovits <sabinbox@laszakovits.net>
Date: Sun, 25 Oct 2020 21:01:25 +0100
Subject: [PATCH 009/179] fixed comments that were copied over from
 mautrix-telegram role

---
 .../defaults/main.yml                         | 15 +++---
 .../tasks/init.yml                            | 47 -------------------
 .../tasks/setup_install.yml                   | 17 -------
 3 files changed, 7 insertions(+), 72 deletions(-)

diff --git a/roles/matrix-bridge-mautrix-signal/defaults/main.yml b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
index 2d61d207..c15f1ed1 100644
--- a/roles/matrix-bridge-mautrix-signal/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
@@ -1,5 +1,5 @@
-# mautrix-telegram is a Matrix <-> Telegram bridge
-# See: https://github.com/tulir/mautrix-telegram
+# mautrix-signal is a Matrix <-> Signal bridge
+# See: https://github.com/tulir/mautrix-signal
 
 matrix_mautrix_signal_enabled: true
 
@@ -17,9 +17,8 @@ matrix_mautrix_signal_daemon_path: "{{ matrix_mautrix_signal_base_path }}/signal
 matrix_mautrix_signal_homeserver_address: ''
 matrix_mautrix_signal_homeserver_domain: ''
 matrix_mautrix_signal_appservice_address: 'http://matrix-mautrix-signal:29328'
-# matrix_mautrix_telegram_appservice_public_external: 'https://{{ matrix_server_fqn_matrix }}{{ matrix_mautrix_telegram_public_endpoint }}'
 
-# Controls whether the matrix-mautrix-telegram container exposes its HTTP port (tcp/8080 in the container).
+# Controls whether the matrix-mautrix-signal container exposes its port (tcp/29328 in the container).
 #
 # Takes an "<ip>:<port>" or "<port>" value (e.g. "127.0.0.1:9006"), or empty string to not expose.
 matrix_mautrix_signal_container_http_host_bind_port: ''
@@ -59,23 +58,23 @@ matrix_mautrix_signal_login_shared_secret: ''
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
 #
-# For a more advanced customization, you can extend the default (see `matrix_mautrix_telegram_configuration_extension_yaml`)
+# For a more advanced customization, you can extend the default (see `matrix_mautrix_signal_configuration_extension_yaml`)
 # or completely replace this variable with your own template.
 matrix_mautrix_signal_configuration_yaml: "{{ lookup('template', 'templates/config.yaml.j2') }}"
 
 matrix_mautrix_signal_configuration_extension_yaml: |
   # Your custom YAML configuration goes here.
-  # This configuration extends the default starting configuration (`matrix_mautrix_telegram_configuration_yaml`).
+  # This configuration extends the default starting configuration (`matrix_mautrix_signal_configuration_yaml`).
   #
   # You can override individual variables from the default configuration, or introduce new ones.
   #
   # If you need something more special, you can take full control by
-  # completely redefining `matrix_mautrix_telegram_configuration_yaml`.
+  # completely redefining `matrix_mautrix_signal_configuration_yaml`.
 
 matrix_mautrix_signal_configuration_extension: "{{ matrix_mautrix_signal_configuration_extension_yaml|from_yaml if matrix_mautrix_signal_configuration_extension_yaml|from_yaml is mapping else {} }}"
 
 # Holds the final configuration (a combination of the default and its extension).
-# You most likely don't need to touch this variable. Instead, see `matrix_mautrix_telegram_configuration_yaml`.
+# You most likely don't need to touch this variable. Instead, see `matrix_mautrix_signal_configuration_yaml`.
 matrix_mautrix_signal_configuration: "{{ matrix_mautrix_signal_configuration_yaml|from_yaml|combine(matrix_mautrix_signal_configuration_extension, recursive=True) }}"
 
 matrix_mautrix_signal_registration_yaml: "{{ lookup('template', 'templates/registration.yaml.j2') }}"
diff --git a/roles/matrix-bridge-mautrix-signal/tasks/init.yml b/roles/matrix-bridge-mautrix-signal/tasks/init.yml
index 8fdc9834..c2b37841 100644
--- a/roles/matrix-bridge-mautrix-signal/tasks/init.yml
+++ b/roles/matrix-bridge-mautrix-signal/tasks/init.yml
@@ -14,50 +14,3 @@
       +
       {{ ["/matrix-mautrix-signal-registration.yaml"] }}
   when: matrix_mautrix_signal_enabled|bool
-
-# - block:
-#   - name: Fail if matrix-nginx-proxy role already executed
-#     fail:
-#       msg: >-
-#         Trying to append Mautrix Signal's reverse-proxying configuration to matrix-nginx-proxy,
-#         but it's pointless since the matrix-nginx-proxy role had already executed.
-#         To fix this, please change the order of roles in your plabook,
-#         so that the matrix-nginx-proxy role would run after the matrix-bridge-mautrix-signal role.
-#     when: matrix_nginx_proxy_role_executed|default(False)|bool
-
-  # - name: Generate Mautrix Signal proxying configuration for matrix-nginx-proxy
-  #   set_fact:
-  #     matrix_mautrix_telegram_matrix_nginx_proxy_configuration: |
-  #       location {{ matrix_mautrix_telegram_public_endpoint }} {
-  #       {% if matrix_nginx_proxy_enabled|default(False) %}
-  #       	{# Use the embedded DNS resolver in Docker containers to discover the service #}
-  #       	resolver 127.0.0.11 valid=5s;
-  #       	set $backend "matrix-mautrix-telegram:8080";
-  #       	proxy_pass http://$backend;
-  #       {% else %}
-  #       	{# Generic configuration for use outside of our container setup #}
-  #       	proxy_pass http://127.0.0.1:9006;
-  #       {% endif %}
-  #       }
-
-  # - name: Register Mautrix Telegram proxying configuration with matrix-nginx-proxy
-  #   set_fact:
-  #     matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks: |
-  #       {{
-  #         matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks|default([])
-  #         +
-  #         [matrix_mautrix_telegram_matrix_nginx_proxy_configuration]
-  #       }}
-  # tags:
-  #  - always
-  # when: matrix_mautrix_telegram_enabled|bool
-
-# - name: Warn about reverse-proxying if matrix-nginx-proxy not used
-#   debug:
-#     msg: >-
-#       NOTE: You've enabled the Mautrix Telegram bridge but are not using the matrix-nginx-proxy
-#       reverse proxy.
-#       Please make sure that you're proxying the `{{ matrix_mautrix_telegram_public_endpoint }}`
-#       URL endpoint to the matrix-mautrix-telegram container.
-#       You can expose the container's port using the `matrix_mautrix_telegram_container_http_host_bind_port` variable.
-#   when: "matrix_mautrix_telegram_enabled|bool and matrix_nginx_proxy_enabled is not defined"
diff --git a/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml
index cacb4c6c..16b68d25 100644
--- a/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml
@@ -42,23 +42,6 @@
     - "{{ matrix_mautrix_signal_daemon_path }}"
     - "{{ matrix_mautrix_signal_db_storage_path }}"
 
-# - name: Check if an old database file already exists
-#   stat:
-#     path: "{{ matrix_mautrix_telegram_base_path }}/mautrix-telegram.db"
-#   register: matrix_mautrix_telegram_stat_database
-#
-# - name: (Data relocation) Ensure matrix-mautrix-telegram.service is stopped
-#   service:
-#     name: matrix-mautrix-telegram
-#     state: stopped
-#     daemon_reload: yes
-#   failed_when: false
-#   when: "matrix_mautrix_telegram_stat_database.stat.exists"
-#
-# - name: (Data relocation) Move mautrix-telegram database file to ./data directory
-#   command: "mv {{ matrix_mautrix_telegram_base_path }}/mautrix-telegram.db {{ matrix_mautrix_telegram_data_path }}/mautrix-telegram.db"
-#   when: "matrix_mautrix_telegram_stat_database.stat.exists"
-
 - name: Ensure mautrix-signal config.yaml installed
   copy:
     content: "{{ matrix_mautrix_signal_configuration|to_nice_yaml }}"

From dd50ee19ab80daef38ae59b2b4f68bd679a4cee7 Mon Sep 17 00:00:00 2001
From: Sabine Laszakovits <sabinbox@laszakovits.net>
Date: Sun, 25 Oct 2020 21:42:40 +0100
Subject: [PATCH 010/179] fixed bridge permissions

---
 docs/configuring-playbook-bridge-mautrix-signal.md | 14 ++++++++++++++
 .../matrix-bridge-mautrix-signal/defaults/main.yml |  9 +++++++++
 .../templates/config.yaml.j2                       |  4 +---
 3 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/docs/configuring-playbook-bridge-mautrix-signal.md b/docs/configuring-playbook-bridge-mautrix-signal.md
index 7bdd9924..066dbaff 100644
--- a/docs/configuring-playbook-bridge-mautrix-signal.md
+++ b/docs/configuring-playbook-bridge-mautrix-signal.md
@@ -10,6 +10,20 @@ Use the following playbook configuration:
 matrix_mautrix_signal_enabled: true
 ```
 
+To specify which users have access to the bridge, use the variable `matrix_mautrix_signal_configuration_permissions`.
+Refer to the documentation for
+```yaml
+bridge:
+  permissions:
+```
+in [the example config in mautrix-signal](https://github.com/tulir/mautrix-signal/blob/master/mautrix_signal/example-config.yaml).
+For instance, use
+```yaml
+matrix_mautrix_signal_configuration_permissions:
+  "YOUR_DOMAIN": user
+```
+to allow all users registered to `YOUR_DOMAIN` access to the bridge (where `YOUR_DOMAIN` is your base domain, not the `matrix.` domain).
+
 
 ## Set up Double Puppeting
 
diff --git a/roles/matrix-bridge-mautrix-signal/defaults/main.yml b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
index c15f1ed1..e6c5fe05 100644
--- a/roles/matrix-bridge-mautrix-signal/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
@@ -62,6 +62,15 @@ matrix_mautrix_signal_login_shared_secret: ''
 # or completely replace this variable with your own template.
 matrix_mautrix_signal_configuration_yaml: "{{ lookup('template', 'templates/config.yaml.j2') }}"
 
+# Permitted values:
+#       user - Use the bridge with puppeting.
+#      admin - Use and administrate the bridge.
+# Permitted keys:
+#        * - All Matrix users
+#   domain - All users on that homeserver
+#     mxid - Specific user
+matrix_mautrix_signal_configuration_permissions: []
+
 matrix_mautrix_signal_configuration_extension_yaml: |
   # Your custom YAML configuration goes here.
   # This configuration extends the default starting configuration (`matrix_mautrix_signal_configuration_yaml`).
diff --git a/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2
index 6865d3c1..ec2a4764 100644
--- a/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2
@@ -170,9 +170,7 @@ bridge:
     #        * - All Matrix users
     #   domain - All users on that homeserver
     #     mxid - Specific user
-    permissions:
-        "1m.at": "user"
-        "@admin:1m.at": "admin"
+    permissions: {{ matrix_mautrix_signal_configuration_permissions }} 
 
 
 # Python logging configuration.

From 468cc3946546a7d939be5f25201eb5829c0f3745 Mon Sep 17 00:00:00 2001
From: Sabine <sabinbox@laszakovits.net>
Date: Wed, 28 Oct 2020 21:56:13 +0100
Subject: [PATCH 011/179] added a workaround for postgres's issue with initdb

---
 .../templates/systemd/matrix-mautrix-signal-db.service.j2        | 1 +
 1 file changed, 1 insertion(+)

diff --git a/roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal-db.service.j2 b/roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal-db.service.j2
index 653b67b8..b4baed4c 100644
--- a/roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal-db.service.j2
+++ b/roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal-db.service.j2
@@ -19,6 +19,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name {{ matrix_mautrix_sig
 			--env POSTGRES_PASSWORD={{ matrix_mautrix_signal_db_password }} \
 			--env POSTGRES_DB={{ matrix_mautrix_signal_db_database }} \
 			-v {{ matrix_mautrix_signal_db_storage_path }}:/var/lib/postgresql/data:z \
+			-v /etc/passwd:/etc/passwd:ro \
 			{{ matrix_mautrix_signal_db_docker_image }}
 
 ExecStop=-{{ matrix_host_command_docker }} kill {{ matrix_mautrix_signal_db_host }}

From aabefe21f46a353258c784d1209e7dea4f2e5f05 Mon Sep 17 00:00:00 2001
From: Sabine <sabinbox@laszakovits.net>
Date: Wed, 28 Oct 2020 21:58:59 +0100
Subject: [PATCH 012/179] fixed yaml

---
 roles/matrix-bridge-mautrix-signal/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-bridge-mautrix-signal/defaults/main.yml b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
index e6c5fe05..5d6bb7d3 100644
--- a/roles/matrix-bridge-mautrix-signal/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
@@ -69,7 +69,7 @@ matrix_mautrix_signal_configuration_yaml: "{{ lookup('template', 'templates/conf
 #        * - All Matrix users
 #   domain - All users on that homeserver
 #     mxid - Specific user
-matrix_mautrix_signal_configuration_permissions: []
+matrix_mautrix_signal_configuration_permissions: {}
 
 matrix_mautrix_signal_configuration_extension_yaml: |
   # Your custom YAML configuration goes here.

From cea2faa1b34362b08d3224276ac6b9e2a81070ad Mon Sep 17 00:00:00 2001
From: Sabine <sabinbox@laszakovits.net>
Date: Wed, 28 Oct 2020 21:59:36 +0100
Subject: [PATCH 013/179] added docs for bridge permissions

---
 docs/configuring-playbook-bridge-mautrix-signal.md | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/docs/configuring-playbook-bridge-mautrix-signal.md b/docs/configuring-playbook-bridge-mautrix-signal.md
index 066dbaff..a5410665 100644
--- a/docs/configuring-playbook-bridge-mautrix-signal.md
+++ b/docs/configuring-playbook-bridge-mautrix-signal.md
@@ -19,10 +19,13 @@ bridge:
 in [the example config in mautrix-signal](https://github.com/tulir/mautrix-signal/blob/master/mautrix_signal/example-config.yaml).
 For instance, use
 ```yaml
-matrix_mautrix_signal_configuration_permissions:
-  "YOUR_DOMAIN": user
+matrix_mautrix_signal_configuration_permissions: |
+  {
+    '{{ matrix_domain }}': 'user'
+  }
 ```
-to allow all users registered to `YOUR_DOMAIN` access to the bridge (where `YOUR_DOMAIN` is your base domain, not the `matrix.` domain).
+to allow all users registered to the current host's matrix domain access to the bridge, or hard-code whatever you like.
+(See [this issue](https://github.com/ansible/ansible/issues/17324#issuecomment-449642731) on how to use variable names as dictionary keys.)
 
 
 ## Set up Double Puppeting

From fe7bed5df37dbe8bf575143dff095dde4286428f Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 10 Nov 2020 23:10:59 +0200
Subject: [PATCH 014/179] Upgrade appservice-discord

---
 .../defaults/main.yml                              |  8 +++++---
 .../tasks/validate_config.yml                      |  1 +
 .../templates/config.yaml.j2                       | 14 +++++++++-----
 3 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml
index 97ad8e89..392f3028 100644
--- a/roles/matrix-bridge-appservice-discord/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml
@@ -3,7 +3,7 @@
 
 matrix_appservice_discord_enabled: true
 
-matrix_appservice_discord_docker_image: "halfshot/matrix-appservice-discord:latest"
+matrix_appservice_discord_docker_image: "halfshot/matrix-appservice-discord:v1.0.0-rc3"
 matrix_appservice_discord_docker_image_force_pull: "{{ matrix_appservice_discord_docker_image.endswith(':latest') }}"
 
 matrix_appservice_discord_base_path: "{{ matrix_base_data_path }}/appservice-discord"
@@ -17,6 +17,8 @@ matrix_appservice_discord_bot_token: ''
 matrix_appservice_discord_appservice_token: ''
 matrix_appservice_discord_homeserver_token: ''
 
+matrix_appservice_discord_homeserver_domain: "{{ matrix_domain }}"
+
 # Controls whether the matrix-appservice-discord container exposes its HTTP port (tcp/9005 in the container).
 #
 # Takes an "<ip>:<port>" or "<port>" value (e.g. "127.0.0.1:9005"), or empty string to not expose.
@@ -62,10 +64,10 @@ matrix_appservice_discord_registration_yaml: |
   namespaces:
     users:
     - exclusive: true
-      regex: '^@_discord_.*'
+      regex: '@_discord_.*:{{ matrix_appservice_discord_homeserver_domain|regex_escape }}'
     aliases:
     - exclusive: true
-      regex: '^#_discord_.*'
+      regex: '#_discord_.*:{{ matrix_appservice_discord_homeserver_domain|regex_escape }}'
   url: {{ matrix_appservice_discord_appservice_url }}
   sender_localpart: _discord_bot
   rate_limited: false
diff --git a/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml b/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml
index 46612ba5..b0713a43 100644
--- a/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml
+++ b/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml
@@ -10,6 +10,7 @@
     - "matrix_appservice_discord_bot_token"
     - "matrix_appservice_discord_appservice_token"
     - "matrix_appservice_discord_homeserver_token"
+    - "matrix_appservice_discord_homeserver_domain"
 
 - name: (Deprecation) Catch and report renamed appservice-discord variables
   fail:
diff --git a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2
index e91c60de..0c2aab41 100644
--- a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2
+++ b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2
@@ -26,6 +26,10 @@ bridge:
   disableReadReceipts: false
   # Disable Join Leave echos from matrix
   disableJoinLeaveNotifications: false
+  # Disable Invite echos from matrix
+  disableInviteNotifications: false
+  # Auto-determine the language of code blocks (this can be CPU-intensive)
+  determineCodeLanguage: false
 # Authentication configuration for the discord bot.
 auth:
   clientID: {{ matrix_appservice_discord_client_id|string|to_json }}
@@ -45,8 +49,6 @@ logging:
   #     enable:
   #       - "DiscordBot"
 database:
-  userStorePath: "/data/user-store.db"
-  roomStorePath: "/data/room-store.db"
   # You may either use SQLite or Postgresql for the bridge database, which contains
   # important mappings for events and user puppeting configurations.
   # Use the filename option for SQLite, or connString for Postgresql.
@@ -82,10 +84,12 @@ channel:
 limits:
     # Delay in milliseconds between discord users joining a room.
     roomGhostJoinDelay: 6000
-    # Delay in milliseconds before sending messages to discord to avoid echos.
-    # (Copies of a sent message may arrive from discord before we've
+    # Lock timeout in milliseconds before sending messages to discord to avoid
+    # echos. Default is rather high as the lock will most likely time out
+    # before anyways.
+    # echos = (Copies of a sent message may arrive from discord before we've
     # fininished handling it, causing us to echo it back to the room)
-    discordSendDelay: 750
+    discordSendDelay: 1500
 ghosts:
     # Pattern for the ghosts nick, available is :nick, :username, :tag and :id
     nickPattern: ":nick"

From 6dbb90258e143ed1452eb381dfc793760b993fe7 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Fri, 13 Nov 2020 08:23:40 +0200
Subject: [PATCH 015/179] Mention and recommend enabling usePrivilegedIntents

---
 ...configuring-playbook-bridge-appservice-discord.md | 12 +++++++-----
 .../defaults/main.yml                                |  8 ++++++++
 .../templates/config.yaml.j2                         |  3 +++
 3 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/docs/configuring-playbook-bridge-appservice-discord.md b/docs/configuring-playbook-bridge-appservice-discord.md
index 13a5174b..11be678e 100644
--- a/docs/configuring-playbook-bridge-appservice-discord.md
+++ b/docs/configuring-playbook-bridge-appservice-discord.md
@@ -14,18 +14,20 @@ Instructions loosely based on [this](https://github.com/Half-Shot/matrix-appserv
 1. Create a Discord Application [here](https://discordapp.com/developers/applications).
 2. Retrieve Client ID.
 3. Create a bot from the Bot tab and retrieve the Bot token.
-4. Enable the bridge with the following configuration in your `vars.yml` file:
+4. From the Bot tab, enable all checkboxes related to Privileged Gateway Intents (you can skip this step if you're not using `matrix_appservice_discord_auth_usePrivilegedIntents: true` below)
+5. Enable the bridge with the following configuration in your `vars.yml` file:
 
 ```yaml
 matrix_appservice_discord_enabled: true
 matrix_appservice_discord_client_id: "YOUR DISCORD APP CLIENT ID"
 matrix_appservice_discord_bot_token: "YOUR DISCORD APP BOT TOKEN"
+matrix_appservice_discord_auth_usePrivilegedIntents: true
 ```
 
-4. If you've already installed Matrix services using the playbook before, you'll need to re-run it (`--tags=setup-all,start`). If not, proceed with [configuring other playbook services](configuring-playbook.md) and then with [Installing](installing.md). Get back to this guide once ready.
-5. Retrieve Discord invite link from the `{{ matrix_appservice_discord_config_path }}/invite_link` file on the server (this defaults to `/matrix/appservice-discord/config/invite_link`). You need to peek at the file on the server via SSH, etc., because it's not available via HTTP(S).
-6. Invite the Bot to Discord servers you wish to bridge. Administrator permission is recommended.
-7. Room addresses follow this syntax: `#_discord_guildid_channelid`. You can easily find the guild and channel ids by logging into Discord in a browser and opening the desired channel. The URL will have this format: `discordapp.com/channels/guild_id/channel_id`. Once you have figured out the appropriate room addrss, you can join by doing `/join #_discord_guildid_channelid` in your Matrix client.
+6. If you've already installed Matrix services using the playbook before, you'll need to re-run it (`--tags=setup-all,start`). If not, proceed with [configuring other playbook services](configuring-playbook.md) and then with [Installing](installing.md). Get back to this guide once ready.
+7. Retrieve Discord invite link from the `{{ matrix_appservice_discord_config_path }}/invite_link` file on the server (this defaults to `/matrix/appservice-discord/config/invite_link`). You need to peek at the file on the server via SSH, etc., because it's not available via HTTP(S).
+8. Invite the Bot to Discord servers you wish to bridge. Administrator permission is recommended.
+9. Room addresses follow this syntax: `#_discord_guildid_channelid`. You can easily find the guild and channel ids by logging into Discord in a browser and opening the desired channel. The URL will have this format: `discordapp.com/channels/guild_id/channel_id`. Once you have figured out the appropriate room addrss, you can join by doing `/join #_discord_guildid_channelid` in your Matrix client.
 
 Other configuration options are available via the `matrix_appservice_discord_configuration_extension_yaml` variable.
 
diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml
index 392f3028..cb262166 100644
--- a/roles/matrix-bridge-appservice-discord/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml
@@ -41,6 +41,14 @@ matrix_appservice_discord_bridge_homeserverUrl: "{{ matrix_homeserver_url }}"
 matrix_appservice_discord_bridge_disablePresence: false
 matrix_appservice_discord_bridge_enableSelfServiceBridging: false
 
+# Tells whether the bot should make use of "Privileged Gateway Intents".
+#
+# Enabling this means that you need to enable it for the bot (Discord application) as well,
+# by triggering all Intent checkboxes on a page like this: `https://discord.com/developers/applications/694448564151123988/bot`
+#
+# Learn more: https://gist.github.com/advaith1/e69bcc1cdd6d0087322734451f15aa2f
+matrix_appservice_discord_auth_usePrivilegedIntents: false
+
 matrix_appservice_discord_configuration_yaml: "{{ lookup('template', 'templates/config.yaml.j2') }}"
 
 matrix_appservice_discord_configuration_extension_yaml: |
diff --git a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2
index 0c2aab41..aca49228 100644
--- a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2
+++ b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2
@@ -34,6 +34,9 @@ bridge:
 auth:
   clientID: {{ matrix_appservice_discord_client_id|string|to_json }}
   botToken: {{ matrix_appservice_discord_bot_token }}
+  # You must enable "Privileged Gateway Intents" in your bot settings on discord.com (e.g. https://discord.com/developers/applications/12345/bot)
+  # for this to work
+  usePrivilegedIntents: false
 logging:
   # What level should the logger output to the console at.
   console: "warn" #silly, verbose, info, http, warn, error, silent

From edd40811a59172b48d62d4c774f37c4854f213c2 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Thu, 3 Dec 2020 15:16:26 +0200
Subject: [PATCH 016/179] Update matrix-appservice-discord to v1.0.0 final

---
 roles/matrix-bridge-appservice-discord/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml
index cb262166..9ca06b05 100644
--- a/roles/matrix-bridge-appservice-discord/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml
@@ -3,7 +3,7 @@
 
 matrix_appservice_discord_enabled: true
 
-matrix_appservice_discord_docker_image: "halfshot/matrix-appservice-discord:v1.0.0-rc3"
+matrix_appservice_discord_docker_image: "docker.io/halfshot/matrix-appservice-discord:v1.0.0"
 matrix_appservice_discord_docker_image_force_pull: "{{ matrix_appservice_discord_docker_image.endswith(':latest') }}"
 
 matrix_appservice_discord_base_path: "{{ matrix_base_data_path }}/appservice-discord"

From b3d91ed4885be0672cdfb8045107215982fd2e3f Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Fri, 4 Dec 2020 01:06:42 +0200
Subject: [PATCH 017/179] Fix passing of
 matrix_appservice_discord_auth_usePrivilegedIntents

---
 roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2
index aca49228..b99f522a 100644
--- a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2
+++ b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2
@@ -36,7 +36,7 @@ auth:
   botToken: {{ matrix_appservice_discord_bot_token }}
   # You must enable "Privileged Gateway Intents" in your bot settings on discord.com (e.g. https://discord.com/developers/applications/12345/bot)
   # for this to work
-  usePrivilegedIntents: false
+  usePrivilegedIntents: {{ matrix_appservice_discord_auth_usePrivilegedIntents|to_json }}
 logging:
   # What level should the logger output to the console at.
   console: "warn" #silly, verbose, info, http, warn, error, silent

From a5ae7e9ef045c81c401da3cd5d84ac5677346aac Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Fri, 4 Dec 2020 01:48:08 +0200
Subject: [PATCH 018/179] Add self-building support to matrix-corporal

---
 docs/self-building.md                         |  1 +
 group_vars/matrix_servers                     |  2 ++
 roles/matrix-corporal/defaults/main.yml       |  8 ++++++-
 .../matrix-corporal/tasks/setup_corporal.yml  | 22 ++++++++++++++++++-
 4 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/docs/self-building.md b/docs/self-building.md
index 169e4aa5..fa4db222 100644
--- a/docs/self-building.md
+++ b/docs/self-building.md
@@ -15,6 +15,7 @@ List of roles where self-building the Docker image is currently possible:
 - `matrix-client-element`
 - `matrix-registration`
 - `matrix-coturn`
+- `matrix-corporal`
 - `matrix-ma1sd`
 - `matrix-mailer`
 - `matrix-bridge-mautrix-facebook`
diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 8e46a48f..d68db7ed 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -541,6 +541,8 @@ matrix_bot_matrix_reminder_bot_enabled: false
 
 matrix_corporal_enabled: false
 
+matrix_corporal_container_image_self_build: "{{ matrix_architecture != 'amd64' }}"
+
 # Normally, matrix-nginx-proxy is enabled and nginx can reach matrix-corporal over the container network.
 # If matrix-nginx-proxy is not enabled, or you otherwise have a need for it, you can expose
 # matrix-corporal's web-server ports to the local host.
diff --git a/roles/matrix-corporal/defaults/main.yml b/roles/matrix-corporal/defaults/main.yml
index def3fcc9..1cab3119 100644
--- a/roles/matrix-corporal/defaults/main.yml
+++ b/roles/matrix-corporal/defaults/main.yml
@@ -3,6 +3,9 @@
 
 matrix_corporal_enabled: true
 
+matrix_corporal_container_image_self_build: false
+matrix_corporal_container_image_self_build_repo: "https://github.com/devture/matrix-corporal.git"
+
 # Controls whether the matrix-corporal container exposes its gateway HTTP port (tcp/41080 in the container).
 #
 # Takes an "<ip>:<port>" or "<port>" value (e.g. "127.0.0.1:41080"), or empty string to not expose.
@@ -19,10 +22,13 @@ matrix_corporal_container_extra_arguments: []
 # List of systemd services that matrix-corporal.service depends on
 matrix_corporal_systemd_required_services_list: ['docker.service']
 
-matrix_corporal_docker_image: "docker.io/devture/matrix-corporal:1.11.0"
+matrix_corporal_docker_image: "{{ matrix_corporal_docker_image_name_prefix }}devture/matrix-corporal:{{ matrix_corporal_docker_image_tag }}"
+matrix_corporal_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_container_image_self_build else 'docker.io/' }}"
+matrix_corporal_docker_image_tag: "1.11.0"
 matrix_corporal_docker_image_force_pull: "{{ matrix_corporal_docker_image.endswith(':latest') }}"
 
 matrix_corporal_base_path: "{{ matrix_base_data_path }}/corporal"
+matrix_corporal_container_src_files_path: "{{ matrix_corporal_base_path }}/container-src"
 matrix_corporal_config_dir_path: "{{ matrix_corporal_base_path }}/config"
 matrix_corporal_cache_dir_path: "{{ matrix_corporal_base_path }}/cache"
 matrix_corporal_var_dir_path: "{{ matrix_corporal_base_path }}/var"
diff --git a/roles/matrix-corporal/tasks/setup_corporal.yml b/roles/matrix-corporal/tasks/setup_corporal.yml
index 72b6a5b2..188f09bf 100644
--- a/roles/matrix-corporal/tasks/setup_corporal.yml
+++ b/roles/matrix-corporal/tasks/setup_corporal.yml
@@ -17,13 +17,33 @@
     - "{{ matrix_corporal_var_dir_path }}"
   when: matrix_corporal_enabled|bool
 
+- name: Ensure Matrix Corporal repository is present on self-build
+  git:
+    repo: "{{ matrix_corporal_container_image_self_build_repo }}"
+    dest: "{{ matrix_corporal_container_src_files_path }}"
+    version: "{{ matrix_corporal_docker_image.split(':')[1] }}"
+    force: "yes"
+  register: matrix_corporal_git_pull_results
+  when: "matrix_corporal_enabled|bool and matrix_corporal_container_image_self_build|bool"
+
+- name: Ensure Matrix Corporal Docker image is built
+  docker_image:
+    name: "{{ matrix_corporal_docker_image }}"
+    source: build
+    force_source: "{{ matrix_corporal_git_pull_results.changed }}"
+    build:
+      dockerfile: etc/docker/Dockerfile
+      path: "{{ matrix_corporal_container_src_files_path }}"
+      pull: yes
+  when: "matrix_corporal_enabled|bool and matrix_corporal_container_image_self_build|bool"
+
 - name: Ensure Matrix Corporal Docker image is pulled
   docker_image:
     name: "{{ matrix_corporal_docker_image }}"
     source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
     force_source: "{{ matrix_corporal_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
     force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_corporal_docker_image_force_pull }}"
-  when: matrix_corporal_enabled|bool
+  when: "matrix_corporal_enabled|bool and not matrix_corporal_container_image_self_build|bool"
 
 - name: Ensure Matrix Corporal config installed
   copy:

From 6921ec4b8af0e5b584d5286d5903dfda769be747 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?B=C3=A9la=20Becker?= <bela@becker.rocks>
Date: Sun, 2 Aug 2020 20:54:13 +0200
Subject: [PATCH 019/179] Revert "Work around buggy docker_network sometimes
 failing to work"

The docker_network bug was fixed two years ago
This reverts commit 36658addcd688074eb9062ee71f16ed8a0bcad1b.
---
 roles/matrix-base/tasks/setup_matrix_base.yml | 29 +++----------------
 1 file changed, 4 insertions(+), 25 deletions(-)

diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml
index 22d330f2..0fad2b3d 100644
--- a/roles/matrix-base/tasks/setup_matrix_base.yml
+++ b/roles/matrix-base/tasks/setup_matrix_base.yml
@@ -19,31 +19,10 @@
     mode: '0660'
   when: "matrix_vars_yml_snapshotting_enabled|bool"
 
-# `docker_network` doesn't work as expected when the given network
-# is a substring of a network that already exists.
-#
-# See:
-# - https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/12
-# - https://github.com/ansible/ansible/issues/32926
-#
-# Due to that, we employ a workaround below.
-#
-# - name: Ensure Matrix network is created in Docker
-#   docker_network:
-#     name: "{{ matrix_docker_network }}"
-#     driver: bridge
-
-- name: Check existence of Matrix network in Docker
-  shell:
-    cmd: "docker network ls -q --filter='name=^{{ matrix_docker_network }}$'"
-  register: result_check_docker_network
-  changed_when: false
-  check_mode: no
-
-- name: Create Matrix network in Docker
-  shell:
-    cmd: "docker network create --driver=bridge {{ matrix_docker_network }}"
-  when: "result_check_docker_network.stdout == '' and not ansible_check_mode"
+- name: Ensure Matrix network is created in Docker
+  docker_network:
+    name: "{{ matrix_docker_network }}"
+    driver: bridge
 
 - name: Ensure matrix-remove-all script created
   template:

From 6f9b4bd9ac8f257a9ac8e1c4b72cfdf1c4875f56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?B=C3=A9la=20Becker?= <bela@becker.rocks>
Date: Sun, 2 Aug 2020 21:13:45 +0200
Subject: [PATCH 020/179] Drop workaround for old Ansible docker_network bug

---
 roles/matrix-coturn/tasks/setup_coturn.yml | 19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)

diff --git a/roles/matrix-coturn/tasks/setup_coturn.yml b/roles/matrix-coturn/tasks/setup_coturn.yml
index 1f13da03..f3c1048f 100644
--- a/roles/matrix-coturn/tasks/setup_coturn.yml
+++ b/roles/matrix-coturn/tasks/setup_coturn.yml
@@ -59,22 +59,11 @@
     mode: 0644
   when: matrix_coturn_enabled|bool
 
-# `docker_network` doesn't work as expected when the given network
-# is a substring of a network that already exists.
-#
-# See our other comments in `roles/matrix-base/tasks/setup_matrix_base.yml`
-- name: Check existence of Coturn network in Docker
-  shell:
-    cmd: "docker network ls -q --filter='name=^{{ matrix_coturn_docker_network }}$'"
-  register: matrix_coturn_result_docker_network
-  changed_when: false
+- name: Ensure Coturn network is created in Docker
+  docker_network:
+    name: "{{ matrix_coturn_docker_network }}"
+    driver: bridge
   when: matrix_coturn_enabled|bool
-  check_mode: no
-
-- name: Create Coturn network in Docker
-  shell:
-    cmd: "docker network create --driver=bridge {{ matrix_coturn_docker_network }}"
-  when: "matrix_coturn_enabled|bool and matrix_coturn_result_docker_network.stdout == '' and not ansible_check_mode"
 
 - name: Ensure matrix-coturn.service installed
   template:

From ec2a9d4852c49029c08df9f3651bab94fb2057fd Mon Sep 17 00:00:00 2001
From: Hardy Erlinger <hardy.erlinger@netspectrum.de>
Date: Sun, 6 Dec 2020 13:50:45 +0100
Subject: [PATCH 021/179] Remove the recording button from the Jitsi UI if
 recording is disabled.

---
 roles/matrix-jitsi/templates/web/interface_config.js.j2 | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/roles/matrix-jitsi/templates/web/interface_config.js.j2 b/roles/matrix-jitsi/templates/web/interface_config.js.j2
index 60fac4d5..a12ca973 100644
--- a/roles/matrix-jitsi/templates/web/interface_config.js.j2
+++ b/roles/matrix-jitsi/templates/web/interface_config.js.j2
@@ -205,9 +205,11 @@ var interfaceConfig = {
 		{% if matrix_jitsi_enable_transcriptions %}
             'closedcaptions',
 		{% endif %}
-
+		{% if matrix_jitsi_enable_recording %}
+            'recording',
+		{% endif %}
         'microphone', 'camera', 'desktop', 'embedmeeting', 'fullscreen',
-        'fodeviceselection', 'hangup', 'profile', 'chat', 'recording',
+        'fodeviceselection', 'hangup', 'profile', 'chat',
         'livestreaming', 'etherpad', 'sharedvideo', 'settings', 'raisehand',
         'videoquality', 'filmstrip', 'invite', 'feedback', 'stats', 'shortcuts',
         'tileview', 'videobackgroundblur', 'download', 'help', 'mute-everyone', 'security'

From 7372480e95d9fa60c05270236cc6f4d6753c3e67 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 6 Dec 2020 23:59:58 +0200
Subject: [PATCH 022/179] Properly serialize some ma1sd configuration values

We've had a report of the `connection` value getting cut off,
supposedly because it contains something that breaks off the string.

Using `|to_json` takes care of it.
---
 roles/matrix-ma1sd/templates/ma1sd.yaml.j2 | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
index 8f3569b1..84585707 100644
--- a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
+++ b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
@@ -73,10 +73,10 @@ hashing:
     - none   # the same as v1 bulk lookup
     - sha256 # hash the 3PID and pepper.
   delay: 2m # how often hashes will be updated if rotation policy = per_seconds (default is 10s)
-  requests: 10 
+  requests: 10
 {% endif %}
 
 synapseSql:
-  enabled: {{ matrix_ma1sd_synapsesql_enabled }}
-  type: {{ matrix_ma1sd_synapsesql_type }}
-  connection: {{ matrix_ma1sd_synapsesql_connection }}
+  enabled: {{ matrix_ma1sd_synapsesql_enabled|to_json }}
+  type: {{ matrix_ma1sd_synapsesql_type|to_json }}
+  connection: {{ matrix_ma1sd_synapsesql_connection|to_json }}

From 9713ac96177d71cb4e126a0442a134179f4af8cf Mon Sep 17 00:00:00 2001
From: Mike <wabuMike@users.noreply.github.com>
Date: Sun, 6 Dec 2020 23:42:44 +0100
Subject: [PATCH 023/179] fixes config.js

comments in config.js must begin with two slashes
---
 docs/configuring-playbook-jitsi.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuring-playbook-jitsi.md b/docs/configuring-playbook-jitsi.md
index 62edc578..f72241e1 100644
--- a/docs/configuring-playbook-jitsi.md
+++ b/docs/configuring-playbook-jitsi.md
@@ -99,7 +99,7 @@ matrix_jitsi_web_custom_config_extension: |
 
   config.disableAudioLevels = true;
 
-  # Limit the number of video feeds forwarded to each client
+  // Limit the number of video feeds forwarded to each client
   config.channelLastN = 4;
 
 matrix_jitsi_web_config_resolution_width_ideal_and_max: 480

From d556aa943fbd736c0401d3fec2c78d5f827a5e78 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 7 Dec 2020 06:33:35 +0200
Subject: [PATCH 024/179] Update docker-ce.repo to not hardcode $releasever=7

This keeps it in line with https://download.docker.com/linux/centos/docker-ce.repo

Whether or not Docker works well on CentOS 8 for our purposes
hasn't been verified yet.

Related to https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/300
---
 .../files/yum.repos.d/docker-ce.repo          | 42 +++++++++----------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/roles/matrix-base/files/yum.repos.d/docker-ce.repo b/roles/matrix-base/files/yum.repos.d/docker-ce.repo
index 56242d98..1abdbe36 100644
--- a/roles/matrix-base/files/yum.repos.d/docker-ce.repo
+++ b/roles/matrix-base/files/yum.repos.d/docker-ce.repo
@@ -1,62 +1,62 @@
 [docker-ce-stable]
 name=Docker CE Stable - $basearch
-baseurl=https://download.docker.com/linux/centos/7/$basearch/stable
+baseurl=https://download.docker.com/linux/centos/$releasever/$basearch/stable
 enabled=1
 gpgcheck=1
 gpgkey=https://download.docker.com/linux/centos/gpg
 
 [docker-ce-stable-debuginfo]
 name=Docker CE Stable - Debuginfo $basearch
-baseurl=https://download.docker.com/linux/centos/7/debug-$basearch/stable
+baseurl=https://download.docker.com/linux/centos/$releasever/debug-$basearch/stable
 enabled=0
 gpgcheck=1
 gpgkey=https://download.docker.com/linux/centos/gpg
 
 [docker-ce-stable-source]
 name=Docker CE Stable - Sources
-baseurl=https://download.docker.com/linux/centos/7/source/stable
+baseurl=https://download.docker.com/linux/centos/$releasever/source/stable
 enabled=0
 gpgcheck=1
 gpgkey=https://download.docker.com/linux/centos/gpg
 
-[docker-ce-edge]
-name=Docker CE Edge - $basearch
-baseurl=https://download.docker.com/linux/centos/7/$basearch/edge
+[docker-ce-test]
+name=Docker CE Test - $basearch
+baseurl=https://download.docker.com/linux/centos/$releasever/$basearch/test
 enabled=0
 gpgcheck=1
 gpgkey=https://download.docker.com/linux/centos/gpg
 
-[docker-ce-edge-debuginfo]
-name=Docker CE Edge - Debuginfo $basearch
-baseurl=https://download.docker.com/linux/centos/7/debug-$basearch/edge
+[docker-ce-test-debuginfo]
+name=Docker CE Test - Debuginfo $basearch
+baseurl=https://download.docker.com/linux/centos/$releasever/debug-$basearch/test
 enabled=0
 gpgcheck=1
 gpgkey=https://download.docker.com/linux/centos/gpg
 
-[docker-ce-edge-source]
-name=Docker CE Edge - Sources
-baseurl=https://download.docker.com/linux/centos/7/source/edge
+[docker-ce-test-source]
+name=Docker CE Test - Sources
+baseurl=https://download.docker.com/linux/centos/$releasever/source/test
 enabled=0
 gpgcheck=1
 gpgkey=https://download.docker.com/linux/centos/gpg
 
-[docker-ce-test]
-name=Docker CE Test - $basearch
-baseurl=https://download.docker.com/linux/centos/7/$basearch/test
+[docker-ce-nightly]
+name=Docker CE Nightly - $basearch
+baseurl=https://download.docker.com/linux/centos/$releasever/$basearch/nightly
 enabled=0
 gpgcheck=1
 gpgkey=https://download.docker.com/linux/centos/gpg
 
-[docker-ce-test-debuginfo]
-name=Docker CE Test - Debuginfo $basearch
-baseurl=https://download.docker.com/linux/centos/7/debug-$basearch/test
+[docker-ce-nightly-debuginfo]
+name=Docker CE Nightly - Debuginfo $basearch
+baseurl=https://download.docker.com/linux/centos/$releasever/debug-$basearch/nightly
 enabled=0
 gpgcheck=1
 gpgkey=https://download.docker.com/linux/centos/gpg
 
-[docker-ce-test-source]
-name=Docker CE Test - Sources
-baseurl=https://download.docker.com/linux/centos/7/source/test
+[docker-ce-nightly-source]
+name=Docker CE Nightly - Sources
+baseurl=https://download.docker.com/linux/centos/$releasever/source/nightly
 enabled=0
 gpgcheck=1
 gpgkey=https://download.docker.com/linux/centos/gpg

From 8c02f7b79bfe752b42e9efef069e675eba5b3d6a Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 7 Dec 2020 15:18:03 +0200
Subject: [PATCH 025/179] Upgrade services

---
 roles/matrix-client-element/defaults/main.yml | 2 +-
 roles/matrix-nginx-proxy/defaults/main.yml    | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml
index cbe70503..8b032ac2 100644
--- a/roles/matrix-client-element/defaults/main.yml
+++ b/roles/matrix-client-element/defaults/main.yml
@@ -3,7 +3,7 @@ matrix_client_element_enabled: true
 matrix_client_element_container_image_self_build: false
 matrix_client_element_container_image_self_build_repo: "https://github.com/vector-im/riot-web.git"
 
-matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:v1.7.14"
+matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:v1.7.15"
 matrix_client_element_docker_image_name_prefix: "{{ 'localhost/' if matrix_client_element_container_image_self_build else 'docker.io/' }}"
 matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}"
 
diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml
index ceba9ab6..7331c589 100644
--- a/roles/matrix-nginx-proxy/defaults/main.yml
+++ b/roles/matrix-nginx-proxy/defaults/main.yml
@@ -3,7 +3,7 @@ matrix_nginx_proxy_enabled: true
 # We use an official nginx image, which we fix-up to run unprivileged.
 # An alternative would be an `nginxinc/nginx-unprivileged` image, but
 # that is frequently out of date.
-matrix_nginx_proxy_docker_image: "docker.io/nginx:1.19.4-alpine"
+matrix_nginx_proxy_docker_image: "docker.io/nginx:1.19.5-alpine"
 matrix_nginx_proxy_docker_image_force_pull: "{{ matrix_nginx_proxy_docker_image.endswith(':latest') }}"
 
 matrix_nginx_proxy_base_path: "{{ matrix_base_data_path }}/nginx-proxy"
@@ -259,7 +259,7 @@ matrix_ssl_domains_to_obtain_certificates_for: []
 
 # Controls whether to obtain production or staging certificates from Let's Encrypt.
 matrix_ssl_lets_encrypt_staging: false
-matrix_ssl_lets_encrypt_certbot_docker_image: "docker.io/certbot/certbot:{{ matrix_ssl_architecture }}-v1.9.0"
+matrix_ssl_lets_encrypt_certbot_docker_image: "docker.io/certbot/certbot:{{ matrix_ssl_architecture }}-v1.10.1"
 matrix_ssl_lets_encrypt_certbot_docker_image_force_pull: "{{ matrix_ssl_lets_encrypt_certbot_docker_image.endswith(':latest') }}"
 matrix_ssl_lets_encrypt_certbot_standalone_http_port: 2402
 matrix_ssl_lets_encrypt_support_email: ~

From c07c927d9f2f37958300169ab27ecf6a4eeabd52 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 8 Dec 2020 16:48:25 +0200
Subject: [PATCH 026/179] Automatically enable openid listeners when ma1sd
 enabled

ma1sd requires the openid endpoints for certain functionality.
Example: https://github.com/ma1uta/ma1sd/blob/90b2b5301c34168346fdc5e7eccc09d6958e999f/src/main/java/io/kamax/mxisd/auth/AccountManager.java#L67-L99

If federation is disabled, we still need to expose these openid APIs on the
federation port.

Previously, we were doing similar magic for Dimension.
As per its documentation, when running unfederated, one is to enable
the openid listener as well. As per their recommendation, people
are advised to do enable it on the Client-Server API port
and use the `federationUrl` variable to override where the federation
port is (making federation requests go to the Client-Server API).

Because ma1sd always uses the federation port (unless you do some
DNS overwriting magic using its configuration -- which we'd rather not
do), it's better if we just default to putting the `openid` listener
where it belongs - on the federation port.

With this commit, we retain the "automatically enable openid APIs" thing
we've been doing for Dimension, but move it to the federation port instead.
We also now do the same thing when ma1sd is enabled.
---
 CHANGELOG.md                                  | 12 ++++++++++
 docs/configuring-playbook-dimension.md        |  5 ++++-
 docs/configuring-playbook-federation.md       | 10 +++++++++
 docs/configuring-playbook-ma1sd.md            |  7 +++++-
 docs/prerequisites.md                         | 13 ++++++++++-
 group_vars/matrix_servers                     | 12 +++-------
 roles/matrix-synapse/defaults/main.yml        | 22 ++++++++++++++++++-
 .../templates/synapse/homeserver.yaml.j2      |  8 +++----
 8 files changed, 72 insertions(+), 17 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 27afb64e..5024d841 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,15 @@
+# 2020-12-08
+
+## openid APIs exposed by default on the federation port when federation disabled
+
+We've changed some defaults. People running with our default configuration (federation enabled), are not affected at all.
+
+If you are running an unfederated server (`matrix_synapse_federation_enabled: false`), this may be of interest to you.
+
+When federation is disabled, but ma1sd or Dimension are enabled, we'll now expose the `openid` APIs on the federation port.
+These APIs are necessary for some ma1sd features to work. If you'd like to prevent this, you can: `matrix_synapse_federation_port_openid_resource_required: false`.
+
+
 # 2020-11-27
 
 ## Recent Jitsi updates may require configuration changes
diff --git a/docs/configuring-playbook-dimension.md b/docs/configuring-playbook-dimension.md
index 604e6aa6..d5f0a9e6 100644
--- a/docs/configuring-playbook-dimension.md
+++ b/docs/configuring-playbook-dimension.md
@@ -3,6 +3,9 @@
 **[Dimension](https://dimension.t2bot.io) can only be installed after Matrix services are installed and running.**
 If you're just installing Matrix services for the first time, please continue with the [Configuration](configuring-playbook.md) / [Installation](installing.md) flow and come back here later.
 
+**Note**: enabling Dimension, means that the `openid` API endpoints will be exposed on the Matrix Federation port (usually `8448`), even if [federation](configuring-playbook-federation.md) is disabled. It's something to be aware of, especially in terms of firewall whitelisting (make sure port `8448` is accessible).
+
+
 ## Prerequisites
 
 This playbook now supports running [Dimension](https://dimension.t2bot.io) in both a federated and an [unfederated](https://github.com/turt2live/matrix-dimension/blob/master/docs/unfederated.md) environment. This is handled automatically based on the value of `matrix_synapse_federation_enabled`.
@@ -48,7 +51,7 @@ To get an access token for the Dimension user, you can follow one of two options
 3. Copy the highlighted text to your configuration.
 4. Close the private browsing session. **Do not log out**. Logging out will invalidate the token, making it not work.
 
-*With CURL* 
+*With CURL*
 
 ```
 curl -X POST --header 'Content-Type: application/json' -d '{
diff --git a/docs/configuring-playbook-federation.md b/docs/configuring-playbook-federation.md
index 1e4ad61e..2e6410ec 100644
--- a/docs/configuring-playbook-federation.md
+++ b/docs/configuring-playbook-federation.md
@@ -37,3 +37,13 @@ matrix_synapse_federation_enabled: false
 ```
 
 With that, your server's users will only be able to talk among themselves, but not to anyone who is on another server.
+
+**Disabling federation does not necessarily disable the federation port** (`8448`). Services like [Dimension](configuring-playbook-dimension.md) and [ma1sd](configuring-playbook-ma1sd.md) normally rely on `openid` APIs exposed on that port. Even if you disable federation and only if necessary, we may still be exposing the federation port and serving the `openid` APIs there. To override this and completely disable Synapse's federation port use:
+
+```yaml
+# This stops the federation port on the Synapse side (normally `matrix-synapse:8048` on the container network).
+matrix_synapse_federation_port_enabled: false
+
+# This removes the `8448` virtual host from the matrix-nginx-proxy reverse-proxy server.
+matrix_nginx_proxy_proxy_matrix_federation_api_enabled: false
+```
diff --git a/docs/configuring-playbook-ma1sd.md b/docs/configuring-playbook-ma1sd.md
index 03208337..70c507cb 100644
--- a/docs/configuring-playbook-ma1sd.md
+++ b/docs/configuring-playbook-ma1sd.md
@@ -4,7 +4,9 @@ By default, this playbook configures an [ma1sd](https://github.com/ma1uta/ma1sd)
 
 This server is private by default, potentially at the expense of user discoverability.
 
-ma1sd is a fork of [mxisd](https://github.com/kamax-io/mxisd) which was pronounced end of life 2019-06-21.
+*ma1sd is a fork of [mxisd](https://github.com/kamax-io/mxisd) which was pronounced end of life 2019-06-21.*
+
+**Note**: enabling ma1sd (which is also the default), means that the `openid` API endpoints will be exposed on the Matrix Federation port (usually `8448`), even if [federation](configuring-playbook-federation.md) is disabled. It's something to be aware of, especially in terms of firewall whitelisting (make sure port `8448` is accessible).
 
 
 ## Disabling ma1sd
@@ -50,6 +52,9 @@ To use the [Registration](https://github.com/ma1uta/ma1sd/blob/master/docs/featu
 
 - `matrix_ma1sd_configuration_extension_yaml` - to configure ma1sd as required. See the [Registration feature's docs](https://github.com/ma1uta/ma1sd/blob/master/docs/features/registration.md) for inspiration. Also see the [Additional features](#additional-features) section below to learn more about how to use `matrix_ma1sd_configuration_extension_yaml`.
 
+**Note**: For this to work, either the homeserver needs to [federate](configuring-playbook-federation.md) or the `openid` APIs need to exposed on the federation port. When federation is disabled and ma1sd is enabled, we automatically expose the `openid` APIs (only!) on the federation port. Make sure the federation port (usually `https://matrix.DOMAIN:8448`) is whitelisted in your firewall (even if you don't actually use/need federation).
+
+
 ## Authentication
 
 [Authentication](https://github.com/ma1uta/ma1sd/blob/master/docs/features/authentication.md) provides the possibility to use your own [Identity Stores](https://github.com/ma1uta/ma1sd/blob/master/docs/stores/README.md) (for example LDAP) to authenticate users on your Homeserver. The following configuration can be used to authenticate against an LDAP server:
diff --git a/docs/prerequisites.md b/docs/prerequisites.md
index 4356081b..daf6ff80 100644
--- a/docs/prerequisites.md
+++ b/docs/prerequisites.md
@@ -22,6 +22,17 @@ This playbook doesn't support running on ARM (see [this issue](https://github.co
 
 - Properly configured DNS records for `<your-domain>` (details in [Configuring DNS](configuring-dns.md)).
 
-- Some TCP/UDP ports open. This playbook configures the server's internal firewall for you. In most cases, you don't need to do anything special. But **if your server is running behind another firewall**, you'd need to open these ports: `80/tcp` (HTTP webserver), `443/tcp` (HTTPS webserver), `3478/tcp` (TURN over TCP), `3478/udp` (TURN over UDP), `5349/tcp` (TURN over TCP), `5349/udp` (TURN over UDP), `8448/tcp` (Matrix Federation API HTTPS webserver), the range `49152-49172/udp` (TURN over UDP), `4443/tcp` (Jitsi Harvester fallback), `10000/udp` (Jitsi video RTP). Depending on your firewall/NAT setup, incoming RTP packets on port 10000 may have the external IP of your firewall as destination address, due to the usage of STUN in JVB (see [`matrix_jitsi_jvb_stun_servers`](../roles/matrix-jitsi/defaults/main.yml)).
+- Some TCP/UDP ports open. This playbook configures the server's internal firewall for you. In most cases, you don't need to do anything special. But **if your server is running behind another firewall**, you'd need to open these ports:
+
+  - `80/tcp`: HTTP webserver
+  - `443/tcp`: HTTPS webserver
+  - `3478/tcp`: TURN over TCP (used by Coturn)
+  - `3478/udp`: TURN over UDP (used by Coturn)
+  - `5349/tcp`: TURN over TCP (used by Coturn)
+  - `5349/udp`: TURN over UDP (used by Coturn)
+  - `8448/tcp`: Matrix Federation API HTTPS webserver. In some cases, this **may necessary even with federation disabled**. Integration Servers (like Dimension) and Identity Servers (like ma1sd) may need to access `openid` APIs on the federation port.
+  - the range `49152-49172/udp`: TURN over UDP
+  - `4443/tcp`: Jitsi Harvester fallback
+  - `10000/udp`: Jitsi video RTP. Depending on your firewall/NAT setup, incoming RTP packets on port `10000` may have the external IP of your firewall as destination address, due to the usage of STUN in JVB (see [`matrix_jitsi_jvb_stun_servers`](../roles/matrix-jitsi/defaults/main.yml)).
 
 When ready to proceed, continue with [Configuring DNS](configuring-dns.md).
diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index d68db7ed..78a03851 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -620,8 +620,6 @@ matrix_dimension_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_ena
 matrix_integration_manager_rest_url: "{{ matrix_dimension_integrations_rest_url if matrix_dimension_enabled else None }}"
 matrix_integration_manager_ui_url: "{{ matrix_dimension_integrations_ui_url if matrix_dimension_enabled else None }}"
 
-matrix_dimension_homeserver_federationUrl: "http://matrix-synapse:{{ 8048 if matrix_synapse_federation_enabled|bool else 8008 }}"
-
 ######################################################################
 #
 # /matrix-dimension
@@ -816,7 +814,8 @@ matrix_nginx_proxy_proxy_matrix_identity_api_addr_with_container: "matrix-ma1sd:
 matrix_nginx_proxy_proxy_matrix_identity_api_addr_sans_container: "127.0.0.1:8090"
 
 # By default, we do TLS termination for the Matrix Federation API (port 8448) at matrix-nginx-proxy.
-matrix_nginx_proxy_proxy_matrix_federation_api_enabled: true
+# Unless this is handled there OR Synapse's federation listener port is disabled, we'll reverse-proxy.
+matrix_nginx_proxy_proxy_matrix_federation_api_enabled: "{{ matrix_synapse_federation_port_enabled and not matrix_synapse_tls_federation_listener_enabled }}"
 matrix_nginx_proxy_proxy_matrix_federation_api_addr_with_container: "matrix-synapse:8048"
 matrix_nginx_proxy_proxy_matrix_federation_api_addr_sans_container: "127.0.0.1:8048"
 
@@ -990,12 +989,7 @@ matrix_synapse_tls_federation_listener_enabled: false
 matrix_synapse_tls_certificate_path: ~
 matrix_synapse_tls_private_key_path: ~
 
-matrix_synapse_http_listener_resource_names: |
-  {{
-    ["client"]
-    +
-    ( ["openid"] if matrix_dimension_enabled and not matrix_synapse_federation_enabled else [] )
-  }}
+matrix_synapse_federation_port_openid_resource_required: "{{ not matrix_synapse_federation_enabled and (matrix_dimension_enabled or matrix_ma1sd_enabled) }}"
 
 matrix_synapse_email_enabled: "{{ matrix_mailer_enabled }}"
 matrix_synapse_email_smtp_host: "matrix-mailer"
diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml
index f5c2c433..35d40c7c 100644
--- a/roles/matrix-synapse/defaults/main.yml
+++ b/roles/matrix-synapse/defaults/main.yml
@@ -136,6 +136,11 @@ matrix_synapse_tls_private_key_path: "/data/{{ matrix_server_fqn_matrix }}.tls.k
 # names.
 matrix_synapse_http_listener_resource_names: ["client"]
 
+# Resources served on Synapse's federation port.
+# When disabling federation, we may wish to serve the `openid` resource here,
+# so that services like Dimension and ma1sd can work.
+matrix_synapse_federation_listener_resource_names: "{{ ['federation'] if matrix_synapse_federation_enabled else (['openid'] if matrix_synapse_federation_port_openid_resource_required else []) }}"
+
 # Enable this to allow Synapse to report utilization statistics about your server to matrix.org
 # (things like number of users, number of messages sent, uptime, load, etc.)
 matrix_synapse_report_stats: false
@@ -211,10 +216,25 @@ matrix_synapse_caches_global_factor: 0.5
 
 # Controls whether Synapse will federate at all.
 # Disable this to completely isolate your server from the rest of the Matrix network.
-# Also see: `matrix_synapse_tls_federation_listener_enabled` if you wish to keep federation enabled,
+#
+# Disabling this still keeps the federation port exposed, because it may be used for other services (`openid`).
+#
+# Also see:
+# - `matrix_synapse_tls_federation_listener_enabled` if you wish to keep federation enabled,
 # but want to stop the TLS listener (port 8448).
+# - `matrix_synapse_federation_port_enabled` to avoid exposing the federation ports
 matrix_synapse_federation_enabled: true
 
+# Controls whether the federation ports are used at all.
+# One may wish to disable federation (`matrix_synapse_federation_enabled: true`),
+# but still run other resources (like `openid`) on the federation port
+# by enabling them in `matrix_synapse_federation_listener_resource_names`.
+matrix_synapse_federation_port_enabled: "{{ matrix_synapse_federation_enabled or matrix_synapse_federation_port_openid_resource_required }}"
+
+# Controls whether an `openid` listener is to be enabled. Useful when disabling federation,
+# but needing the `openid` APIs for Dimension or an identity server like ma1sd.
+matrix_synapse_federation_port_openid_resource_required: false
+
 # A list of domain names that are allowed to federate with the given Synapse server.
 # An empty list value (`[]`) will also effectively stop federation, but if that's the desired
 # result, it's better to accomplish it by changing `matrix_synapse_federation_enabled`.
diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2
index 8c68189b..c64ed9cc 100644
--- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2
+++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2
@@ -188,7 +188,7 @@ listeners:
       - '0.0.0.0'
 {% endif %}
 
-{% if matrix_synapse_federation_enabled and matrix_synapse_tls_federation_listener_enabled %}
+{% if matrix_synapse_federation_port_enabled and matrix_synapse_tls_federation_listener_enabled %}
   # TLS-enabled listener: for when matrix traffic is sent directly to synapse.
   - port: 8448
     tls: true
@@ -197,7 +197,7 @@ listeners:
     x_forwarded: false
 
     resources:
-      - names: [federation]
+      - names: {{ matrix_synapse_federation_listener_resource_names|to_json }}
         compress: false
 {% endif %}
 
@@ -213,7 +213,7 @@ listeners:
       - names: {{ matrix_synapse_http_listener_resource_names|to_json }}
         compress: false
 
-{% if matrix_synapse_federation_enabled %}
+{% if matrix_synapse_federation_port_enabled %}
   # Unsecure HTTP listener (Federation API): for when matrix traffic passes through a reverse proxy
   # that unwraps TLS.
   - port: 8048
@@ -223,7 +223,7 @@ listeners:
     x_forwarded: true
 
     resources:
-      - names: [federation]
+      - names: {{ matrix_synapse_federation_listener_resource_names|to_json }}
         compress: false
 {% endif %}
 

From ad92c61fdddb21539f1286c65696902aad035a0c Mon Sep 17 00:00:00 2001
From: benkuly <12199167+benkuly@users.noreply.github.com>
Date: Wed, 9 Dec 2020 09:45:44 +0100
Subject: [PATCH 027/179] updated matrix-sms-bridge

---
 roles/matrix-bridge-sms/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-bridge-sms/defaults/main.yml b/roles/matrix-bridge-sms/defaults/main.yml
index 82f9b841..3c6b4c61 100644
--- a/roles/matrix-bridge-sms/defaults/main.yml
+++ b/roles/matrix-bridge-sms/defaults/main.yml
@@ -3,7 +3,7 @@
 
 matrix_sms_bridge_enabled: true
 
-matrix_sms_bridge_docker_image: "docker.io/folivonet/matrix-sms-bridge:0.5.4"
+matrix_sms_bridge_docker_image: "docker.io/folivonet/matrix-sms-bridge:0.5.5"
 
 matrix_sms_bridge_base_path: "{{ matrix_base_data_path }}/matrix-sms-bridge"
 matrix_sms_bridge_config_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/config"

From aa86e0dac65df44c21820b6cc451617c830b4ae1 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Wed, 9 Dec 2020 13:30:28 +0200
Subject: [PATCH 028/179] Upgrade Synapse (v1.23.0 -> v1.24.0)

Because the ARM images are not pushed yet, we hold back to v1.23.0
for now.
---
 roles/matrix-synapse/defaults/main.yml        |  3 +-
 .../templates/synapse/homeserver.yaml.j2      | 59 ++++++++++++++-----
 2 files changed, 46 insertions(+), 16 deletions(-)

diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml
index 35d40c7c..02d788a3 100644
--- a/roles/matrix-synapse/defaults/main.yml
+++ b/roles/matrix-synapse/defaults/main.yml
@@ -6,8 +6,9 @@ matrix_synapse_enabled: true
 matrix_synapse_container_image_self_build: false
 matrix_synapse_container_image_self_build_repo: "https://github.com/matrix-org/synapse.git"
 
-matrix_synapse_docker_image: "{{ matrix_synapse_docker_image_name_prefix }}matrixdotorg/synapse:v1.23.0"
+matrix_synapse_docker_image: "{{ matrix_synapse_docker_image_name_prefix }}matrixdotorg/synapse:{{ matrix_synapse_docker_image_tag }}"
 matrix_synapse_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_container_image_self_build else 'docker.io/' }}"
+matrix_synapse_docker_image_tag: "{{ 'v1.24.0' if matrix_architecture == 'amd64' else 'v1.23.0' }}"
 matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}"
 
 matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse"
diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2
index c64ed9cc..41b28c88 100644
--- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2
+++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2
@@ -1202,8 +1202,9 @@ allow_guest_access: {{ matrix_synapse_allow_guest_access|to_json }}
 # email will be globally disabled.
 #
 # Additionally, if `msisdn` is not set, registration and password resets via msisdn
-# will be disabled regardless. This is due to Synapse currently not supporting any
-# method of sending SMS messages on its own.
+# will be disabled regardless, and users will not be able to associate an msisdn
+# identifier to their account. This is due to Synapse currently not supporting
+# any method of sending SMS messages on its own.
 #
 # To enable using an identity server for operations regarding a particular third-party
 # identifier type, set the value to the URL of that identity server as shown in the
@@ -1522,6 +1523,12 @@ saml2_config:
     #  remote:
     #    - url: https://our_idp/metadata.xml
 
+    # Allowed clock difference in seconds between the homeserver and IdP.
+    #
+    # Uncomment the below to increase the accepted time difference from 0 to 3 seconds.
+    #
+    #accepted_time_diff: 3
+
     # By default, the user has to go to our login page first. If you'd like
     # to allow IdP-initiated login, set 'allow_unsolicited: true' in a
     # 'service.sp' section:
@@ -1644,6 +1651,14 @@ saml2_config:
   #  - attribute: department
   #    value: "sales"
 
+  # If the metadata XML contains multiple IdP entities then the `idp_entityid`
+  # option must be set to the entity to redirect users to.
+  #
+  # Most deployments only have a single IdP entity and so should omit this
+  # option.
+  #
+  #idp_entityid: 'https://our_idp/entityid'
+
 
 # Enable OpenID Connect (OIDC) / OAuth 2.0 for registration and login.
 #
@@ -2222,21 +2237,35 @@ password_providers:
 {% endif %}
 
 
+## Push ##
 
-# Clients requesting push notifications can either have the body of
-# the message sent in the notification poke along with other details
-# like the sender, or just the event ID and room ID (`event_id_only`).
-# If clients choose the former, this option controls whether the
-# notification request includes the content of the event (other details
-# like the sender are still included). For `event_id_only` push, it
-# has no effect.
-#
-# For modern android devices the notification content will still appear
-# because it is loaded by the app. iPhone, however will send a
-# notification saying only that a message arrived and who it came from.
-#
 push:
-   include_content: {{ matrix_synapse_push_include_content|to_json }}
+  # Clients requesting push notifications can either have the body of
+  # the message sent in the notification poke along with other details
+  # like the sender, or just the event ID and room ID (`event_id_only`).
+  # If clients choose the former, this option controls whether the
+  # notification request includes the content of the event (other details
+  # like the sender are still included). For `event_id_only` push, it
+  # has no effect.
+  #
+  # For modern android devices the notification content will still appear
+  # because it is loaded by the app. iPhone, however will send a
+  # notification saying only that a message arrived and who it came from.
+  #
+  # The default value is "true" to include message details. Uncomment to only
+  # include the event ID and room ID in push notification payloads.
+  #
+  include_content: {{ matrix_synapse_push_include_content|to_json }}
+
+  # When a push notification is received, an unread count is also sent.
+  # This number can either be calculated as the number of unread messages
+  # for the user, or the number of *rooms* the user has unread messages in.
+  #
+  # The default value is "true", meaning push clients will see the number of
+  # rooms with unread messages in them. Uncomment to instead send the number
+  # of unread messages.
+  #
+  #group_unread_count_by_room: false
 
 
 # Spam checkers are third-party modules that can block specific actions

From 245b749946a971e3b019d7be5500a59da80d1b24 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Wed, 9 Dec 2020 20:54:18 +0200
Subject: [PATCH 029/179] Upgrade Synapse for ARM (v1.23.0 -> v1.24.0)

Continuation of aa86e0dac65, now that ARM images are out.
---
 roles/matrix-synapse/defaults/main.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml
index 02d788a3..0dc71646 100644
--- a/roles/matrix-synapse/defaults/main.yml
+++ b/roles/matrix-synapse/defaults/main.yml
@@ -8,7 +8,10 @@ matrix_synapse_container_image_self_build_repo: "https://github.com/matrix-org/s
 
 matrix_synapse_docker_image: "{{ matrix_synapse_docker_image_name_prefix }}matrixdotorg/synapse:{{ matrix_synapse_docker_image_tag }}"
 matrix_synapse_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_container_image_self_build else 'docker.io/' }}"
-matrix_synapse_docker_image_tag: "{{ 'v1.24.0' if matrix_architecture == 'amd64' else 'v1.23.0' }}"
+# The if statement below may look silly at times (leading to the same version being returned),
+# but ARM-compatible container images are only released 1-7 hours after a release,
+# so we may often be on different versions for different architectures when new Synapse releases come out.
+matrix_synapse_docker_image_tag: "{{ 'v1.24.0' if matrix_architecture == 'amd64' else 'v1.24.0' }}"
 matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}"
 
 matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse"

From 673e19f8307bdfc76dc4d1e63dcab40c84ebb37e Mon Sep 17 00:00:00 2001
From: John Goerzen <jgoerzen@complete.org>
Date: Wed, 9 Dec 2020 21:20:06 -0600
Subject: [PATCH 030/179] Correct inabillity for appservice-discord to connect

After recently updating my matrix-docker-ansible-deploy installation, matrix-appservice-discord would refuse to start, logging ECONNREFUSED to https://matrix.[mydomain]:443, which was resolving to 172.18.0.2 due to the `--hostname` in mailer grabbing that hostname.

Curious why the IRC bridge didn't have this issue, I looked into it, and it was connecting to `http://matrix-synapse:8008`.  Correcting this one to that URL resolved the issue.
---
 roles/matrix-bridge-appservice-discord/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml
index 9ca06b05..357b93d6 100644
--- a/roles/matrix-bridge-appservice-discord/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml
@@ -37,7 +37,7 @@ matrix_appservice_discord_appservice_url: 'http://matrix-appservice-discord:9005
 
 matrix_appservice_discord_bridge_domain: "{{ matrix_domain }}"
 # As of right now, the homeserver URL must be a public URL. See below.
-matrix_appservice_discord_bridge_homeserverUrl: "{{ matrix_homeserver_url }}"
+matrix_appservice_discord_bridge_homeserverUrl: "http://matrix-synapse:8008"
 matrix_appservice_discord_bridge_disablePresence: false
 matrix_appservice_discord_bridge_enableSelfServiceBridging: false
 

From d08b27784f222effcbce2abf924bf07bbe0893be Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Thu, 10 Dec 2020 11:36:39 +0200
Subject: [PATCH 031/179] Fix systemd services autostart problem with Docker
 20.10

The Docker 19.04 -> 20.10 upgrade contains the following change
in `/usr/lib/systemd/system/docker.service`:

```
-BindsTo=containerd.service
-After=network-online.target firewalld.service containerd.service
+After=network-online.target firewalld.service containerd.service multi-user.target
-Requires=docker.socket
+Requires=docker.socket containerd.service
Wants=network-online.target
```

The `multi-user.target` requirement in `After` seems to be in conflict
with our `WantedBy=multi-user.target` and `After=docker.service` /
`Requires=docker.service` definitions, causing the following error on
startup for all of our systemd services:

> Job matrix-synapse.service/start deleted to break ordering cycle starting with multi-user.target/start

A workaround which appears to work is to add `DefaultDependencies=no`
to all of our services.
---
 .../templates/systemd/matrix-bot-matrix-reminder-bot.service.j2  | 1 +
 .../templates/systemd/matrix-appservice-discord.service.j2       | 1 +
 .../templates/systemd/matrix-appservice-irc.service.j2           | 1 +
 .../templates/systemd/matrix-appservice-slack.service.j2         | 1 +
 .../templates/systemd/matrix-appservice-webhooks.service.j2      | 1 +
 .../templates/systemd/matrix-mautrix-facebook.service.j2         | 1 +
 .../templates/systemd/matrix-mautrix-hangouts.service.j2         | 1 +
 .../templates/systemd/matrix-mautrix-telegram.service.j2         | 1 +
 .../templates/systemd/matrix-mautrix-whatsapp.service.j2         | 1 +
 .../templates/systemd/matrix-mx-puppet-discord.service.j2        | 1 +
 .../templates/systemd/matrix-mx-puppet-instagram.service.j2      | 1 +
 .../templates/systemd/matrix-mx-puppet-skype.service.j2          | 1 +
 .../templates/systemd/matrix-mx-puppet-slack.service.j2          | 1 +
 .../templates/systemd/matrix-mx-puppet-steam.service.j2          | 1 +
 .../templates/systemd/matrix-mx-puppet-twitter.service.j2        | 1 +
 .../templates/systemd/matrix-sms-bridge.service.j2               | 1 +
 .../templates/systemd/matrix-client-element.service.j2           | 1 +
 .../matrix-corporal/templates/systemd/matrix-corporal.service.j2 | 1 +
 roles/matrix-coturn/templates/systemd/matrix-coturn.service.j2   | 1 +
 .../templates/systemd/matrix-dimension.service.j2                | 1 +
 .../templates/systemd/matrix-dynamic-dns.service.j2              | 1 +
 .../templates/systemd/matrix-email2matrix.service.j2             | 1 +
 .../matrix-jitsi/templates/jicofo/matrix-jitsi-jicofo.service.j2 | 1 +
 roles/matrix-jitsi/templates/jvb/matrix-jitsi-jvb.service.j2     | 1 +
 .../templates/prosody/matrix-jitsi-prosody.service.j2            | 1 +
 roles/matrix-jitsi/templates/web/matrix-jitsi-web.service.j2     | 1 +
 roles/matrix-ma1sd/templates/systemd/matrix-ma1sd.service.j2     | 1 +
 roles/matrix-mailer/templates/systemd/matrix-mailer.service.j2   | 1 +
 .../templates/systemd/matrix-nginx-proxy.service.j2              | 1 +
 .../matrix-postgres/templates/systemd/matrix-postgres.service.j2 | 1 +
 .../templates/systemd/matrix-registration.service.j2             | 1 +
 .../templates/systemd/matrix-synapse-admin.service.j2            | 1 +
 .../templates/goofys/systemd/matrix-goofys.service.j2            | 1 +
 .../templates/synapse/systemd/matrix-synapse.service.j2          | 1 +
 34 files changed, 34 insertions(+)

diff --git a/roles/matrix-bot-matrix-reminder-bot/templates/systemd/matrix-bot-matrix-reminder-bot.service.j2 b/roles/matrix-bot-matrix-reminder-bot/templates/systemd/matrix-bot-matrix-reminder-bot.service.j2
index 23493c54..950242e5 100644
--- a/roles/matrix-bot-matrix-reminder-bot/templates/systemd/matrix-bot-matrix-reminder-bot.service.j2
+++ b/roles/matrix-bot-matrix-reminder-bot/templates/systemd/matrix-bot-matrix-reminder-bot.service.j2
@@ -8,6 +8,7 @@ After={{ service }}
 {% for service in matrix_bot_matrix_reminder_bot_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-bridge-appservice-discord/templates/systemd/matrix-appservice-discord.service.j2 b/roles/matrix-bridge-appservice-discord/templates/systemd/matrix-appservice-discord.service.j2
index 6a44a3e6..7ec15752 100644
--- a/roles/matrix-bridge-appservice-discord/templates/systemd/matrix-appservice-discord.service.j2
+++ b/roles/matrix-bridge-appservice-discord/templates/systemd/matrix-appservice-discord.service.j2
@@ -8,6 +8,7 @@ After={{ service }}
 {% for service in matrix_appservice_discord_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-bridge-appservice-irc/templates/systemd/matrix-appservice-irc.service.j2 b/roles/matrix-bridge-appservice-irc/templates/systemd/matrix-appservice-irc.service.j2
index 95638612..c86eb835 100644
--- a/roles/matrix-bridge-appservice-irc/templates/systemd/matrix-appservice-irc.service.j2
+++ b/roles/matrix-bridge-appservice-irc/templates/systemd/matrix-appservice-irc.service.j2
@@ -8,6 +8,7 @@ After={{ service }}
 {% for service in matrix_appservice_irc_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-bridge-appservice-slack/templates/systemd/matrix-appservice-slack.service.j2 b/roles/matrix-bridge-appservice-slack/templates/systemd/matrix-appservice-slack.service.j2
index 1c68294f..b16b2fed 100644
--- a/roles/matrix-bridge-appservice-slack/templates/systemd/matrix-appservice-slack.service.j2
+++ b/roles/matrix-bridge-appservice-slack/templates/systemd/matrix-appservice-slack.service.j2
@@ -8,6 +8,7 @@ After={{ service }}
 {% for service in matrix_appservice_slack_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-bridge-appservice-webhooks/templates/systemd/matrix-appservice-webhooks.service.j2 b/roles/matrix-bridge-appservice-webhooks/templates/systemd/matrix-appservice-webhooks.service.j2
index 08f5813f..2049ee65 100644
--- a/roles/matrix-bridge-appservice-webhooks/templates/systemd/matrix-appservice-webhooks.service.j2
+++ b/roles/matrix-bridge-appservice-webhooks/templates/systemd/matrix-appservice-webhooks.service.j2
@@ -8,6 +8,7 @@ After={{ service }}
 {% for service in matrix_appservice_webhooks_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 b/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2
index b593a2e3..18184650 100644
--- a/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2
+++ b/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2
@@ -8,6 +8,7 @@ After={{ service }}
 {% for service in matrix_mautrix_facebook_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2 b/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2
index 58433982..26280da3 100644
--- a/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2
+++ b/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2
@@ -8,6 +8,7 @@ After={{ service }}
 {% for service in matrix_mautrix_hangouts_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-bridge-mautrix-telegram/templates/systemd/matrix-mautrix-telegram.service.j2 b/roles/matrix-bridge-mautrix-telegram/templates/systemd/matrix-mautrix-telegram.service.j2
index c0fb8938..d61cdaa4 100644
--- a/roles/matrix-bridge-mautrix-telegram/templates/systemd/matrix-mautrix-telegram.service.j2
+++ b/roles/matrix-bridge-mautrix-telegram/templates/systemd/matrix-mautrix-telegram.service.j2
@@ -8,6 +8,7 @@ After={{ service }}
 {% for service in matrix_mautrix_telegram_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-bridge-mautrix-whatsapp/templates/systemd/matrix-mautrix-whatsapp.service.j2 b/roles/matrix-bridge-mautrix-whatsapp/templates/systemd/matrix-mautrix-whatsapp.service.j2
index ac2b961e..972d4e01 100644
--- a/roles/matrix-bridge-mautrix-whatsapp/templates/systemd/matrix-mautrix-whatsapp.service.j2
+++ b/roles/matrix-bridge-mautrix-whatsapp/templates/systemd/matrix-mautrix-whatsapp.service.j2
@@ -8,6 +8,7 @@ After={{ service }}
 {% for service in matrix_mautrix_whatsapp_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-bridge-mx-puppet-discord/templates/systemd/matrix-mx-puppet-discord.service.j2 b/roles/matrix-bridge-mx-puppet-discord/templates/systemd/matrix-mx-puppet-discord.service.j2
index 78737a17..36a4da8b 100644
--- a/roles/matrix-bridge-mx-puppet-discord/templates/systemd/matrix-mx-puppet-discord.service.j2
+++ b/roles/matrix-bridge-mx-puppet-discord/templates/systemd/matrix-mx-puppet-discord.service.j2
@@ -8,6 +8,7 @@ After={{ service }}
 {% for service in matrix_mx_puppet_discord_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-bridge-mx-puppet-instagram/templates/systemd/matrix-mx-puppet-instagram.service.j2 b/roles/matrix-bridge-mx-puppet-instagram/templates/systemd/matrix-mx-puppet-instagram.service.j2
index 81e3e081..4c94c84a 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/templates/systemd/matrix-mx-puppet-instagram.service.j2
+++ b/roles/matrix-bridge-mx-puppet-instagram/templates/systemd/matrix-mx-puppet-instagram.service.j2
@@ -8,6 +8,7 @@ After={{ service }}
 {% for service in matrix_mx_puppet_instagram_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-bridge-mx-puppet-skype/templates/systemd/matrix-mx-puppet-skype.service.j2 b/roles/matrix-bridge-mx-puppet-skype/templates/systemd/matrix-mx-puppet-skype.service.j2
index 8a46cb25..cd958662 100644
--- a/roles/matrix-bridge-mx-puppet-skype/templates/systemd/matrix-mx-puppet-skype.service.j2
+++ b/roles/matrix-bridge-mx-puppet-skype/templates/systemd/matrix-mx-puppet-skype.service.j2
@@ -8,6 +8,7 @@ After={{ service }}
 {% for service in matrix_mx_puppet_skype_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-bridge-mx-puppet-slack/templates/systemd/matrix-mx-puppet-slack.service.j2 b/roles/matrix-bridge-mx-puppet-slack/templates/systemd/matrix-mx-puppet-slack.service.j2
index 23c2504f..2b1456f5 100644
--- a/roles/matrix-bridge-mx-puppet-slack/templates/systemd/matrix-mx-puppet-slack.service.j2
+++ b/roles/matrix-bridge-mx-puppet-slack/templates/systemd/matrix-mx-puppet-slack.service.j2
@@ -8,6 +8,7 @@ After={{ service }}
 {% for service in matrix_mx_puppet_slack_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-bridge-mx-puppet-steam/templates/systemd/matrix-mx-puppet-steam.service.j2 b/roles/matrix-bridge-mx-puppet-steam/templates/systemd/matrix-mx-puppet-steam.service.j2
index 986c5d2c..e263154b 100644
--- a/roles/matrix-bridge-mx-puppet-steam/templates/systemd/matrix-mx-puppet-steam.service.j2
+++ b/roles/matrix-bridge-mx-puppet-steam/templates/systemd/matrix-mx-puppet-steam.service.j2
@@ -8,6 +8,7 @@ After={{ service }}
 {% for service in matrix_mx_puppet_steam_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-bridge-mx-puppet-twitter/templates/systemd/matrix-mx-puppet-twitter.service.j2 b/roles/matrix-bridge-mx-puppet-twitter/templates/systemd/matrix-mx-puppet-twitter.service.j2
index 6afb6fb4..8d7898eb 100644
--- a/roles/matrix-bridge-mx-puppet-twitter/templates/systemd/matrix-mx-puppet-twitter.service.j2
+++ b/roles/matrix-bridge-mx-puppet-twitter/templates/systemd/matrix-mx-puppet-twitter.service.j2
@@ -8,6 +8,7 @@ After={{ service }}
 {% for service in matrix_mx_puppet_twitter_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge.service.j2 b/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge.service.j2
index 7c96f9f4..0eb0eb58 100644
--- a/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge.service.j2
+++ b/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge.service.j2
@@ -8,6 +8,7 @@ After={{ service }}
 {% for service in matrix_sms_bridge_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-client-element/templates/systemd/matrix-client-element.service.j2 b/roles/matrix-client-element/templates/systemd/matrix-client-element.service.j2
index 39b7aa52..e0dd2e7e 100644
--- a/roles/matrix-client-element/templates/systemd/matrix-client-element.service.j2
+++ b/roles/matrix-client-element/templates/systemd/matrix-client-element.service.j2
@@ -5,6 +5,7 @@ Description=Matrix Element server
 Requires={{ service }}
 After={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-corporal/templates/systemd/matrix-corporal.service.j2 b/roles/matrix-corporal/templates/systemd/matrix-corporal.service.j2
index 97ec7d89..e8ce8c0a 100644
--- a/roles/matrix-corporal/templates/systemd/matrix-corporal.service.j2
+++ b/roles/matrix-corporal/templates/systemd/matrix-corporal.service.j2
@@ -5,6 +5,7 @@ Description=Matrix Corporal
 Requires={{ service }}
 After={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-coturn/templates/systemd/matrix-coturn.service.j2 b/roles/matrix-coturn/templates/systemd/matrix-coturn.service.j2
index e484b59b..16ca5d2a 100644
--- a/roles/matrix-coturn/templates/systemd/matrix-coturn.service.j2
+++ b/roles/matrix-coturn/templates/systemd/matrix-coturn.service.j2
@@ -5,6 +5,7 @@ Description=Matrix Coturn server
 Requires={{ service }}
 After={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 b/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2
index bad94f64..94c38491 100644
--- a/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2
+++ b/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2
@@ -3,6 +3,7 @@
 Description=Matrix Dimension
 After=docker.service
 Requires=docker.service
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-dynamic-dns/templates/systemd/matrix-dynamic-dns.service.j2 b/roles/matrix-dynamic-dns/templates/systemd/matrix-dynamic-dns.service.j2
index 7e73b587..df7d810a 100644
--- a/roles/matrix-dynamic-dns/templates/systemd/matrix-dynamic-dns.service.j2
+++ b/roles/matrix-dynamic-dns/templates/systemd/matrix-dynamic-dns.service.j2
@@ -8,6 +8,7 @@ After={{ service }}
 {% for service in matrix_dynamic_dns_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-email2matrix/templates/systemd/matrix-email2matrix.service.j2 b/roles/matrix-email2matrix/templates/systemd/matrix-email2matrix.service.j2
index 52806f35..1577877b 100644
--- a/roles/matrix-email2matrix/templates/systemd/matrix-email2matrix.service.j2
+++ b/roles/matrix-email2matrix/templates/systemd/matrix-email2matrix.service.j2
@@ -3,6 +3,7 @@
 Description=Email2Matrix
 After=docker.service
 Requires=docker.service
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-jitsi/templates/jicofo/matrix-jitsi-jicofo.service.j2 b/roles/matrix-jitsi/templates/jicofo/matrix-jitsi-jicofo.service.j2
index f19d08ff..6b5cc941 100644
--- a/roles/matrix-jitsi/templates/jicofo/matrix-jitsi-jicofo.service.j2
+++ b/roles/matrix-jitsi/templates/jicofo/matrix-jitsi-jicofo.service.j2
@@ -5,6 +5,7 @@ Description=Matrix jitsi-jicofo server
 Requires={{ service }}
 After={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-jitsi/templates/jvb/matrix-jitsi-jvb.service.j2 b/roles/matrix-jitsi/templates/jvb/matrix-jitsi-jvb.service.j2
index 7fcfeec6..2931133f 100644
--- a/roles/matrix-jitsi/templates/jvb/matrix-jitsi-jvb.service.j2
+++ b/roles/matrix-jitsi/templates/jvb/matrix-jitsi-jvb.service.j2
@@ -5,6 +5,7 @@ Description=Matrix jitsi-jvb server
 Requires={{ service }}
 After={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-jitsi/templates/prosody/matrix-jitsi-prosody.service.j2 b/roles/matrix-jitsi/templates/prosody/matrix-jitsi-prosody.service.j2
index 74b13df3..4f532d89 100644
--- a/roles/matrix-jitsi/templates/prosody/matrix-jitsi-prosody.service.j2
+++ b/roles/matrix-jitsi/templates/prosody/matrix-jitsi-prosody.service.j2
@@ -5,6 +5,7 @@ Description=Matrix jitsi-prosody server
 Requires={{ service }}
 After={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-jitsi/templates/web/matrix-jitsi-web.service.j2 b/roles/matrix-jitsi/templates/web/matrix-jitsi-web.service.j2
index e13f9633..1978fb0e 100644
--- a/roles/matrix-jitsi/templates/web/matrix-jitsi-web.service.j2
+++ b/roles/matrix-jitsi/templates/web/matrix-jitsi-web.service.j2
@@ -5,6 +5,7 @@ Description=Matrix jitsi-web server
 Requires={{ service }}
 After={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-ma1sd/templates/systemd/matrix-ma1sd.service.j2 b/roles/matrix-ma1sd/templates/systemd/matrix-ma1sd.service.j2
index 3c49ec75..95f15254 100644
--- a/roles/matrix-ma1sd/templates/systemd/matrix-ma1sd.service.j2
+++ b/roles/matrix-ma1sd/templates/systemd/matrix-ma1sd.service.j2
@@ -8,6 +8,7 @@ After={{ service }}
 {% for service in matrix_ma1sd_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-mailer/templates/systemd/matrix-mailer.service.j2 b/roles/matrix-mailer/templates/systemd/matrix-mailer.service.j2
index 1371a861..cab03128 100644
--- a/roles/matrix-mailer/templates/systemd/matrix-mailer.service.j2
+++ b/roles/matrix-mailer/templates/systemd/matrix-mailer.service.j2
@@ -3,6 +3,7 @@
 Description=Matrix mailer
 After=docker.service
 Requires=docker.service
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2 b/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2
index 0945c25e..58f5c953 100644
--- a/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2
+++ b/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2
@@ -8,6 +8,7 @@ After={{ service }}
 {% for service in matrix_nginx_proxy_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2 b/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2
index 47f19e9b..0a935fb0 100644
--- a/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2
+++ b/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2
@@ -3,6 +3,7 @@
 Description=Matrix Postgres server
 After=docker.service
 Requires=docker.service
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-registration/templates/systemd/matrix-registration.service.j2 b/roles/matrix-registration/templates/systemd/matrix-registration.service.j2
index 052b7d95..3744c2de 100644
--- a/roles/matrix-registration/templates/systemd/matrix-registration.service.j2
+++ b/roles/matrix-registration/templates/systemd/matrix-registration.service.j2
@@ -8,6 +8,7 @@ After={{ service }}
 {% for service in matrix_registration_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-synapse-admin/templates/systemd/matrix-synapse-admin.service.j2 b/roles/matrix-synapse-admin/templates/systemd/matrix-synapse-admin.service.j2
index c03c627b..7b1e40de 100644
--- a/roles/matrix-synapse-admin/templates/systemd/matrix-synapse-admin.service.j2
+++ b/roles/matrix-synapse-admin/templates/systemd/matrix-synapse-admin.service.j2
@@ -8,6 +8,7 @@ After={{ service }}
 {% for service in matrix_synapse_admin_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-synapse/templates/goofys/systemd/matrix-goofys.service.j2 b/roles/matrix-synapse/templates/goofys/systemd/matrix-goofys.service.j2
index c3ec9249..0bbfde99 100644
--- a/roles/matrix-synapse/templates/goofys/systemd/matrix-goofys.service.j2
+++ b/roles/matrix-synapse/templates/goofys/systemd/matrix-goofys.service.j2
@@ -3,6 +3,7 @@
 Description=Matrix Goofys media store
 After=docker.service
 Requires=docker.service
+DefaultDependencies=no
 
 [Service]
 Type=simple
diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2
index 791d324e..30c85b99 100644
--- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2
+++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2
@@ -8,6 +8,7 @@ After={{ service }}
 {% for service in matrix_synapse_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+DefaultDependencies=no
 
 [Service]
 Type=simple

From d9f4914e0d78fa45c7a76526a660faf94213b3df Mon Sep 17 00:00:00 2001
From: transcaffeine <transcaffeine@finallycoffee.eu>
Date: Sun, 29 Nov 2020 15:20:22 +0100
Subject: [PATCH 032/179] WIP: postgres: create databases for all services

If a service is enabled, a database for it is created in postgres with a uniqque password. The service can then use this database for data storage instead of relying on sqlite.
---
 group_vars/matrix_servers                     | 64 +++++++++++++++++++
 .../templates/config.yaml.j2                  |  4 +-
 .../templates/config.yaml.j2                  |  4 +-
 .../templates/config.yaml.j2                  |  3 +
 .../templates/config.yaml.j2                  |  2 +-
 .../templates/config.yaml.j2                  |  2 +-
 .../templates/config.yaml.j2                  |  2 +-
 .../templates/config.yaml.j2                  |  2 +-
 .../templates/config.yaml.j2                  |  4 +-
 .../templates/config.yaml.j2                  |  4 +-
 .../templates/config.yaml.j2                  |  4 +-
 .../templates/config.yaml.j2                  |  4 +-
 .../templates/config.yaml.j2                  |  4 +-
 .../templates/config.yaml.j2                  |  4 +-
 .../matrix-dimension/templates/config.yaml.j2 |  2 +-
 .../matrix-postgres/tasks/setup_postgres.yml  | 30 +++++++++
 16 files changed, 118 insertions(+), 21 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 78a03851..8a099275 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -889,6 +889,70 @@ matrix_postgres_connection_username: "synapse"
 matrix_postgres_connection_password: "synapse-password"
 matrix_postgres_db_name: "homeserver"
 
+matrix_postgres_additional_databases: |
+  {{
+    ([{
+      name: 'matrix_appservice_discord',
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string
+    }] if matrix_appservice_discord_enabled else [])
+    + ([{
+      name: 'matrix_appservice_slack'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string
+    }] if matrix_appservice_slack_enabled else [])
+    + ([{
+      name: 'matrix_appservice_irc'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string
+    }] if matrix_appservice_irc_enabled else [])
+    + ([{
+      name: 'mautrix-bridge-facebook'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string
+    }] if matrix_mautrix_facebook_enabled else [])
+    + ([{
+      name: 'mautrix_bridge_hangouts'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string
+    }] if matrix_mautrix_hangouts_enabled else [])
+    + ([{
+      name: 'mautrix_bridge_telegram'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string
+    }] if matrix_mautrix_telegram_enabled else [])
+    + ([{
+      name: 'mautrix_bridge_whatsapp'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string
+    }] if matrix_mautrix_whatsapp_enabled else [])
+    + ([{
+      name: 'matrix_bridge_sms'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string
+    }] if matrix_sms_bridge_enabled else [])
+    + ([{
+      name: 'matrix_puppet_skype'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string
+    }] if matrix_mx_puppet_skype_enabled else [])
+    + ([{
+      name: 'matrix_puppet_slack'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string
+    }] if matrix_mx_puppet_slack_enabled else [])
+    + ([{
+      name: 'matrix_puppet_twitter'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string
+    }] if matrix_mx_puppet_twitter_enabled else [])
+    + ([{
+      name: 'matrix_puppet_instagram'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string
+    ] if matrix_mx_puppet_instagram_enabled else [])
+    + ([{
+      name: 'matrix_puppet_discord'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string
+    }] if matrix_mx_puppet_discord_enabled else [])
+    + ([{
+      name: 'matrix_puppet_steam'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string
+    }] if matrix_mx_puppet_steam_enabled else [])
+    + ([{
+      name: 'matrix_dimension'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string
+    }] if matrix_dimension_enabled else [])
+  }}
+
 ######################################################################
 #
 # /matrix-postgres
diff --git a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2
index b99f522a..de4182d9 100644
--- a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2
+++ b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2
@@ -58,8 +58,8 @@ database:
   # If you are migrating, see https://github.com/Half-Shot/matrix-appservice-discord/blob/master/docs/howto.md#migrate-to-postgres-from-sqlite
   # WARNING: You will almost certainly be fine with sqlite unless your bridge
   # is in heavy demand and you suffer from IO slowness.
-  filename: "/data/discord.db"
-  # connString: "postgresql://user:password@localhost/database_name"
+  #filename: "/data/discord.db"
+  connString: "postgresql://matrix_appservice_discord:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_appservice_discord') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_discord"
 room:
   # Set the default visibility of alias rooms, defaults to "public".
   # One of: "public", "private"
diff --git a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2
index 3daa1898..83b71835 100644
--- a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2
+++ b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2
@@ -127,8 +127,8 @@ advanced:
 # Use an external database to store bridge state.
 database:
   # database engine (must be 'postgres' or 'nedb'). Default: nedb
-  engine: "nedb"
+  engine: "postgres"
   # Either a PostgreSQL connection string, or a path to the NeDB storage directory.
   # For postgres, it must start with postgres://
   # For NeDB, it must start with nedb://. The path is relative to the project directory.
-  connectionString: "nedb:///data"
+  connectionString: "postgres://matrix_appservice_irc:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_appservice_irc') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_irc"
diff --git a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2
index 8f48d317..6c491134 100644
--- a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2
+++ b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2
@@ -10,5 +10,8 @@ homeserver:
     server_name: "{{ matrix_domain }}"
 
 dbdir: "/data"
+db:
+    engine: "postgres"
+    connectionString: "postgresql://matrix_appservice_slack:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_appservice_slack') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_slack"
 
 matrix_admin_room: "{{ matrix_appservice_slack_control_room_id }}"
diff --git a/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2
index 9d8de2d5..304571c1 100644
--- a/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2
@@ -27,7 +27,7 @@ appservice:
     # Format examples:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
-    database: sqlite:////data/mautrix-facebook.db
+    database: sqlite://matrix_bridge_facebook:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_bridge_facebook') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_bridge_facebook
 
     # Public part of web server for out-of-Matrix interaction with the bridge.
     public:
diff --git a/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2
index f274b203..e4e59ad2 100644
--- a/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2
@@ -27,7 +27,7 @@ appservice:
     # Format examples:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
-    database: sqlite:////data/mautrix-hangouts.db
+    database: postgres://mautrix_bridge_hangouts:{{ matrix_additional_databases | selectattr('name', 'equalto', 'mautrix_bridge_hangouts') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/mautrix_bridge_hangouts
 
     # The unique ID of this appservice.
     id: hangouts
diff --git a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2
index 1a4ac43e..65f17d33 100644
--- a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2
@@ -27,7 +27,7 @@ appservice:
     # Format examples:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
-    database: sqlite:////data/mautrix-telegram.db
+    database: postgres://mautrix_bridge_telegram:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_bridge_telegram') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/mautrix_bridge_telegram
 
     # Public part of web server for out-of-Matrix interaction with the bridge.
     # Used for things like login if the user wants to make sure the 2FA password isn't stored in
diff --git a/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2
index a527a188..93956049 100644
--- a/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2
@@ -23,7 +23,7 @@ appservice:
         # The database URI.
         #   SQLite: File name is enough. https://github.com/mattn/go-sqlite3#connection-string
         #   Postgres: Connection string. For example, postgres://user:password@host/database
-        uri: mautrix-whatsapp.db
+        uri: postgres://matrix_bridge_whatsapp@{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_bridge_whatsapp') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_bridge_whatsapp
         # Maximum number of connections. Mostly relevant for Postgres.
         max_open_conns: 20
         max_idle_conns: 2
diff --git a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
index 2c703796..88c591e2 100644
--- a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
@@ -105,10 +105,10 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  #connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
+  connString: "postgres://matrix_puppet_discord:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_discord') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_discord?sslmode=disable"
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: /data/database.db
+  #filename: /data/database.db
 
 logging:
   # Log level of console output
diff --git a/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2
index 634fbaec..2d189195 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2
@@ -49,10 +49,10 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  #connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
+  connString: "postgres://matrix_puppet_instagram:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_instagram') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_instagram?sslmode=disable"
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: /data/database.db
+  #filename: /data/database.db
 
 logging:
   # Log level of console output
diff --git a/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2
index c7b5c870..a54ca758 100644
--- a/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2
@@ -73,10 +73,10 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  #connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
+  connString: "postgres://matrix_puppet_skype:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_skype') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_skype?sslmode=disable"
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: /data/database.db
+  #filename: /data/database.db
 
 provisioning:
   # Regex of Matrix IDs allowed to use the puppet bridge
diff --git a/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2
index b6e88784..7f1b21d7 100644
--- a/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2
@@ -63,10 +63,10 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  #connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
+  connString: "postgres://matrix_puppet_slack:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_slack') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_slack?sslmode=disable"
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: /data/database.db
+  #filename: /data/database.db
 
 logging:
   # Log level of console output
diff --git a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2
index d08982ca..14f86319 100644
--- a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2
@@ -66,10 +66,10 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  #connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
+  connString: "postgres://matrix_puppet_steam:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_steam') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_steam?sslmode=disable"
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: /data/database.db
+  #filename: /data/database.db
 
 logging:
   # Log level of console output
diff --git a/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2
index 7d3033b3..853d23fc 100644
--- a/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2
@@ -59,10 +59,10 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  #connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
+  connString: "postgres://matrix_puppet_twitter:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_twitter') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_twitter?sslmode=disable"
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: /data/database.db
+  #filename: /data/database.db
 
 logging:
   # Log level of console output
diff --git a/roles/matrix-dimension/templates/config.yaml.j2 b/roles/matrix-dimension/templates/config.yaml.j2
index a05b6c35..b7ca1ee3 100644
--- a/roles/matrix-dimension/templates/config.yaml.j2
+++ b/roles/matrix-dimension/templates/config.yaml.j2
@@ -44,7 +44,7 @@ widgetBlacklist:
 
 # Where the database for Dimension is
 database:
-  file: "dimension.db"
+  uri: "postgres://matrix_dimension:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_dimension') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_dimension"
 
 # Display settings that apply to self-hosted go-neb instances
 goneb:
diff --git a/roles/matrix-postgres/tasks/setup_postgres.yml b/roles/matrix-postgres/tasks/setup_postgres.yml
index f186bdca..85a8604c 100644
--- a/roles/matrix-postgres/tasks/setup_postgres.yml
+++ b/roles/matrix-postgres/tasks/setup_postgres.yml
@@ -162,3 +162,33 @@
     - matrix-change-user-admin-status
     - matrix-postgres-update-user-password-hash
   when: "not matrix_postgres_enabled|bool"
+
+# Create additional databases
+- name: Retrieve IP of postgres container
+  shell: "docker inspect matrix-postgres | jq -r '.[0].NetworkSettings.Networks.{{ matrix_docker_network }}.IPAddress'"
+  register: matirx_postgres_container_ip
+
+- name: Create additional users in postgres
+  postgresql_user:
+    name: "{{ item.name }}"
+    password: "{{ item.pass }}"
+    login_host: "{{ matrx_postgres_container_ip.stdout }}"
+    login_port: 5432
+    login_user: "{{ matrix_postgres_connection_username }}"
+    login_password: "{{ matrix_postgres_connection_password }}"
+    login_db: "{{ matrix_postgres_db_name }}"
+  loop: matrix_postgres_additional_databases
+  when: matrix_postgres_enabed|bool
+
+- name: Create additional users in postgres
+  postgresql_db:
+    name: "{{ item.name }}"
+    owner: "{{ item.name }}"
+    lc_ctype: 'C'
+    lc_collate: 'C'
+    login_host: "{{ matrx_postgres_container_ip.stdout }}"
+    login_port: 5432
+    login_user: "{{ matrix_postgres_connection_username }}"
+    login_password: "{{ matrix_postgres_connection_password }}"
+  loop: matrix_postgres_additional_databases
+  when: matrix_postgres_enabled|bool

From 7593d969e316cc0144bce378a5be58c76c2c37ee Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Thu, 10 Dec 2020 23:51:11 +0200
Subject: [PATCH 033/179] Make matrix-mailer not occupy
 matrix_server_fqn_matrix

Starting with Docker 20.10, `--hostname` seems to have the side-effect
of making Docker's internal DNS server resolve said hostname to the IP
address of the container.

Because we were giving the mailer service a hostname of `matrix.DOMAIN`,
all requests destined for `matrix.DOMAIN` originating from other
services on the container network were resolving to `matrix-mailer`.
This is obviously wrong.

Initially reported here: https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/748

We normally try to not use the public hostname (and IP address) on the
container network and try to make services talk to one another locally,
but it sometimes could happen.

With this, we use a `matrix-mailer` hostname for the matrix-mailer
container. My testing shows that it doesn't cause any trouble with
email deliverability.
---
 roles/matrix-mailer/templates/systemd/matrix-mailer.service.j2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-mailer/templates/systemd/matrix-mailer.service.j2 b/roles/matrix-mailer/templates/systemd/matrix-mailer.service.j2
index cab03128..9345a1d6 100644
--- a/roles/matrix-mailer/templates/systemd/matrix-mailer.service.j2
+++ b/roles/matrix-mailer/templates/systemd/matrix-mailer.service.j2
@@ -19,7 +19,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-mailer \
 			--tmpfs=/var/spool/exim:rw,noexec,nosuid,size=100m \
 			--network={{ matrix_docker_network }} \
 			--env-file={{ matrix_mailer_base_path }}/env-mailer \
-			--hostname={{ matrix_server_fqn_matrix }} \
+			--hostname=matrix-mailer \
 			{% for arg in matrix_mailer_container_extra_arguments %}
 			{{ arg }} \
 			{% endfor %}

From eae4f674708939c6b4bfd68e596fecad61b939ca Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Fri, 11 Dec 2020 00:12:56 +0200
Subject: [PATCH 034/179] Mention Docker 20.10 in the changelog

---
 CHANGELOG.md | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5024d841..85700b58 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,15 @@
+# 2020-12-11
+
+## Docker 20.10 is here
+
+(No need to do anything special in relation to this. Just something to keep in mind)
+
+Docker 20.10 got released recently and your server will likely get it the next time you update.
+
+This is the first major Docker update in a long time and it packs a lot of changes.
+Some of them introduced some breakage for us initially (see [here](https://github.com/spantaleev/matrix-docker-ansible-deploy/commit/d08b27784f222effcbce2abf924bf07bbe0893be) and [here](https://github.com/spantaleev/matrix-docker-ansible-deploy/commit/7593d969e316cc0144bce378a5be58c76c2c37ee)), but it should be all good now.
+
+
 # 2020-12-08
 
 ## openid APIs exposed by default on the federation port when federation disabled

From 3c2a644e5c8b672344b5aeafe374bcf70432b469 Mon Sep 17 00:00:00 2001
From: Aaron Raimist <aaron@raim.ist>
Date: Thu, 10 Dec 2020 16:28:48 -0600
Subject: [PATCH 035/179] Upgrade synapse-admin (v0.5.0 -> 0.6.1)

---
 roles/matrix-synapse-admin/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-synapse-admin/defaults/main.yml b/roles/matrix-synapse-admin/defaults/main.yml
index ce149dfd..1dbf0ad2 100644
--- a/roles/matrix-synapse-admin/defaults/main.yml
+++ b/roles/matrix-synapse-admin/defaults/main.yml
@@ -8,7 +8,7 @@ matrix_synapse_admin_container_self_build_repo: "https://github.com/Awesome-Tech
 
 matrix_synapse_admin_docker_src_files_path: "{{ matrix_base_data_path }}/synapse-admin/docker-src"
 
-matrix_synapse_admin_docker_image: "{{ matrix_synapse_admin_docker_image_name_prefix }}awesometechnologies/synapse-admin:0.5.0"
+matrix_synapse_admin_docker_image: "{{ matrix_synapse_admin_docker_image_name_prefix }}awesometechnologies/synapse-admin:0.6.1"
 matrix_synapse_admin_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_admin_container_self_build else 'docker.io/' }}"
 matrix_synapse_admin_docker_image_force_pull: "{{ matrix_synapse_admin_docker_image.endswith(':latest') }}"
 

From 0a9109771dd1d380fed00208fe1f98b1078265b6 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Fri, 11 Dec 2020 22:17:47 +0200
Subject: [PATCH 036/179] Use latest/master version of matrix-registration

v0.7.0 is broken right now, because it calls
`/_matrix/client/r0/admin/register`, which is now at
`/_synapse/admin/v1/register`.

This has been fixed here: https://github.com/ZerataX/matrix-registration/commit/6b26255feada4c4f0ec49bb16a60a12b85476a0f

.. but it's not part of any release.

Switching to `master` (`docker.io/devture/zeratax-matrix-registration:latest`) until it gets resolved.

Reported upstream here: https://github.com/ZerataX/matrix-registration/issues/43
---
 roles/matrix-registration/defaults/main.yml | 3 ++-
 roles/matrix-registration/tasks/setup.yml   | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/roles/matrix-registration/defaults/main.yml b/roles/matrix-registration/defaults/main.yml
index 680e8bfe..87779faf 100644
--- a/roles/matrix-registration/defaults/main.yml
+++ b/roles/matrix-registration/defaults/main.yml
@@ -5,13 +5,14 @@ matrix_registration_enabled: true
 
 matrix_registration_container_image_self_build: false
 matrix_registration_container_image_self_build_repo: "https://github.com/ZerataX/matrix-registration"
+matrix_registration_container_image_self_build_branch: "{{ 'master' if matrix_registration_version == 'latest' else matrix_registration_version }}"
 
 matrix_registration_base_path: "{{ matrix_base_data_path }}/matrix-registration"
 matrix_registration_config_path: "{{ matrix_registration_base_path }}/config"
 matrix_registration_data_path: "{{ matrix_registration_base_path }}/data"
 matrix_registration_docker_src_files_path: "{{ matrix_registration_base_path }}/docker-src"
 
-matrix_registration_version: "v0.7.0"
+matrix_registration_version: "latest"
 
 matrix_registration_docker_image: "{{ matrix_registration_docker_image_name_prefix }}devture/zeratax-matrix-registration:{{ matrix_registration_version }}"
 matrix_registration_docker_image_name_prefix: "{{ 'localhost/' if matrix_registration_container_image_self_build else 'docker.io/' }}"
diff --git a/roles/matrix-registration/tasks/setup.yml b/roles/matrix-registration/tasks/setup.yml
index 8427d950..dfe1ba8b 100644
--- a/roles/matrix-registration/tasks/setup.yml
+++ b/roles/matrix-registration/tasks/setup.yml
@@ -30,7 +30,7 @@
   git:
     repo: "{{ matrix_registration_container_image_self_build_repo }}"
     dest: "{{ matrix_registration_docker_src_files_path }}"
-    version: "{{ matrix_registration_version }}"
+    version: "{{ matrix_registration_container_image_self_build_branch }}"
     force: "yes"
   register: matrix_registration_git_pull_results
   when: "matrix_registration_enabled|bool and matrix_registration_container_image_self_build|bool"

From 86988ae1805525ab7c48ce95b34559c7f0647228 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Fri, 11 Dec 2020 22:52:42 +0200
Subject: [PATCH 037/179] Switch matrix-registration to v0.7.1

Now that a new release has been made, we no longer need to use
`latest` / `master`.

Related to 0a9109771dd1d380f and https://github.com/ZerataX/matrix-registration/issues/43
---
 roles/matrix-registration/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-registration/defaults/main.yml b/roles/matrix-registration/defaults/main.yml
index 87779faf..95147665 100644
--- a/roles/matrix-registration/defaults/main.yml
+++ b/roles/matrix-registration/defaults/main.yml
@@ -12,7 +12,7 @@ matrix_registration_config_path: "{{ matrix_registration_base_path }}/config"
 matrix_registration_data_path: "{{ matrix_registration_base_path }}/data"
 matrix_registration_docker_src_files_path: "{{ matrix_registration_base_path }}/docker-src"
 
-matrix_registration_version: "latest"
+matrix_registration_version: "v0.7.1"
 
 matrix_registration_docker_image: "{{ matrix_registration_docker_image_name_prefix }}devture/zeratax-matrix-registration:{{ matrix_registration_version }}"
 matrix_registration_docker_image_name_prefix: "{{ 'localhost/' if matrix_registration_container_image_self_build else 'docker.io/' }}"

From 47613e5a27ad226236c617eda86dbb61c692eb7a Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Fri, 11 Dec 2020 23:24:42 +0200
Subject: [PATCH 038/179] Remove synapse-janitor support

Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/746
---
 CHANGELOG.md                                  |   7 ++
 docs/maintenance-synapse.md                   |  24 ----
 roles/matrix-base/defaults/main.yml           |   1 -
 roles/matrix-postgres/defaults/main.yml       |   2 -
 roles/matrix-postgres/tasks/main.yml          |   5 -
 .../tasks/run_synapse_janitor.yml             | 117 ------------------
 6 files changed, 7 insertions(+), 149 deletions(-)
 delete mode 100644 roles/matrix-postgres/tasks/run_synapse_janitor.yml

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 85700b58..946ec5b0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,12 @@
 # 2020-12-11
 
+## synapse-janitor support removed
+
+We've removed support for the unmaintained [synapse-janitor](https://github.com/xwiki-labs/synapse_scripts) script. There's been past reports of it corrupting the Synapse database. Since there hasn't been any new development on it and it doesn't seem too useful nowadays, there's no point in including it in the playbook.
+
+If you need to clean up or compact your database, consider using the Synapse Admin APIs directly. See our [Synapse maintenance](docs/maintenance-synapse.md) and [Postgres maintenance](docs/maintenance-postgres.md) documentation pages for more details.
+
+
 ## Docker 20.10 is here
 
 (No need to do anything special in relation to this. Just something to keep in mind)
diff --git a/docs/maintenance-synapse.md b/docs/maintenance-synapse.md
index 79998299..143238c1 100644
--- a/docs/maintenance-synapse.md
+++ b/docs/maintenance-synapse.md
@@ -4,14 +4,11 @@ This document shows you how to perform various maintenance tasks related to the
 
 Table of contents:
 
-- [Purging unused data with synapse-janitor](#purging-unused-data-with-synapse-janitor), for when you wish to delete unused data from the Synapse database
-
 - [Purging old data with the Purge History API](#purging-old-data-with-the-purge-history-api), for when you wish to delete in-use (but old) data from the Synapse database
 
 - [Synapse maintenance](#synapse-maintenance)
 	- [Purging old data with the Purge History API](#purging-old-data-with-the-purge-history-api)
 	- [Compressing state with rust-synapse-compress-state](#compressing-state-with-rust-synapse-compress-state)
-	- [Purging unused data with synapse-janitor](#purging-unused-data-with-synapse-janitor)
 	- [Browse and manipulate the database](#browse-and-manipulate-the-database)
 
 - [Browse and manipulate the database](#browse-and-manipulate-the-database), for when you really need to take matters into your own hands
@@ -57,27 +54,6 @@ If you need to adjust this, pass: `--extra-vars='matrix_synapse_rust_synapse_com
 After state compression, you may wish to run a [`FULL` Postgres `VACUUM`](./maintenance-postgres.md#vacuuming-postgresql).
 
 
-## Purging unused data with synapse-janitor
-
-**NOTE**: There are [reports](https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/465) that **synapse-janitor is dangerous to use and causes database corruption**. You may wish to refrain from using it.
-
-When you **leave** and **forget** a room, Synapse can clean up its data, but currently doesn't.
-This **unused and unreachable data** remains in your database forever.
-
-There are external tools (like [synapse-janitor](https://github.com/xwiki-labs/synapse_scripts)), which are meant to solve this problem.
-
-To ask the playbook to run synapse-janitor, execute:
-
-```bash
-ansible-playbook -i inventory/hosts setup.yml --tags=run-postgres-synapse-janitor,start
-```
-
-**Note**: this will automatically stop Synapse temporarily and restart it later.
-
-Running synapse-janitor potentially deletes a lot of data from the Postgres database.
-You may wish to run a [`FULL` Postgres `VACUUM`](./maintenance-postgres.md#vacuuming-postgresql) after that.
-
-
 ## Browse and manipulate the database
 
 When the [matrix admin API](https://github.com/matrix-org/synapse/tree/master/docs/admin_api) and the other tools do not provide a more convenient way, having a look at synapse's postgresql database can satisfy a lot of admins' needs.
diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml
index 164509b7..e0522ba8 100644
--- a/roles/matrix-base/defaults/main.yml
+++ b/roles/matrix-base/defaults/main.yml
@@ -106,7 +106,6 @@ matrix_docker_package_name: docker-ce
 run_postgres_import: true
 run_postgres_upgrade: true
 run_postgres_import_sqlite_db: true
-run_postgres_synapse_janitor: true
 run_postgres_vacuum: true
 run_synapse_register_user: true
 run_synapse_update_user_password: true
diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml
index 0d2f50e3..ec5cb3dc 100644
--- a/roles/matrix-postgres/defaults/main.yml
+++ b/roles/matrix-postgres/defaults/main.yml
@@ -30,5 +30,3 @@ matrix_postgres_container_extra_arguments: []
 #
 # Takes an "<ip>:<port>" or "<port>" value (e.g. "127.0.0.1:5432"), or empty string to not expose.
 matrix_postgres_container_postgres_bind_port: ""
-
-matrix_postgres_tool_synapse_janitor: "https://raw.githubusercontent.com/xwiki-labs/synapse_scripts/a9188ff175ae581610f92d58ea6eac9a114d854b/synapse_janitor.sql"
diff --git a/roles/matrix-postgres/tasks/main.yml b/roles/matrix-postgres/tasks/main.yml
index 41b9c861..717b73e4 100644
--- a/roles/matrix-postgres/tasks/main.yml
+++ b/roles/matrix-postgres/tasks/main.yml
@@ -29,11 +29,6 @@
   tags:
     - upgrade-postgres
 
-- import_tasks: "{{ role_path }}/tasks/run_synapse_janitor.yml"
-  when: run_postgres_synapse_janitor|bool
-  tags:
-    - run-postgres-synapse-janitor
-
 - import_tasks: "{{ role_path }}/tasks/run_vacuum.yml"
   when: run_postgres_vacuum|bool
   tags:
diff --git a/roles/matrix-postgres/tasks/run_synapse_janitor.yml b/roles/matrix-postgres/tasks/run_synapse_janitor.yml
deleted file mode 100644
index d7f283be..00000000
--- a/roles/matrix-postgres/tasks/run_synapse_janitor.yml
+++ /dev/null
@@ -1,117 +0,0 @@
----
-
-# Pre-checks
-
-- name: Fail if Postgres not enabled
-  fail:
-    msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot run synapse-janitor."
-  when: "not matrix_postgres_enabled|bool"
-
-- name: Fail if not aware of the risks
-  fail:
-    msg: >-
-      Using Synapse Janitor is considered dangerous and may break your database.
-      See https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/465.
-      If you'd like to run it anyway, add `--extra-vars='i_know_synapse_janitor_is_dangerous=1'` to your command.
-  when: "i_know_synapse_janitor_is_dangerous|default('') == ''"
-
-# Defaults
-
-- name: Set postgres_start_wait_time, if not provided
-  set_fact:
-    postgres_start_wait_time: 15
-  when: "postgres_start_wait_time|default('') == ''"
-
-- name: Set postgres_synapse_janitor_wait_time, if not provided
-  set_fact:
-    postgres_synapse_janitor_wait_time: "{{ 7 * 86400 }}"
-  when: "postgres_synapse_janitor_wait_time|default('') == ''"
-
-- name: Set postgres_synapse_janitor_tool_path, if not provided
-  set_fact:
-    postgres_synapse_janitor_tool_path: "{{ matrix_postgres_base_path }}/synapse_janitor.sql"
-  when: "postgres_synapse_janitor_tool_path|default('') == ''"
-
-
-# Actual janitor work
-
-- name: Download synapse-janitor tool
-  get_url:
-    url: "{{ matrix_postgres_tool_synapse_janitor }}"
-    dest: "{{ postgres_synapse_janitor_tool_path }}"
-    force: true
-    mode: 0550
-    owner: "{{ matrix_user_username }}"
-    group: "{{ matrix_user_groupname }}"
-
-- name: Ensure matrix-postgres is started
-  service:
-    name: matrix-postgres
-    state: started
-    daemon_reload: yes
-
-- name: Wait a bit, so that Postgres can start
-  wait_for:
-    timeout: "{{ postgres_start_wait_time }}"
-  delegate_to: 127.0.0.1
-  become: false
-
-- import_tasks: tasks/util/detect_existing_postgres_version.yml
-
-- name: Abort, if no existing Postgres version detected
-  fail:
-    msg: "Could not find existing Postgres installation"
-  when: "not matrix_postgres_detected_existing|bool"
-
-- name: Generate Postgres database synapse-janitor command
-  set_fact:
-    matrix_postgres_synapse_janitor_command: >-
-      {{ matrix_host_command_docker }} run --rm --name matrix-postgres-synapse-janitor
-      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
-      --cap-drop=ALL
-      --network={{ matrix_docker_network }}
-      --env-file={{ matrix_postgres_base_path }}/env-postgres-psql
-      --mount type=bind,src={{ postgres_synapse_janitor_tool_path }},dst=/synapse_janitor.sql,ro=true
-      {{ matrix_postgres_docker_image_latest }}
-      psql -v ON_ERROR_STOP=1 -h matrix-postgres {{ matrix_synapse_database_database }} -f /synapse_janitor.sql
-
-- name: Note about Postgres purging alternative
-  debug:
-    msg: >-
-      Running synapse-janitor with the following Postgres command: `{{ matrix_postgres_synapse_janitor_command }}`.
-      If this crashes, you can stop all processes (`systemctl stop matrix-*`),
-      start Postgres only (`systemctl start matrix-postgres`)
-      and manually run the above command directly on the server.
-
-- name: Populate service facts
-  service_facts:
-
-- set_fact:
-    matrix_postgres_synapse_was_running: "{{ ansible_facts.services['matrix-synapse.service']|default(none) is not none and ansible_facts.services['matrix-synapse.service'].state == 'running' }}"
-
-- name: Ensure matrix-synapse is stopped
-  service:
-    name: matrix-synapse
-    state: stopped
-    daemon_reload: yes
-
-- name: Run synapse-janitor
-  command: "{{ matrix_postgres_synapse_janitor_command }}"
-  async: "{{ postgres_synapse_janitor_wait_time }}"
-  poll: 10
-  register: matrix_postgres_synapse_janitor_result
-
-# Intentionally show the results
-- debug: var="matrix_postgres_synapse_janitor_result"
-
-- name: Ensure matrix-synapse is started, if it previously was
-  service:
-    name: matrix-synapse
-    state: started
-    daemon_reload: yes
-  when: "matrix_postgres_synapse_was_running|bool"
-
-- name: Delete synapse-janitor tool
-  file:
-    path: "{{ postgres_synapse_janitor_tool_path }}"
-    state: absent

From d96d7f2a43306e9b74673c2d5f6930a87f44bd2a Mon Sep 17 00:00:00 2001
From: chagai95 <31655082+chagai95@users.noreply.github.com>
Date: Sat, 12 Dec 2020 00:24:38 +0100
Subject: [PATCH 039/179] Adding jitsi as well

---
 docs/configuring-playbook-ssl-certificates.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/configuring-playbook-ssl-certificates.md b/docs/configuring-playbook-ssl-certificates.md
index 5b5c7cdc..3dc65516 100644
--- a/docs/configuring-playbook-ssl-certificates.md
+++ b/docs/configuring-playbook-ssl-certificates.md
@@ -80,6 +80,7 @@ matrix_ssl_domains_to_obtain_certificates_for:
   - '{{ matrix_server_fqn_matrix }}'
   - '{{ matrix_server_fqn_element }}'
   - '{{ matrix_server_fqn_dimension }}'
+  - '{{ matrix_server_fqn_jitsi }}'
   - '{{ matrix_domain }}'
 ```
 

From bae411342204a42cab69710c33870fed02c33a4b Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sat, 12 Dec 2020 01:44:34 +0200
Subject: [PATCH 040/179] Mention that we possibly obtain a Jitsi certificate

---
 docs/configuring-playbook-ssl-certificates.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/configuring-playbook-ssl-certificates.md b/docs/configuring-playbook-ssl-certificates.md
index 3dc65516..7f05a5b2 100644
--- a/docs/configuring-playbook-ssl-certificates.md
+++ b/docs/configuring-playbook-ssl-certificates.md
@@ -67,6 +67,7 @@ By default, it obtains certificates for:
 - possibly for `element.<your-domain>`, unless you have disabled the [Element client component](configuring-playbook-client-element.md) using `matrix_client_element_enabled: false`
 - possibly for `riot.<your-domain>`, if you have explicitly enabled Riot to Element redirection (for background compatibility) using `matrix_nginx_proxy_proxy_riot_compat_redirect_enabled: true`
 - possibly for `dimension.<your-domain>`, if you have explicitly [set up Dimension](configuring-playbook-dimension.md).
+- possibly for `jitsi.<your-domain>`, if you have explicitly [set up Jitsi](configuring-playbook-jitsi.md).
 - possibly for your base domain (`<your-domain>`), if you have explicitly configured [Serving the base domain](configuring-playbook-base-domain-serving.md)
 
 If you are hosting other domains on the Matrix machine, you can make the playbook obtain and renew certificates for those other domains too.

From dac0d3a682d6a288dea3c22115c3d8385e79c720 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 13 Dec 2020 21:07:16 +0200
Subject: [PATCH 041/179] Add default matrix_postgres_additional_databases

---
 roles/matrix-postgres/defaults/main.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml
index ec5cb3dc..548cac15 100644
--- a/roles/matrix-postgres/defaults/main.yml
+++ b/roles/matrix-postgres/defaults/main.yml
@@ -30,3 +30,13 @@ matrix_postgres_container_extra_arguments: []
 #
 # Takes an "<ip>:<port>" or "<port>" value (e.g. "127.0.0.1:5432"), or empty string to not expose.
 matrix_postgres_container_postgres_bind_port: ""
+
+# A list of additional (databases and their credentials) to create.
+#
+# Example:
+# matrix_postgres_additional_databases:
+#   - name: matrix_appservice_discord
+#     pass: some-password
+#   - name: matrix_appservice_slack
+#     pass: some-password
+matrix_postgres_additional_databases: []

From 527d5f57d58ad7c1feba743fc99fe8508cc90534 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 13 Dec 2020 21:40:32 +0200
Subject: [PATCH 042/179] Relocate Postgres additional database creation logic

Moving it above the "uninstalling" set of tasks is better.
Extracting it out to another file at the same time, for readability,
especially given that it will probably have to become more complex in
the future (potentially installing `jq`, etc.)
---
 .../matrix-postgres/tasks/setup_postgres.yml  | 37 ++++---------------
 .../util/create_additional_databases.yml      | 28 ++++++++++++++
 2 files changed, 35 insertions(+), 30 deletions(-)
 create mode 100644 roles/matrix-postgres/tasks/util/create_additional_databases.yml

diff --git a/roles/matrix-postgres/tasks/setup_postgres.yml b/roles/matrix-postgres/tasks/setup_postgres.yml
index 85a8604c..6a0f9854 100644
--- a/roles/matrix-postgres/tasks/setup_postgres.yml
+++ b/roles/matrix-postgres/tasks/setup_postgres.yml
@@ -113,6 +113,13 @@
     daemon_reload: yes
   when: "matrix_postgres_enabled|bool and matrix_postgres_systemd_service_result.changed"
 
+- include_tasks:
+    file: "{{ role_path }}/tasks/util/create_additional_databases.yml"
+    apply:
+      tags:
+        - always
+  when: "matrix_postgres_enabled|bool"
+
 #
 # Tasks related to getting rid of the internal postgres server (if it was previously enabled)
 #
@@ -162,33 +169,3 @@
     - matrix-change-user-admin-status
     - matrix-postgres-update-user-password-hash
   when: "not matrix_postgres_enabled|bool"
-
-# Create additional databases
-- name: Retrieve IP of postgres container
-  shell: "docker inspect matrix-postgres | jq -r '.[0].NetworkSettings.Networks.{{ matrix_docker_network }}.IPAddress'"
-  register: matirx_postgres_container_ip
-
-- name: Create additional users in postgres
-  postgresql_user:
-    name: "{{ item.name }}"
-    password: "{{ item.pass }}"
-    login_host: "{{ matrx_postgres_container_ip.stdout }}"
-    login_port: 5432
-    login_user: "{{ matrix_postgres_connection_username }}"
-    login_password: "{{ matrix_postgres_connection_password }}"
-    login_db: "{{ matrix_postgres_db_name }}"
-  loop: matrix_postgres_additional_databases
-  when: matrix_postgres_enabed|bool
-
-- name: Create additional users in postgres
-  postgresql_db:
-    name: "{{ item.name }}"
-    owner: "{{ item.name }}"
-    lc_ctype: 'C'
-    lc_collate: 'C'
-    login_host: "{{ matrx_postgres_container_ip.stdout }}"
-    login_port: 5432
-    login_user: "{{ matrix_postgres_connection_username }}"
-    login_password: "{{ matrix_postgres_connection_password }}"
-  loop: matrix_postgres_additional_databases
-  when: matrix_postgres_enabled|bool
diff --git a/roles/matrix-postgres/tasks/util/create_additional_databases.yml b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
new file mode 100644
index 00000000..ec37ea63
--- /dev/null
+++ b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
@@ -0,0 +1,28 @@
+---
+
+- name: Retrieve IP of postgres container
+  shell: "docker inspect matrix-postgres | jq -r '.[0].NetworkSettings.Networks.{{ matrix_docker_network }}.IPAddress'"
+  register: matrix_postgres_container_ip
+
+- name: Create additional users in postgres
+  postgresql_user:
+    name: "{{ item.name }}"
+    password: "{{ item.pass }}"
+    login_host: "{{ matrix_postgres_container_ip.stdout }}"
+    login_port: 5432
+    login_user: "{{ matrix_postgres_connection_username }}"
+    login_password: "{{ matrix_postgres_connection_password }}"
+    login_db: "{{ matrix_postgres_db_name }}"
+  loop: matrix_postgres_additional_databases
+
+- name: Create additional users in postgres
+  postgresql_db:
+    name: "{{ item.name }}"
+    owner: "{{ item.name }}"
+    lc_ctype: 'C'
+    lc_collate: 'C'
+    login_host: "{{ matrix_postgres_container_ip.stdout }}"
+    login_port: 5432
+    login_user: "{{ matrix_postgres_connection_username }}"
+    login_password: "{{ matrix_postgres_connection_password }}"
+  loop: matrix_postgres_additional_databases

From 0641106370aed272e16cecaf2cf17754d30eb6c6 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 13 Dec 2020 21:43:53 +0200
Subject: [PATCH 043/179] Allow username of additional Postgres databases to be
 different

We'll most likely use one that matches the database name, but
it's better to have it configurable.
---
 group_vars/matrix_servers                       | 17 ++++++++++++++++-
 roles/matrix-postgres/defaults/main.yml         |  6 ++++--
 .../tasks/util/create_additional_databases.yml  |  4 ++--
 3 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 8a099275..f131ab9e 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -893,62 +893,77 @@ matrix_postgres_additional_databases: |
   {{
     ([{
       name: 'matrix_appservice_discord',
+      username: 'matrix_appservice_discord',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string
     }] if matrix_appservice_discord_enabled else [])
     + ([{
       name: 'matrix_appservice_slack'
+      username: 'matrix_appservice_slack',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string
     }] if matrix_appservice_slack_enabled else [])
     + ([{
       name: 'matrix_appservice_irc'
+      username: 'matrix_appservice_irc',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string
     }] if matrix_appservice_irc_enabled else [])
     + ([{
-      name: 'mautrix-bridge-facebook'
+      name: 'mautrix_bridge_facebook'
+      username: 'mautrix_bridge_facebook',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string
     }] if matrix_mautrix_facebook_enabled else [])
     + ([{
       name: 'mautrix_bridge_hangouts'
+      username: 'mautrix_bridge_hangouts',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string
     }] if matrix_mautrix_hangouts_enabled else [])
     + ([{
       name: 'mautrix_bridge_telegram'
+      username: 'mautrix_bridge_telegram',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string
     }] if matrix_mautrix_telegram_enabled else [])
     + ([{
       name: 'mautrix_bridge_whatsapp'
+      username: 'mautrix_bridge_whatsapp',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string
     }] if matrix_mautrix_whatsapp_enabled else [])
     + ([{
       name: 'matrix_bridge_sms'
+      username: 'matrix_bridge_sms',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string
     }] if matrix_sms_bridge_enabled else [])
     + ([{
       name: 'matrix_puppet_skype'
+      username: 'matrix_puppet_skype',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string
     }] if matrix_mx_puppet_skype_enabled else [])
     + ([{
       name: 'matrix_puppet_slack'
+      username: 'matrix_puppet_slack',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string
     }] if matrix_mx_puppet_slack_enabled else [])
     + ([{
       name: 'matrix_puppet_twitter'
+      username: 'matrix_puppet_twitter',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string
     }] if matrix_mx_puppet_twitter_enabled else [])
     + ([{
       name: 'matrix_puppet_instagram'
+      username: 'matrix_puppet_instagram',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string
     ] if matrix_mx_puppet_instagram_enabled else [])
     + ([{
       name: 'matrix_puppet_discord'
+      username: 'matrix_puppet_discord',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string
     }] if matrix_mx_puppet_discord_enabled else [])
     + ([{
       name: 'matrix_puppet_steam'
+      username: 'matrix_puppet_steam',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string
     }] if matrix_mx_puppet_steam_enabled else [])
     + ([{
       name: 'matrix_dimension'
+      username: 'matrix_dimension',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string
     }] if matrix_dimension_enabled else [])
   }}
diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml
index 548cac15..2645d4b0 100644
--- a/roles/matrix-postgres/defaults/main.yml
+++ b/roles/matrix-postgres/defaults/main.yml
@@ -36,7 +36,9 @@ matrix_postgres_container_postgres_bind_port: ""
 # Example:
 # matrix_postgres_additional_databases:
 #   - name: matrix_appservice_discord
-#     pass: some-password
+#     username: matrix_appservice_discord
+#     pass: some_password
 #   - name: matrix_appservice_slack
-#     pass: some-password
+#     username: matrix_appservice_slack
+#     pass: some_password
 matrix_postgres_additional_databases: []
diff --git a/roles/matrix-postgres/tasks/util/create_additional_databases.yml b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
index ec37ea63..0532c99d 100644
--- a/roles/matrix-postgres/tasks/util/create_additional_databases.yml
+++ b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
@@ -6,7 +6,7 @@
 
 - name: Create additional users in postgres
   postgresql_user:
-    name: "{{ item.name }}"
+    name: "{{ item.username }}"
     password: "{{ item.pass }}"
     login_host: "{{ matrix_postgres_container_ip.stdout }}"
     login_port: 5432
@@ -18,7 +18,7 @@
 - name: Create additional users in postgres
   postgresql_db:
     name: "{{ item.name }}"
-    owner: "{{ item.name }}"
+    owner: "{{ item.username }}"
     lc_ctype: 'C'
     lc_collate: 'C'
     login_host: "{{ matrix_postgres_container_ip.stdout }}"

From d251764c16303dea542326746a0e13a46d13fda2 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 13 Dec 2020 22:19:54 +0200
Subject: [PATCH 044/179] Fix syntax issues in
 matrix_postgres_additional_databases

Quotes are necessary around dictionary field names.

There was a missing `}` as well.
---
 group_vars/matrix_servers | 92 +++++++++++++++++++--------------------
 1 file changed, 46 insertions(+), 46 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index f131ab9e..11c87a64 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -892,79 +892,79 @@ matrix_postgres_db_name: "homeserver"
 matrix_postgres_additional_databases: |
   {{
     ([{
-      name: 'matrix_appservice_discord',
-      username: 'matrix_appservice_discord',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string
+      'name': 'matrix_appservice_discord',
+      'username': 'matrix_appservice_discord',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string
     }] if matrix_appservice_discord_enabled else [])
     + ([{
-      name: 'matrix_appservice_slack'
-      username: 'matrix_appservice_slack',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string
+      'name': 'matrix_appservice_slack',
+      'username': 'matrix_appservice_slack',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string
     }] if matrix_appservice_slack_enabled else [])
     + ([{
-      name: 'matrix_appservice_irc'
-      username: 'matrix_appservice_irc',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string
+      'name': 'matrix_appservice_irc',
+      'username': 'matrix_appservice_irc',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string
     }] if matrix_appservice_irc_enabled else [])
     + ([{
-      name: 'mautrix_bridge_facebook'
-      username: 'mautrix_bridge_facebook',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string
+      'name': 'mautrix_bridge_facebook',
+      'username': 'mautrix_bridge_facebook',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string
     }] if matrix_mautrix_facebook_enabled else [])
     + ([{
-      name: 'mautrix_bridge_hangouts'
-      username: 'mautrix_bridge_hangouts',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string
+      'name': 'mautrix_bridge_hangouts',
+      'username': 'mautrix_bridge_hangouts',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string
     }] if matrix_mautrix_hangouts_enabled else [])
     + ([{
-      name: 'mautrix_bridge_telegram'
-      username: 'mautrix_bridge_telegram',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string
+      'name': 'mautrix_bridge_telegram',
+      'username': 'mautrix_bridge_telegram',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string
     }] if matrix_mautrix_telegram_enabled else [])
     + ([{
-      name: 'mautrix_bridge_whatsapp'
-      username: 'mautrix_bridge_whatsapp',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string
+      'name': 'mautrix_bridge_whatsapp',
+      'username': 'mautrix_bridge_whatsapp',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string
     }] if matrix_mautrix_whatsapp_enabled else [])
     + ([{
-      name: 'matrix_bridge_sms'
-      username: 'matrix_bridge_sms',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string
+      'name': 'matrix_bridge_sms',
+      'username': 'matrix_bridge_sms',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string
     }] if matrix_sms_bridge_enabled else [])
     + ([{
-      name: 'matrix_puppet_skype'
-      username: 'matrix_puppet_skype',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string
+      'name': 'matrix_puppet_skype',
+      'username': 'matrix_puppet_skype',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string
     }] if matrix_mx_puppet_skype_enabled else [])
     + ([{
-      name: 'matrix_puppet_slack'
-      username: 'matrix_puppet_slack',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string
+      'name': 'matrix_puppet_slack',
+      'username': 'matrix_puppet_slack',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string
     }] if matrix_mx_puppet_slack_enabled else [])
     + ([{
-      name: 'matrix_puppet_twitter'
-      username: 'matrix_puppet_twitter',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string
+      'name': 'matrix_puppet_twitter',
+      'username': 'matrix_puppet_twitter',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string
     }] if matrix_mx_puppet_twitter_enabled else [])
     + ([{
-      name: 'matrix_puppet_instagram'
-      username: 'matrix_puppet_instagram',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string
-    ] if matrix_mx_puppet_instagram_enabled else [])
+      'name': 'matrix_puppet_instagram',
+      'username': 'matrix_puppet_instagram',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string
+    }] if matrix_mx_puppet_instagram_enabled else [])
     + ([{
-      name: 'matrix_puppet_discord'
-      username: 'matrix_puppet_discord',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string
+      'name': 'matrix_puppet_discord',
+      'username': 'matrix_puppet_discord',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string
     }] if matrix_mx_puppet_discord_enabled else [])
     + ([{
-      name: 'matrix_puppet_steam'
-      username: 'matrix_puppet_steam',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string
+      'name': 'matrix_puppet_steam',
+      'username': 'matrix_puppet_steam',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string
     }] if matrix_mx_puppet_steam_enabled else [])
     + ([{
-      name: 'matrix_dimension'
-      username: 'matrix_dimension',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string
+      'name': 'matrix_dimension',
+      'username': 'matrix_dimension',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string
     }] if matrix_dimension_enabled else [])
   }}
 

From f47e8a97e6629da02a45d9d75bde3a82ef057bd6 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 13 Dec 2020 22:38:35 +0200
Subject: [PATCH 045/179] Make use of matrix_host_command_docker instead of
 hardcoding

---
 .../matrix-postgres/tasks/util/create_additional_databases.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/roles/matrix-postgres/tasks/util/create_additional_databases.yml b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
index 0532c99d..424be626 100644
--- a/roles/matrix-postgres/tasks/util/create_additional_databases.yml
+++ b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
@@ -1,7 +1,6 @@
 ---
-
 - name: Retrieve IP of postgres container
-  shell: "docker inspect matrix-postgres | jq -r '.[0].NetworkSettings.Networks.{{ matrix_docker_network }}.IPAddress'"
+  shell: "{{ matrix_host_command_docker }} inspect matrix-postgres | jq -r '.[0].NetworkSettings.Networks.{{ matrix_docker_network }}.IPAddress'"
   register: matrix_postgres_container_ip
 
 - name: Create additional users in postgres

From e2952f16f7a97b2883ea4c00c6bc4451c71c785d Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 13 Dec 2020 22:45:48 +0200
Subject: [PATCH 046/179] Determine matrix-postgres IP address without relying
 on jq

To avoid needing to have `jq` installed on the machine, we could:
- try to run jq in a Docker container using some small image providing
that
- better yet, avoid `jq` altogether
---
 .../matrix-postgres/tasks/util/create_additional_databases.yml  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-postgres/tasks/util/create_additional_databases.yml b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
index 424be626..0864d8f8 100644
--- a/roles/matrix-postgres/tasks/util/create_additional_databases.yml
+++ b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
@@ -1,6 +1,6 @@
 ---
 - name: Retrieve IP of postgres container
-  shell: "{{ matrix_host_command_docker }} inspect matrix-postgres | jq -r '.[0].NetworkSettings.Networks.{{ matrix_docker_network }}.IPAddress'"
+  command: "{{ matrix_host_command_docker }} inspect matrix-postgres --format='{% raw %}{{ .NetworkSettings.Networks.{% endraw %}{{ matrix_docker_network }}{% raw %}.IPAddress }}{% endraw %}'"
   register: matrix_postgres_container_ip
 
 - name: Create additional users in postgres

From c765ceb270b7377e799a32db383eef39257d43e8 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 13 Dec 2020 22:56:56 +0200
Subject: [PATCH 047/179] Prevent weird loop error

> Invalid data passed to 'loop', it requires a list, got this instead: matrix_postgres_additional_databases. Hint: If you passed a list/dict of just one element, try adding wantlist=True to your lookup invocation or use q/query instead of lookup.

Well, or working around it, as I've done in this commit (which seems
more sane than `wantlist=True` stuff).
---
 .../tasks/util/create_additional_databases.yml               | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/roles/matrix-postgres/tasks/util/create_additional_databases.yml b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
index 0864d8f8..0b40cabf 100644
--- a/roles/matrix-postgres/tasks/util/create_additional_databases.yml
+++ b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
@@ -1,4 +1,5 @@
 ---
+
 - name: Retrieve IP of postgres container
   command: "{{ matrix_host_command_docker }} inspect matrix-postgres --format='{% raw %}{{ .NetworkSettings.Networks.{% endraw %}{{ matrix_docker_network }}{% raw %}.IPAddress }}{% endraw %}'"
   register: matrix_postgres_container_ip
@@ -12,7 +13,7 @@
     login_user: "{{ matrix_postgres_connection_username }}"
     login_password: "{{ matrix_postgres_connection_password }}"
     login_db: "{{ matrix_postgres_db_name }}"
-  loop: matrix_postgres_additional_databases
+  loop: "{{ matrix_postgres_additional_databases }}"
 
 - name: Create additional users in postgres
   postgresql_db:
@@ -24,4 +25,4 @@
     login_port: 5432
     login_user: "{{ matrix_postgres_connection_username }}"
     login_password: "{{ matrix_postgres_connection_password }}"
-  loop: matrix_postgres_additional_databases
+  loop: "{{ matrix_postgres_additional_databases }}"

From bbc09d013b8037e784004363b7bdfdcc6e8d13c7 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 13 Dec 2020 23:46:02 +0200
Subject: [PATCH 048/179] Do not execute additional databases creation code if
 not necessary

The tasks in `create_additional_databases.yml` will likely
ensure `matrix-postgres.service` is started, etc.

If no additional databases are defined, we'd rather not execute that
file and all these tasks that it may do in the future.
---
 roles/matrix-postgres/tasks/setup_postgres.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-postgres/tasks/setup_postgres.yml b/roles/matrix-postgres/tasks/setup_postgres.yml
index 6a0f9854..518d1a5f 100644
--- a/roles/matrix-postgres/tasks/setup_postgres.yml
+++ b/roles/matrix-postgres/tasks/setup_postgres.yml
@@ -118,7 +118,7 @@
     apply:
       tags:
         - always
-  when: "matrix_postgres_enabled|bool"
+  when: "matrix_postgres_enabled|bool and matrix_postgres_additional_databases|length > 0"
 
 #
 # Tasks related to getting rid of the internal postgres server (if it was previously enabled)

From da4cb2f63998c2c994776b3520303e90b735157c Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 00:25:13 +0200
Subject: [PATCH 049/179] Do not use the postgresql_user/postgresql_db modules

While these modules are really nice and helpful, we can't use them
for at least 2 reasons:

- for us, Postgres runs in a container on a private Docker network
(`--network=matrix`) without usually being exposed to the host.
These modules execute on the host so they won't be able to reach it.

- these modules require `psycopg2`, so we need to install it before
using it. This might or might not be its own can of worms.
---
 .../tasks/util/create_additional_database.yml | 34 ++++++++++++++++++
 .../util/create_additional_databases.yml      | 35 ++++++-------------
 .../init-additional-db-user-and-role.sql.j2   | 19 ++++++++++
 3 files changed, 63 insertions(+), 25 deletions(-)
 create mode 100644 roles/matrix-postgres/tasks/util/create_additional_database.yml
 create mode 100644 roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2

diff --git a/roles/matrix-postgres/tasks/util/create_additional_database.yml b/roles/matrix-postgres/tasks/util/create_additional_database.yml
new file mode 100644
index 00000000..2da505eb
--- /dev/null
+++ b/roles/matrix-postgres/tasks/util/create_additional_database.yml
@@ -0,0 +1,34 @@
+---
+
+# TODO - ensure `additional_db` contains all keys that we expect
+
+# The SQL statements that we'll run against Postgres are stored in a file that others can't read.
+# This file will be mounted into the container and fed to Postgres.
+# This way, we avoid passing sensitive data around in CLI commands that other users on the system can see.
+- name: Create additional database initialization SQL file for {{ additional_db.name }}
+  template:
+    src: "{{ role_path }}/templates/init-additional-db-user-and-role.sql.j2"
+    dest: "/tmp/matrix-postgres-init-additional-db-user-and-role.sql"
+    mode: 0600
+    owner: "{{ matrix_user_uid }}"
+    group: "{{ matrix_user_gid }}"
+
+- name: Execute Postgres additional database initialization SQL file for {{ additional_db.name }}
+  command:
+    cmd: >-
+      {{ matrix_host_command_docker }} run
+      --rm
+      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
+      --cap-drop=ALL
+      --env-file={{ matrix_postgres_base_path }}/env-postgres-psql
+      --network {{ matrix_docker_network }}
+      --mount type=bind,src=/tmp/matrix-postgres-init-additional-db-user-and-role.sql,dst=/matrix-postgres-init-additional-db-user-and-role.sql,ro
+      --entrypoint=/bin/sh
+      {{ matrix_postgres_docker_image_to_use }}
+      -c
+      'psql -h {{ matrix_postgres_connection_hostname }} --file=/matrix-postgres-init-additional-db-user-and-role.sql'
+
+- name: Delete additional database initialization SQL file for {{ additional_db.name }}
+  file:
+    path: /tmp/matrix-postgres-init-additional-db-user-and-role.sql
+    state: absent
diff --git a/roles/matrix-postgres/tasks/util/create_additional_databases.yml b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
index 0b40cabf..51deb228 100644
--- a/roles/matrix-postgres/tasks/util/create_additional_databases.yml
+++ b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
@@ -1,28 +1,13 @@
 ---
 
-- name: Retrieve IP of postgres container
-  command: "{{ matrix_host_command_docker }} inspect matrix-postgres --format='{% raw %}{{ .NetworkSettings.Networks.{% endraw %}{{ matrix_docker_network }}{% raw %}.IPAddress }}{% endraw %}'"
-  register: matrix_postgres_container_ip
+# TODO - we should ensure matrix-postgres.service is started.
+# .. and that if we had just started it, we've given it ample time to initialize,
+# before we attempt to run queries against it.
 
-- name: Create additional users in postgres
-  postgresql_user:
-    name: "{{ item.username }}"
-    password: "{{ item.pass }}"
-    login_host: "{{ matrix_postgres_container_ip.stdout }}"
-    login_port: 5432
-    login_user: "{{ matrix_postgres_connection_username }}"
-    login_password: "{{ matrix_postgres_connection_password }}"
-    login_db: "{{ matrix_postgres_db_name }}"
-  loop: "{{ matrix_postgres_additional_databases }}"
-
-- name: Create additional users in postgres
-  postgresql_db:
-    name: "{{ item.name }}"
-    owner: "{{ item.username }}"
-    lc_ctype: 'C'
-    lc_collate: 'C'
-    login_host: "{{ matrix_postgres_container_ip.stdout }}"
-    login_port: 5432
-    login_user: "{{ matrix_postgres_connection_username }}"
-    login_password: "{{ matrix_postgres_connection_password }}"
-  loop: "{{ matrix_postgres_additional_databases }}"
+- name: Create additional Postgers user and database
+  include_tasks: "{{ role_path }}/tasks/util/create_additional_database.yml"
+  with_items: "{{ matrix_postgres_additional_databases }}"
+  loop_control:
+    loop_var: additional_db
+  # Suppress logging to avoid dumping the credentials to the shell
+  no_log: true
diff --git a/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2 b/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2
new file mode 100644
index 00000000..732b0187
--- /dev/null
+++ b/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2
@@ -0,0 +1,19 @@
+-- `CREATE USER` does not support `IF NOT EXISTS`, so we use this workaround to prevent an error and raise a notice instead.
+-- Seen here: https://stackoverflow.com/a/49858797
+DO $$
+BEGIN
+  CREATE USER {{ additional_db.username }};
+  EXCEPTION WHEN DUPLICATE_OBJECT THEN
+  RAISE NOTICE 'not creating role {{ additional_db.username }}, since it already exists';
+END
+$$;
+
+-- This is useful for initial user creation (since we don't assign a password above) and for handling subsequent password changes
+-- TODO - we should escape quotes in the password.
+ALTER ROLE {{ additional_db.username }} PASSWORD '{{ additional_db.pass }}';
+
+-- This will generate an error on subsequent execution
+CREATE DATABASE {{ additional_db.name }} WITH LC_CTYPE 'C' LC_COLLATE 'C' OWNER {{ additional_db.username }};
+
+-- This is useful for changing the database owner subsequently
+ALTER DATABASE {{ additional_db.name }} OWNER TO {{ additional_db.username }};

From 3a037a59935002729dfe7da742721111956eaf0b Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 00:39:38 +0200
Subject: [PATCH 050/179] Ensure additional databases contain all the keys that
 we expect

---
 .../tasks/util/create_additional_database.yml             | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/roles/matrix-postgres/tasks/util/create_additional_database.yml b/roles/matrix-postgres/tasks/util/create_additional_database.yml
index 2da505eb..a994cc26 100644
--- a/roles/matrix-postgres/tasks/util/create_additional_database.yml
+++ b/roles/matrix-postgres/tasks/util/create_additional_database.yml
@@ -1,6 +1,12 @@
 ---
 
-# TODO - ensure `additional_db` contains all keys that we expect
+# It'd be better if this is belonged to `validate_config.yml`, but it would have to be some loop-within-a-loop there,
+# and that's ugly. We also don't expect this to catch errors often. It's more of a defensive last-minute check.
+- name: Fail if additional database data appears invalid
+  fail:
+    msg: "Additional database definition ({{ additional_db }} lacks a required key: {{ item }}"
+  when: "item not in additional_db"
+  with_items: "{{ ['name', 'username', 'pass'] }}"
 
 # The SQL statements that we'll run against Postgres are stored in a file that others can't read.
 # This file will be mounted into the container and fed to Postgres.

From 46a4034d3ec0b8ae331119e249fc584f7be2b6ed Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 00:43:03 +0200
Subject: [PATCH 051/179] Use "password" for additional Postgres databases, not
 "pass"

Being more explicit sounds better.
---
 group_vars/matrix_servers                     | 30 +++++++++----------
 roles/matrix-postgres/defaults/main.yml       |  4 +--
 .../tasks/util/create_additional_database.yml |  2 +-
 .../init-additional-db-user-and-role.sql.j2   |  2 +-
 4 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 11c87a64..3169e18e 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -894,77 +894,77 @@ matrix_postgres_additional_databases: |
     ([{
       'name': 'matrix_appservice_discord',
       'username': 'matrix_appservice_discord',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string,
     }] if matrix_appservice_discord_enabled else [])
     + ([{
       'name': 'matrix_appservice_slack',
       'username': 'matrix_appservice_slack',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string,
     }] if matrix_appservice_slack_enabled else [])
     + ([{
       'name': 'matrix_appservice_irc',
       'username': 'matrix_appservice_irc',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string,
     }] if matrix_appservice_irc_enabled else [])
     + ([{
       'name': 'mautrix_bridge_facebook',
       'username': 'mautrix_bridge_facebook',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string,
     }] if matrix_mautrix_facebook_enabled else [])
     + ([{
       'name': 'mautrix_bridge_hangouts',
       'username': 'mautrix_bridge_hangouts',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string,
     }] if matrix_mautrix_hangouts_enabled else [])
     + ([{
       'name': 'mautrix_bridge_telegram',
       'username': 'mautrix_bridge_telegram',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string,
     }] if matrix_mautrix_telegram_enabled else [])
     + ([{
       'name': 'mautrix_bridge_whatsapp',
       'username': 'mautrix_bridge_whatsapp',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string,
     }] if matrix_mautrix_whatsapp_enabled else [])
     + ([{
       'name': 'matrix_bridge_sms',
       'username': 'matrix_bridge_sms',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string,
     }] if matrix_sms_bridge_enabled else [])
     + ([{
       'name': 'matrix_puppet_skype',
       'username': 'matrix_puppet_skype',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string,
     }] if matrix_mx_puppet_skype_enabled else [])
     + ([{
       'name': 'matrix_puppet_slack',
       'username': 'matrix_puppet_slack',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string,
     }] if matrix_mx_puppet_slack_enabled else [])
     + ([{
       'name': 'matrix_puppet_twitter',
       'username': 'matrix_puppet_twitter',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string,
     }] if matrix_mx_puppet_twitter_enabled else [])
     + ([{
       'name': 'matrix_puppet_instagram',
       'username': 'matrix_puppet_instagram',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string,
     }] if matrix_mx_puppet_instagram_enabled else [])
     + ([{
       'name': 'matrix_puppet_discord',
       'username': 'matrix_puppet_discord',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string,
     }] if matrix_mx_puppet_discord_enabled else [])
     + ([{
       'name': 'matrix_puppet_steam',
       'username': 'matrix_puppet_steam',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string,
     }] if matrix_mx_puppet_steam_enabled else [])
     + ([{
       'name': 'matrix_dimension',
       'username': 'matrix_dimension',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string,
     }] if matrix_dimension_enabled else [])
   }}
 
diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml
index 2645d4b0..ad417e0b 100644
--- a/roles/matrix-postgres/defaults/main.yml
+++ b/roles/matrix-postgres/defaults/main.yml
@@ -37,8 +37,8 @@ matrix_postgres_container_postgres_bind_port: ""
 # matrix_postgres_additional_databases:
 #   - name: matrix_appservice_discord
 #     username: matrix_appservice_discord
-#     pass: some_password
+#     password: some_password
 #   - name: matrix_appservice_slack
 #     username: matrix_appservice_slack
-#     pass: some_password
+#     password: some_password
 matrix_postgres_additional_databases: []
diff --git a/roles/matrix-postgres/tasks/util/create_additional_database.yml b/roles/matrix-postgres/tasks/util/create_additional_database.yml
index a994cc26..ce064d59 100644
--- a/roles/matrix-postgres/tasks/util/create_additional_database.yml
+++ b/roles/matrix-postgres/tasks/util/create_additional_database.yml
@@ -6,7 +6,7 @@
   fail:
     msg: "Additional database definition ({{ additional_db }} lacks a required key: {{ item }}"
   when: "item not in additional_db"
-  with_items: "{{ ['name', 'username', 'pass'] }}"
+  with_items: "{{ ['name', 'username', 'password'] }}"
 
 # The SQL statements that we'll run against Postgres are stored in a file that others can't read.
 # This file will be mounted into the container and fed to Postgres.
diff --git a/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2 b/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2
index 732b0187..af3d912c 100644
--- a/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2
+++ b/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2
@@ -10,7 +10,7 @@ $$;
 
 -- This is useful for initial user creation (since we don't assign a password above) and for handling subsequent password changes
 -- TODO - we should escape quotes in the password.
-ALTER ROLE {{ additional_db.username }} PASSWORD '{{ additional_db.pass }}';
+ALTER ROLE {{ additional_db.username }} PASSWORD '{{ additional_db.password }}';
 
 -- This will generate an error on subsequent execution
 CREATE DATABASE {{ additional_db.name }} WITH LC_CTYPE 'C' LC_COLLATE 'C' OWNER {{ additional_db.username }};

From a374d309c8f5d6038bfd5f94a66c3cd9a68c7e23 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 00:52:25 +0200
Subject: [PATCH 052/179] Make appservice-discord support both SQLite and
 Postgres

People can toggle between them now. The playbook also defaults
to using SQLite if an external Postgres server is used.

Ideally, we'd be able to create databases/users in external Postgres
servers as well, but our initialization logic (and `docker run` command,
etc.) hardcode too many things right now.
---
 group_vars/matrix_servers                        | 13 +++++++++----
 .../defaults/main.yml                            | 16 ++++++++++++++++
 .../tasks/validate_config.yml                    |  4 ++++
 .../templates/config.yaml.j2                     |  7 +++++--
 4 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 3169e18e..717e71bc 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -67,6 +67,11 @@ matrix_appservice_discord_appservice_token: "{{ matrix_synapse_macaroon_secret_k
 
 matrix_appservice_discord_homeserver_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'discord.hs.token') | to_uuid }}"
 
+# We only make this use Postgres if our own Postgres server is enabled.
+# It's only then (for now) that we can automatically create the necessary database and user for this service.
+matrix_appservice_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_appservice_discord_database_connString_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.discord.db') }}"
+
 ######################################################################
 #
 # /matrix-bridge-appservice-discord
@@ -892,10 +897,10 @@ matrix_postgres_db_name: "homeserver"
 matrix_postgres_additional_databases: |
   {{
     ([{
-      'name': 'matrix_appservice_discord',
-      'username': 'matrix_appservice_discord',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string,
-    }] if matrix_appservice_discord_enabled else [])
+      'name': matrix_appservice_discord_database_connString_db_name,
+      'username': matrix_appservice_discord_database_connString_username,
+      'password': matrix_appservice_discord_database_connString_password,
+    }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_database_connString_hostname == 'matrix-postgres') else [])
     + ([{
       'name': 'matrix_appservice_slack',
       'username': 'matrix_appservice_slack',
diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml
index 357b93d6..36b39b8e 100644
--- a/roles/matrix-bridge-appservice-discord/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml
@@ -41,6 +41,22 @@ matrix_appservice_discord_bridge_homeserverUrl: "http://matrix-synapse:8008"
 matrix_appservice_discord_bridge_disablePresence: false
 matrix_appservice_discord_bridge_enableSelfServiceBridging: false
 
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_appservice_discord_database_engine: 'postgres'`)
+# - adjust your database credentials
+matrix_appservice_discord_database_engine: 'sqlite'
+matrix_appservice_discord_database_filename: "/data/discord.db"
+matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_connString_username }}:{{ matrix_appservice_discord_database_connString_password }}@{{ matrix_appservice_discord_database_connString_hostname }}:{{ matrix_appservice_discord_database_connString_port }}/{{ matrix_appservice_discord_database_connString_db_name }}'
+matrix_appservice_discord_database_connString_username: 'matrix_appservice_discord'
+matrix_appservice_discord_database_connString_password: 'some-password'
+matrix_appservice_discord_database_connString_hostname: 'matrix-postgres'
+matrix_appservice_discord_database_connString_port: 5432
+matrix_appservice_discord_database_connString_db_name: 'matrix_appservice_discord'
+
 # Tells whether the bot should make use of "Privileged Gateway Intents".
 #
 # Enabling this means that you need to enable it for the bot (Discord application) as well,
diff --git a/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml b/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml
index b0713a43..73253ba0 100644
--- a/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml
+++ b/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml
@@ -20,3 +20,7 @@
   when: "item.old in vars"
   with_items:
     - {'old': 'matrix_appservice_discord_container_expose_client_server_api_port', 'new': '<superseded by matrix_appservice_discord_container_http_host_bind_port>'}
+
+- name: Require a valid database engine
+  fail: msg="`matrix_appservice_discord_database_engine` needs to be either 'sqlite' or 'postgres'"
+  when: "matrix_appservice_discord_database_engine not in ['sqlite', 'postgres']"
diff --git a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2
index de4182d9..b2ecd198 100644
--- a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2
+++ b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2
@@ -58,8 +58,11 @@ database:
   # If you are migrating, see https://github.com/Half-Shot/matrix-appservice-discord/blob/master/docs/howto.md#migrate-to-postgres-from-sqlite
   # WARNING: You will almost certainly be fine with sqlite unless your bridge
   # is in heavy demand and you suffer from IO slowness.
-  #filename: "/data/discord.db"
-  connString: "postgresql://matrix_appservice_discord:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_appservice_discord') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_discord"
+  {% if matrix_appservice_discord_database_engine == 'sqlite' %}
+  filename: {{ matrix_appservice_discord_database_filename|to_json }}
+  {% else %}
+  connString: {{ matrix_appservice_discord_database_connString|to_json }}
+  {% endif %}
 room:
   # Set the default visibility of alias rooms, defaults to "public".
   # One of: "public", "private"

From 183d2a10dbdfc142ace8e88e515e2d2f60ab84bc Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 00:59:59 +0200
Subject: [PATCH 053/179] Ensure matrix-postgres.service is started before
 creating additional users/databases

---
 roles/matrix-postgres/defaults/main.yml        |  6 ++++++
 .../tasks/util/create_additional_databases.yml | 18 ++++++++++++++----
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml
index ad417e0b..89559f5d 100644
--- a/roles/matrix-postgres/defaults/main.yml
+++ b/roles/matrix-postgres/defaults/main.yml
@@ -42,3 +42,9 @@ matrix_postgres_container_postgres_bind_port: ""
 #     username: matrix_appservice_slack
 #     password: some_password
 matrix_postgres_additional_databases: []
+
+# The number of seconds to wait after starting `matrix-postgres.service`
+# and before trying to run queries for creating additional databases/users against it.
+#
+# For most (subsequent) runs, Postgres would already be running, so no waiting will be happening at all.
+matrix_postgres_additional_databases_postgres_start_wait_timeout_seconds: 15
diff --git a/roles/matrix-postgres/tasks/util/create_additional_databases.yml b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
index 51deb228..0ad460dd 100644
--- a/roles/matrix-postgres/tasks/util/create_additional_databases.yml
+++ b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
@@ -1,10 +1,20 @@
 ---
 
-# TODO - we should ensure matrix-postgres.service is started.
-# .. and that if we had just started it, we've given it ample time to initialize,
-# before we attempt to run queries against it.
+- name: Ensure matrix-postgres is started
+  service:
+    name: matrix-postgres
+    state: started
+    daemon_reload: yes
+  register: matrix_postgres_service_start_result
 
-- name: Create additional Postgers user and database
+- name: Wait a bit, so that Postgres can start
+  wait_for:
+    timeout: "{{ matrix_postgres_additional_databases_postgres_start_wait_timeout_seconds }}"
+  delegate_to: 127.0.0.1
+  become: false
+  when: "matrix_postgres_service_start_result.changed|bool"
+
+- name: Create additional Postgres user and database
   include_tasks: "{{ role_path }}/tasks/util/create_additional_database.yml"
   with_items: "{{ matrix_postgres_additional_databases }}"
   loop_control:

From dd994995bca0cb0bd162aec1f5ee4dae35848614 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 01:22:23 +0200
Subject: [PATCH 054/179] Simplify password for additional Postgres databases

Using the result of `password_hash` works for creating them,
but authentication seems to be failing with some tools like pgloader.

It's possible that we're not escaping things properly somewhere.
Ideally, it'd be nice to solve that. But the easier (and still
relatively safe/good) solution is to just turn that password hash
into a UUID that's safe for passing around without worrying about
escaping.
---
 group_vars/matrix_servers | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 717e71bc..b9d0be26 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -70,7 +70,7 @@ matrix_appservice_discord_homeserver_token: "{{ matrix_synapse_macaroon_secret_k
 # We only make this use Postgres if our own Postgres server is enabled.
 # It's only then (for now) that we can automatically create the necessary database and user for this service.
 matrix_appservice_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_appservice_discord_database_connString_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.discord.db') }}"
+matrix_appservice_discord_database_connString_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.discord.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -904,72 +904,72 @@ matrix_postgres_additional_databases: |
     + ([{
       'name': 'matrix_appservice_slack',
       'username': 'matrix_appservice_slack',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | to_uuid,
     }] if matrix_appservice_slack_enabled else [])
     + ([{
       'name': 'matrix_appservice_irc',
       'username': 'matrix_appservice_irc',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | to_uuid,
     }] if matrix_appservice_irc_enabled else [])
     + ([{
       'name': 'mautrix_bridge_facebook',
       'username': 'mautrix_bridge_facebook',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | to_uuid,
     }] if matrix_mautrix_facebook_enabled else [])
     + ([{
       'name': 'mautrix_bridge_hangouts',
       'username': 'mautrix_bridge_hangouts',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | to_uuid,
     }] if matrix_mautrix_hangouts_enabled else [])
     + ([{
       'name': 'mautrix_bridge_telegram',
       'username': 'mautrix_bridge_telegram',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | to_uuid,
     }] if matrix_mautrix_telegram_enabled else [])
     + ([{
       'name': 'mautrix_bridge_whatsapp',
       'username': 'mautrix_bridge_whatsapp',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | to_uuid,
     }] if matrix_mautrix_whatsapp_enabled else [])
     + ([{
       'name': 'matrix_bridge_sms',
       'username': 'matrix_bridge_sms',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid,
     }] if matrix_sms_bridge_enabled else [])
     + ([{
       'name': 'matrix_puppet_skype',
       'username': 'matrix_puppet_skype',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | to_uuid,
     }] if matrix_mx_puppet_skype_enabled else [])
     + ([{
       'name': 'matrix_puppet_slack',
       'username': 'matrix_puppet_slack',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | to_uuid,
     }] if matrix_mx_puppet_slack_enabled else [])
     + ([{
       'name': 'matrix_puppet_twitter',
       'username': 'matrix_puppet_twitter',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | to_uuid,
     }] if matrix_mx_puppet_twitter_enabled else [])
     + ([{
       'name': 'matrix_puppet_instagram',
       'username': 'matrix_puppet_instagram',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | to_uuid,
     }] if matrix_mx_puppet_instagram_enabled else [])
     + ([{
       'name': 'matrix_puppet_discord',
       'username': 'matrix_puppet_discord',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | to_uuid,
     }] if matrix_mx_puppet_discord_enabled else [])
     + ([{
       'name': 'matrix_puppet_steam',
       'username': 'matrix_puppet_steam',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | to_uuid,
     }] if matrix_mx_puppet_steam_enabled else [])
     + ([{
       'name': 'matrix_dimension',
       'username': 'matrix_dimension',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | to_uuid,
     }] if matrix_dimension_enabled else [])
   }}
 

From b87b75437232a8b3fa4c2ba2258382dfa761a62e Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 01:36:15 +0200
Subject: [PATCH 055/179] Fail if appservice-discord wants Postgres, but has
 leftover SQLite data

---
 .../defaults/main.yml                            |  3 ++-
 .../tasks/setup_install.yml                      | 16 ++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml
index 36b39b8e..53d7968d 100644
--- a/roles/matrix-bridge-appservice-discord/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml
@@ -49,7 +49,8 @@ matrix_appservice_discord_bridge_enableSelfServiceBridging: false
 # - change the engine (`matrix_appservice_discord_database_engine: 'postgres'`)
 # - adjust your database credentials
 matrix_appservice_discord_database_engine: 'sqlite'
-matrix_appservice_discord_database_filename: "/data/discord.db"
+matrix_appservice_discord_database_filename: "/data/{{ matrix_appservice_discord_database_filename_name }}"
+matrix_appservice_discord_database_filename_name: "discord.db"
 matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_connString_username }}:{{ matrix_appservice_discord_database_connString_password }}@{{ matrix_appservice_discord_database_connString_hostname }}:{{ matrix_appservice_discord_database_connString_port }}/{{ matrix_appservice_discord_database_connString_db_name }}'
 matrix_appservice_discord_database_connString_username: 'matrix_appservice_discord'
 matrix_appservice_discord_database_connString_password: 'some-password'
diff --git a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
index 935d07eb..cea8fda8 100644
--- a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
+++ b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
@@ -1,5 +1,21 @@
 ---
 
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_appservice_discord_data_path }}/{{ matrix_appservice_discord_database_filename_name }}"
+    register: matrix_appservice_discord_stat_sqlite_db
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_appservice_discord_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_appservice_discord_data_path }}/{{ matrix_appservice_discord_database_filename_name }}.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_appservice_discord_database_engine: sqlite` to your vars.yml file.
+        To migrate to Postgres: TODO - migration instructions here.
+  when: "matrix_appservice_discord_database_engine == 'postgres'"
+
 - name: Ensure Appservice Discord image is pulled
   docker_image:
     name: "{{ matrix_appservice_discord_docker_image }}"

From 6e1dfb62f0d371fdb82fc7a9e635dd32d0179e86 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 01:48:36 +0200
Subject: [PATCH 056/179] Rename some doc files and commands related to
 importing

Since we'll likely have generic SQLite database importing
via [pgloader](https://pgloader.io/) for migrating bridge
databases from SQLite to Postgres, we'd rather avoid
calling the "import Synapse SQLite database" command
as just `--tags=import-sqlite-db`.

Similarly, for the media store, we'd like to mention that it's
related to Synapse as well.

We'd like to be more explicit, so as to be less confusing,
especially in light of other homeserver implementations
coming in the future.
---
 docs/README.md                                              | 6 +++---
 docs/importing-postgres.md                                  | 4 ++--
 ...ting-media-store.md => importing-synapse-media-store.md} | 4 ++--
 docs/{importing-sqlite.md => importing-synapse-sqlite.md}   | 6 +++---
 docs/installing.md                                          | 4 ++--
 .../{import_sqlite_db.yml => import_synapse_sqlite_db.yml}  | 0
 roles/matrix-postgres/tasks/main.yml                        | 6 ++++--
 roles/matrix-synapse/tasks/main.yml                         | 2 +-
 8 files changed, 17 insertions(+), 15 deletions(-)
 rename docs/{importing-media-store.md => importing-synapse-media-store.md} (92%)
 rename docs/{importing-sqlite.md => importing-synapse-sqlite.md} (86%)
 rename roles/matrix-postgres/tasks/{import_sqlite_db.yml => import_synapse_sqlite_db.yml} (100%)

diff --git a/docs/README.md b/docs/README.md
index 8d22477e..8c95eff3 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -10,13 +10,13 @@
 
 - [Installing](installing.md)
 
-- **Importing data from another Synapse server installation**
+- **Importing data from another server installation**
 
-  - [Importing an existing SQLite database (from another installation)](importing-sqlite.md) (optional)
+  - [Importing an existing SQLite database (from another Synapse installation)](importing-synapse-sqlite.md) (optional)
 
   - [Importing an existing Postgres database (from another installation)](importing-postgres.md) (optional)
 
-  - [Importing `media_store` data files from an existing installation](importing-media-store.md) (optional)
+  - [Importing `media_store` data files from an existing Synapse installation](importing-synapse-media-store.md) (optional)
 
 - [Registering users](registering-users.md)
 
diff --git a/docs/importing-postgres.md b/docs/importing-postgres.md
index a88067e1..0dd75cb2 100644
--- a/docs/importing-postgres.md
+++ b/docs/importing-postgres.md
@@ -1,7 +1,7 @@
 # Importing an existing Postgres database from another installation (optional)
 
-Run this if you'd like to import your database from a previous installation of Synapse.
-(don't forget to import your `media_store` files as well - see [the importing-media-store guide](importing-media-store.md)).
+Run this if you'd like to import your database from a previous installation.
+(don't forget to import your Synapse `media_store` files as well - see [the importing-synape-media-store guide](importing-synapse-media-store.md)).
 
 
 ## Prerequisites
diff --git a/docs/importing-media-store.md b/docs/importing-synapse-media-store.md
similarity index 92%
rename from docs/importing-media-store.md
rename to docs/importing-synapse-media-store.md
index 0d86370b..0ba7bacb 100644
--- a/docs/importing-media-store.md
+++ b/docs/importing-synapse-media-store.md
@@ -1,4 +1,4 @@
-# Importing `media_store` data files from an existing installation (optional)
+# Importing `media_store` data files from an existing Synapse installation (optional)
 
 Run this if you'd like to import your `media_store` files from a previous installation of Synapse.
 
@@ -17,6 +17,6 @@ As an alternative, you can perform a manual restore using the [AWS CLI tool](htt
 
 Run this command (make sure to replace `<server-path-to-media_store>` with a path on your server):
 
-	ansible-playbook -i inventory/hosts setup.yml --extra-vars='server_path_media_store=<server-path-to-media_store>' --tags=import-media-store
+	ansible-playbook -i inventory/hosts setup.yml --extra-vars='server_path_media_store=<server-path-to-media_store>' --tags=import-synapse-media-store
 
 **Note**: `<server-path-to-media_store>` must be a file path to a `media_store` directory on the server (not on your local machine!).
diff --git a/docs/importing-sqlite.md b/docs/importing-synapse-sqlite.md
similarity index 86%
rename from docs/importing-sqlite.md
rename to docs/importing-synapse-sqlite.md
index 9e3a910d..aade2226 100644
--- a/docs/importing-sqlite.md
+++ b/docs/importing-synapse-sqlite.md
@@ -1,7 +1,7 @@
-# Importing an existing SQLite database from another installation (optional)
+# Importing an existing SQLite database from another Synapse installation (optional)
 
 Run this if you'd like to import your database from a previous default installation of Synapse.
-(don't forget to import your `media_store` files as well - see [the importing-media-store guide](importing-media-store.md)).
+(don't forget to import your `media_store` files as well - see [the importing-synapse-media-store guide](importing-synapse-media-store.md)).
 
 While this playbook always sets up PostgreSQL, by default a Synapse installation would run
 using an SQLite database.
@@ -18,7 +18,7 @@ Before doing the actual import, **you need to upload your SQLite database file t
 
 Run this command (make sure to replace `<server-path-to-homeserver.db>` with a file path on your server):
 
-	ansible-playbook -i inventory/hosts setup.yml --extra-vars='server_path_homeserver_db=<server-path-to-homeserver.db>' --tags=import-sqlite-db
+	ansible-playbook -i inventory/hosts setup.yml --extra-vars='server_path_homeserver_db=<server-path-to-homeserver.db>' --tags=import-synapse-sqlite-db
 
 **Notes**:
 
diff --git a/docs/installing.md b/docs/installing.md
index 43758ffb..a2ce1371 100644
--- a/docs/installing.md
+++ b/docs/installing.md
@@ -21,11 +21,11 @@ Feel free to **re-run this setup command any time** you think something is off w
 
 After installing, but before starting the services, you may want to do additional things like:
 
-- [Importing an existing SQLite database (from another installation)](importing-sqlite.md) (optional)
+- [Importing an existing SQLite database (from another Synapse installation)](importing-synapse-sqlite.md) (optional)
 
 - [Importing an existing Postgres database (from another installation)](importing-postgres.md) (optional)
 
-- [Importing `media_store` data files from an existing installation](importing-media-store.md) (optional)
+- [Importing `media_store` data files from an existing Synapse installation](importing-synapse-media-store.md) (optional)
 
 
 ## Starting the services
diff --git a/roles/matrix-postgres/tasks/import_sqlite_db.yml b/roles/matrix-postgres/tasks/import_synapse_sqlite_db.yml
similarity index 100%
rename from roles/matrix-postgres/tasks/import_sqlite_db.yml
rename to roles/matrix-postgres/tasks/import_synapse_sqlite_db.yml
diff --git a/roles/matrix-postgres/tasks/main.yml b/roles/matrix-postgres/tasks/main.yml
index 717b73e4..f4c752a0 100644
--- a/roles/matrix-postgres/tasks/main.yml
+++ b/roles/matrix-postgres/tasks/main.yml
@@ -19,10 +19,12 @@
   tags:
     - import-postgres
 
-- import_tasks: "{{ role_path }}/tasks/import_sqlite_db.yml"
+# The `run_postgres_import_sqlite_db` variable had better be renamed to be consistent,
+# but that's a breaking change which may cause trouble for people.
+- import_tasks: "{{ role_path }}/tasks/import_synapse_sqlite_db.yml"
   when: run_postgres_import_sqlite_db|bool
   tags:
-    - import-sqlite-db
+    - import-synapse-sqlite-db
 
 - import_tasks: "{{ role_path }}/tasks/upgrade_postgres.yml"
   when: run_postgres_upgrade|bool
diff --git a/roles/matrix-synapse/tasks/main.yml b/roles/matrix-synapse/tasks/main.yml
index 48e72ace..e366a13f 100644
--- a/roles/matrix-synapse/tasks/main.yml
+++ b/roles/matrix-synapse/tasks/main.yml
@@ -17,7 +17,7 @@
 - import_tasks: "{{ role_path }}/tasks/import_media_store.yml"
   when: run_synapse_import_media_store|bool
   tags:
-    - import-media-store
+    - import-synapse-media-store
 
 - import_tasks: "{{ role_path }}/tasks/register_user.yml"
   when: run_synapse_register_user|bool

From cb969c6ca297b21216f896033ae86702be227812 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 02:23:29 +0200
Subject: [PATCH 057/179] Add --tags=import-generic-sqlite-db (pgloader import)

This can be used by various bridges, etc., to import an SQLite
(or some other supported) database into Postgres.
---
 roles/matrix-postgres/defaults/main.yml       |  2 +
 .../tasks/import_generic_sqlite_db.yml        | 84 +++++++++++++++++++
 roles/matrix-postgres/tasks/main.yml          |  6 ++
 3 files changed, 92 insertions(+)
 create mode 100644 roles/matrix-postgres/tasks/import_generic_sqlite_db.yml

diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml
index 89559f5d..5755742f 100644
--- a/roles/matrix-postgres/defaults/main.yml
+++ b/roles/matrix-postgres/defaults/main.yml
@@ -48,3 +48,5 @@ matrix_postgres_additional_databases: []
 #
 # For most (subsequent) runs, Postgres would already be running, so no waiting will be happening at all.
 matrix_postgres_additional_databases_postgres_start_wait_timeout_seconds: 15
+
+matrix_postgres_pgloader_docker_image: "docker.io/illagrenan/pgloader:3.6.2"
diff --git a/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml b/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml
new file mode 100644
index 00000000..f2798a73
--- /dev/null
+++ b/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml
@@ -0,0 +1,84 @@
+---
+
+# Pre-checks
+
+- name: Fail if Postgres not enabled
+  fail:
+    msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot import."
+  when: "not matrix_postgres_enabled|bool"
+
+- name: Fail if playbook called incorrectly
+  fail:
+    msg: "The `sqlite_database_path` variable needs to be provided to this playbook, via --extra-vars"
+  when: "sqlite_database_path is not defined or sqlite_database_path.startswith('<')"
+
+- name: Fail if playbook called incorrectly
+  fail:
+    msg: >-
+      The `postgres_db_connection_string` variable needs to be provided to this playbook, via `--extra-vars`.
+      Example: `--extra-vars="postgres_db_connection_string=postgresql://username:password@localhost:<port>/database_name`"
+  when: "postgres_db_connection_string is not defined or not postgres_db_connection_string.startswith('postgresql://')"
+
+- name: Check if the provided SQLite database file exists
+  stat:
+    path: "{{ sqlite_database_path }}"
+  register: sqlite_database_path_stat_result
+
+- name: Fail if provided SQLite database file doesn't exist
+  fail:
+    msg: "File cannot be found on the server at {{ sqlite_database_path }}"
+  when: "not sqlite_database_path_stat_result.stat.exists"
+
+
+# Defaults
+
+- name: Set postgres_start_wait_time, if not provided
+  set_fact:
+    postgres_start_wait_time: 15
+  when: "postgres_start_wait_time|default('') == ''"
+
+
+# Actual import work
+
+- name: Ensure matrix-postgres is started
+  service:
+    name: matrix-postgres
+    state: started
+    daemon_reload: yes
+  register: matrix_postgres_service_start_result
+
+- name: Wait a bit, so that Postgres can start
+  wait_for:
+    timeout: "{{ postgres_start_wait_time }}"
+  delegate_to: 127.0.0.1
+  become: false
+  when: "matrix_postgres_service_start_result.changed|bool"
+
+- name: Import SQLite database from {{ sqlite_database_path }} into Postgres
+  command:
+    cmd: >-
+      {{ matrix_host_command_docker }} run
+      --rm
+      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
+      --cap-drop=ALL
+      --network={{ matrix_docker_network }}
+      --mount type=bind,src={{ sqlite_database_path }},dst=/in.db,ro
+      --entrypoint=/bin/sh
+      {{ matrix_postgres_pgloader_docker_image }}
+      -c
+      'pgloader /in.db {{ postgres_db_connection_string }}'
+
+- name: Archive SQLite database ({{ sqlite_database_path }} -> {{ sqlite_database_path }}.backup)
+  command:
+    cmd: "mv {{ sqlite_database_path }} {{ sqlite_database_path }}.backup"
+
+- name: Inject result
+  set_fact:
+    matrix_playbook_runtime_results: |
+      {{
+        matrix_playbook_runtime_results|default([])
+        +
+        [
+          "NOTE: Your SQLite database file has been imported into Postgres. The original file has been moved from `{{ sqlite_database_path }}` to `{{ sqlite_database_path }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file."
+        ]
+      }}
diff --git a/roles/matrix-postgres/tasks/main.yml b/roles/matrix-postgres/tasks/main.yml
index f4c752a0..b9c2ae7c 100644
--- a/roles/matrix-postgres/tasks/main.yml
+++ b/roles/matrix-postgres/tasks/main.yml
@@ -26,6 +26,12 @@
   tags:
     - import-synapse-sqlite-db
 
+# Perhaps we need a new variable here, instead of `run_postgres_import_sqlite_db`.
+- import_tasks: "{{ role_path }}/tasks/import_generic_sqlite_db.yml"
+  when: run_postgres_import_sqlite_db|bool
+  tags:
+    - import-generic-sqlite-db
+
 - import_tasks: "{{ role_path }}/tasks/upgrade_postgres.yml"
   when: run_postgres_upgrade|bool
   tags:

From 4617984b9f998e032f426ec629e408a1f0c2ecf8 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 02:24:32 +0200
Subject: [PATCH 058/179] Add (SQLite -> Postgres) migration instructions

---
 .../tasks/setup_install.yml                       | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
index cea8fda8..57e59df0 100644
--- a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
+++ b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
@@ -1,10 +1,13 @@
 ---
 
 - block:
+  - set_fact:
+      matrix_appservice_discord_sqlite_db_path: "{{ matrix_appservice_discord_data_path }}/{{ matrix_appservice_discord_database_filename_name }}"
+
   - name: Check if an SQLite database already exists
     stat:
-      path: "{{ matrix_appservice_discord_data_path }}/{{ matrix_appservice_discord_database_filename_name }}"
-    register: matrix_appservice_discord_stat_sqlite_db
+      path: "{{ matrix_appservice_discord_sqlite_db_path }}"
+    register: matrix_appservice_discord_sqlite_db_path_stat_result
 
   - name: Fail if an SQLite database already exists when using Postgres
     fail:
@@ -12,8 +15,12 @@
         matrix_appservice_discord_database_engine has been set to `postgres` (which is our new default now).
         However, we've discovered an existing SQLite database in {{ matrix_appservice_discord_data_path }}/{{ matrix_appservice_discord_database_filename_name }}.
         It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_appservice_discord_database_engine: sqlite` to your vars.yml file.
-        To migrate to Postgres: TODO - migration instructions here.
+        To continue using SQLite, opt into it explicitly: add `matrix_appservice_discord_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_discord_sqlite_db_path }} postgres_db_connection_string={{ matrix_appservice_discord_database_connString }}'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_appservice_discord_sqlite_db_path_stat_result.stat.exists"
   when: "matrix_appservice_discord_database_engine == 'postgres'"
 
 - name: Ensure Appservice Discord image is pulled

From b217a2ed6907347200c57d66195dd780e4a1f24f Mon Sep 17 00:00:00 2001
From: transcaffeine <transcaffeine@finallycoffee.eu>
Date: Mon, 14 Dec 2020 07:54:24 +0100
Subject: [PATCH 059/179] postgres: set default in playbook to postgres but
 fallback to sqlite

---
 group_vars/matrix_servers | 175 +++++++++++++++++++++++++-------------
 1 file changed, 118 insertions(+), 57 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index b9d0be26..0cbc4587 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -140,6 +140,10 @@ matrix_appservice_slack_systemd_required_services_list: |
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
   }}
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_appservice_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_appservice_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.slack.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-appservice-slack
@@ -175,6 +179,11 @@ matrix_appservice_irc_appservice_token: "{{ matrix_synapse_macaroon_secret_key |
 
 matrix_appservice_irc_homeserver_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'irc.hs.token') | to_uuid }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_appservice_irc_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_appservice_irc_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.irc.db') | to_uuid }}"
+
+
 ######################################################################
 #
 # /matrix-bridge-appservice-irc
@@ -208,6 +217,10 @@ matrix_mautrix_facebook_login_shared_secret: "{{ matrix_synapse_ext_password_pro
 
 matrix_mautrix_facebook_bridge_presence: "{{ matrix_synapse_use_presence if matrix_synapse_enabled else true }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_mautrix_facebook_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_mautrix_facebook_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_facebook.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-mautrix-facebook
@@ -241,6 +254,10 @@ matrix_mautrix_hangouts_container_http_host_bind_port: "{{ '' if matrix_nginx_pr
 
 matrix_mautrix_hangouts_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_mautrix_hangouts_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_mautrix_hangouts_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_hangouts.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-mautrix-hangouts
@@ -277,6 +294,10 @@ matrix_mautrix_telegram_container_http_host_bind_port: "{{ '' if matrix_nginx_pr
 
 matrix_mautrix_telegram_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_mautrix_telegram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_mautrix_telegram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_telegram.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-mautrix-telegram
@@ -305,6 +326,10 @@ matrix_mautrix_whatsapp_homeserver_token: "{{ matrix_synapse_macaroon_secret_key
 
 matrix_mautrix_whatsapp_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_mautrix_whatsapp_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_mautrix_whatsapp_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_whatsapp.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-mautrix-whatsapp
@@ -361,6 +386,10 @@ matrix_mx_puppet_skype_homeserver_token: "{{ matrix_synapse_macaroon_secret_key
 
 matrix_mx_puppet_skype_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_mx_puppet_skype_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_mx_puppet_skype_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_skype.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-mx-puppet-skype
@@ -392,6 +421,10 @@ matrix_mx_puppet_slack_homeserver_token: "{{ matrix_synapse_macaroon_secret_key
 
 matrix_mx_puppet_slack_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_mx_puppet_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_mx_puppet_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_slack.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-mx-puppet-slack
@@ -424,6 +457,10 @@ matrix_mx_puppet_twitter_login_shared_secret: "{{ matrix_synapse_ext_password_pr
 
 matrix_mx_puppet_twitter_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else ('127.0.0.1:' ~ matrix_mx_puppet_twitter_appservice_port) }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_mx_puppet_twitter_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_mx_puppet_twitter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_twitter.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-mx-puppet-twitter
@@ -455,6 +492,10 @@ matrix_mx_puppet_instagram_homeserver_token: "{{ matrix_synapse_macaroon_secret_
 
 matrix_mx_puppet_instagram_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_mx_puppet_instagram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_mx_puppet_instagram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_instagram.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-mx-puppet-instagram
@@ -485,6 +526,10 @@ matrix_mx_puppet_discord_homeserver_token: "{{ matrix_synapse_macaroon_secret_ke
 
 matrix_mx_puppet_discord_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_mx_puppet_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_mx_puppet_discord_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_discord.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-mx-puppet-discord
@@ -515,6 +560,10 @@ matrix_mx_puppet_steam_homeserver_token: "{{ matrix_synapse_macaroon_secret_key
 
 matrix_mx_puppet_steam_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_mx_puppet_stream_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_steam.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-mx-puppet-steam
@@ -625,6 +674,10 @@ matrix_dimension_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_ena
 matrix_integration_manager_rest_url: "{{ matrix_dimension_integrations_rest_url if matrix_dimension_enabled else None }}"
 matrix_integration_manager_ui_url: "{{ matrix_dimension_integrations_ui_url if matrix_dimension_enabled else None }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_dimension_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_dimension_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.dimension.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-dimension
@@ -902,75 +955,83 @@ matrix_postgres_additional_databases: |
       'password': matrix_appservice_discord_database_connString_password,
     }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_database_connString_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': 'matrix_appservice_slack',
-      'username': 'matrix_appservice_slack',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | to_uuid,
-    }] if matrix_appservice_slack_enabled else [])
+      'name': matrix_appservice_slack_database_db_name
+      'username': matrix_appservice_slack_database_username
+      'password': matrix_appservice_slack_database_password
+    }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres' else [])
     + ([{
-      'name': 'matrix_appservice_irc',
-      'username': 'matrix_appservice_irc',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | to_uuid,
-    }] if matrix_appservice_irc_enabled else [])
+      'name': matrix_appservice_irc_database_db_name
+      'username': matrix_appservice_irc_database_username
+      'password': matrix_appservice_irc_database_password
+    }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres' else [])
     + ([{
-      'name': 'mautrix_bridge_facebook',
-      'username': 'mautrix_bridge_facebook',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | to_uuid,
-    }] if matrix_mautrix_facebook_enabled else [])
+      'name': matrix_mautrix_facebook_database_db_name
+      'username': matrix_mautrix_facebook_database_username
+      'password': matrix_mautrix_facebook_database_password
+    }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_database_hostname == 'matrix-postgres' else [])
     + ([{
-      'name': 'mautrix_bridge_hangouts',
-      'username': 'mautrix_bridge_hangouts',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | to_uuid,
-    }] if matrix_mautrix_hangouts_enabled else [])
+      'name': matrix_mautrix_hangouts_database_db_name
+      'username': matrix_mautrix_hangouts_database_username
+      'password': matrix_mautrix_hangouts_database_password
+    }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres' else [])
     + ([{
-      'name': 'mautrix_bridge_telegram',
-      'username': 'mautrix_bridge_telegram',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | to_uuid,
-    }] if matrix_mautrix_telegram_enabled else [])
+      'name': matrix_mautrix_telegram_database_db_name
+      'username': matrix_mautrix_telegram_database_username
+      'password': matrix_mautrix_telegram_database_password
+    }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_telegram_facebook_database_hostname == 'matrix-postgres' else [])
     + ([{
-      'name': 'mautrix_bridge_whatsapp',
-      'username': 'mautrix_bridge_whatsapp',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | to_uuid,
-    }] if matrix_mautrix_whatsapp_enabled else [])
+      'name': matrix_mautrix_whatsapp_database_db_name
+      'username': matrix_mautrix_whatsapp_database_username
+      'password': matrix_mautrix_whatsapp_database_password
+    }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres' else [])
     + ([{
       'name': 'matrix_bridge_sms',
       'username': 'matrix_bridge_sms',
       'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid,
     }] if matrix_sms_bridge_enabled else [])
     + ([{
-      'name': 'matrix_puppet_skype',
-      'username': 'matrix_puppet_skype',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | to_uuid,
-    }] if matrix_mx_puppet_skype_enabled else [])
-    + ([{
-      'name': 'matrix_puppet_slack',
-      'username': 'matrix_puppet_slack',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | to_uuid,
-    }] if matrix_mx_puppet_slack_enabled else [])
-    + ([{
-      'name': 'matrix_puppet_twitter',
-      'username': 'matrix_puppet_twitter',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | to_uuid,
-    }] if matrix_mx_puppet_twitter_enabled else [])
-    + ([{
-      'name': 'matrix_puppet_instagram',
-      'username': 'matrix_puppet_instagram',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | to_uuid,
-    }] if matrix_mx_puppet_instagram_enabled else [])
-    + ([{
-      'name': 'matrix_puppet_discord',
-      'username': 'matrix_puppet_discord',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | to_uuid,
-    }] if matrix_mx_puppet_discord_enabled else [])
-    + ([{
-      'name': 'matrix_puppet_steam',
-      'username': 'matrix_puppet_steam',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | to_uuid,
-    }] if matrix_mx_puppet_steam_enabled else [])
+      'name': matrix_mx_puppet_skype_database_db_name
+      'username': matrix_mx_puppet_skype_database_username
+      'password': matrix_mx_puppet_skype_database_password
+    }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres' else [])
     + ([{
-      'name': 'matrix_dimension',
-      'username': 'matrix_dimension',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | to_uuid,
-    }] if matrix_dimension_enabled else [])
+      'name': matrix_mx_puppet_slack_database_db_name
+      'username': matrix_mx_puppet_slack_database_username
+      'password': matrix_mx_puppet_slack_database_password
+    }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres' else [])
+     + ([{
+      'name': matrix_mx_puppet_twitter_database_db_name
+      'username': matrix_mx_puppet_twitter_database_username
+      'password': matrix_mx_puppet_twitter_database_password
+    }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres' else [])
+     + ([{
+      'name': matrix_mx_puppet_instagram_database_db_name
+      'username': matrix_mx_puppet_instagram_database_username
+      'password': matrix_mx_puppet_instagram_database_password
+    }] if (matrix_mx_puppet_instagram_enabled
+           and matrix_mx_puppet_instagram_database_engine == 'postgres'
+           and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres' else [])
+     + ([{
+      'name': matrix_mx_puppet_discord_database_db_name
+      'username': matrix_mx_puppet_discord_database_username
+      'password': matrix_mx_puppet_discord_database_password
+    }] if (matrix_mx_puppet_discord_enabled
+           and matrix_mx_puppet_discord_database_engine == 'postgres'
+           and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres' else [])
+     + ([{
+      'name': matrix_mx_puppet_steam_database_db_name
+      'username': matrix_mx_puppet_steam_database_username
+      'password': matrix_mx_puppet_steam_database_password
+    }] if (matrix_mx_puppet_steam_enabled
+           and matrix_mx_puppet_steam_database_engine == 'postgres'
+           and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres' else [])
+     + ([{
+      'name': matrix_dimension_database_db_name
+      'username': matrix_dimension_database_username
+      'password': matrix_dimension_database_password
+    }] if (matrix_dimension_enabled
+           and matrix_dimension_database_engine == 'postgres'
+           and matrix_dimension_database_hostname == 'matrix-postgres' else [])
   }}
 
 ######################################################################

From f1e85f7112ffa9dd3309e8963c9a8e69723f48a0 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 10:04:37 +0200
Subject: [PATCH 060/179] Don't mention Postgres roles, just say users

---
 .../templates/init-additional-db-user-and-role.sql.j2           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2 b/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2
index af3d912c..609a1344 100644
--- a/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2
+++ b/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2
@@ -4,7 +4,7 @@ DO $$
 BEGIN
   CREATE USER {{ additional_db.username }};
   EXCEPTION WHEN DUPLICATE_OBJECT THEN
-  RAISE NOTICE 'not creating role {{ additional_db.username }}, since it already exists';
+  RAISE NOTICE 'not creating user {{ additional_db.username }}, since it already exists';
 END
 $$;
 

From d91aa5a060ab0742f25c15a951429104c1793a91 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 10:51:53 +0200
Subject: [PATCH 061/179] Do not introduce sub-variables exposing
 implementation details

---
 group_vars/matrix_servers                     | 10 ++++----
 .../defaults/main.yml                         | 25 ++++++++++++-------
 .../tasks/setup_install.yml                   | 13 ++++------
 3 files changed, 26 insertions(+), 22 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 0cbc4587..efb29d4a 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -70,7 +70,7 @@ matrix_appservice_discord_homeserver_token: "{{ matrix_synapse_macaroon_secret_k
 # We only make this use Postgres if our own Postgres server is enabled.
 # It's only then (for now) that we can automatically create the necessary database and user for this service.
 matrix_appservice_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_appservice_discord_database_connString_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.discord.db') | to_uuid }}"
+matrix_appservice_discord_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.discord.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -950,10 +950,10 @@ matrix_postgres_db_name: "homeserver"
 matrix_postgres_additional_databases: |
   {{
     ([{
-      'name': matrix_appservice_discord_database_connString_db_name,
-      'username': matrix_appservice_discord_database_connString_username,
-      'password': matrix_appservice_discord_database_connString_password,
-    }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_database_connString_hostname == 'matrix-postgres') else [])
+      'name': matrix_appservice_discord_postgres_db_name,
+      'username': matrix_appservice_discord_postgres_username,
+      'password': matrix_appservice_discord_postgres_password,
+    }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_postgres_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_appservice_slack_database_db_name
       'username': matrix_appservice_slack_database_username
diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml
index 53d7968d..fb517cd5 100644
--- a/roles/matrix-bridge-appservice-discord/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml
@@ -47,16 +47,23 @@ matrix_appservice_discord_bridge_enableSelfServiceBridging: false
 #
 # To use Postgres:
 # - change the engine (`matrix_appservice_discord_database_engine: 'postgres'`)
-# - adjust your database credentials
+# - adjust your database credentials via the `matrix_appservice_discord_postgres_*` variables
 matrix_appservice_discord_database_engine: 'sqlite'
-matrix_appservice_discord_database_filename: "/data/{{ matrix_appservice_discord_database_filename_name }}"
-matrix_appservice_discord_database_filename_name: "discord.db"
-matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_connString_username }}:{{ matrix_appservice_discord_database_connString_password }}@{{ matrix_appservice_discord_database_connString_hostname }}:{{ matrix_appservice_discord_database_connString_port }}/{{ matrix_appservice_discord_database_connString_db_name }}'
-matrix_appservice_discord_database_connString_username: 'matrix_appservice_discord'
-matrix_appservice_discord_database_connString_password: 'some-password'
-matrix_appservice_discord_database_connString_hostname: 'matrix-postgres'
-matrix_appservice_discord_database_connString_port: 5432
-matrix_appservice_discord_database_connString_db_name: 'matrix_appservice_discord'
+
+matrix_appservice_discord_sqlite_database_path_local: "{{ matrix_appservice_discord_data_path }}/discord.db"
+matrix_appservice_discord_sqlite_database_path_in_container: "/data/discord.db"
+
+matrix_appservice_discord_postgres_username: 'matrix_appservice_discord'
+matrix_appservice_discord_postgres_password: 'some-password'
+matrix_appservice_discord_postgres_hostname: 'matrix-postgres'
+matrix_appservice_discord_postgres_port: 5432
+matrix_appservice_discord_postgres_db_name: 'matrix_appservice_discord'
+
+# These 2 variables are what actually ends up in the bridge configuration.
+# It's best if you don't change them directly, but rather redefine the sub-variables that constitute them.
+matrix_appservice_discord_database_filename: "{{ matrix_appservice_discord_sqlite_database_path_in_container }}"
+matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_postgres_username }}:{{ matrix_appservice_discord_postgres_password }}@{{ matrix_appservice_discord_postgres_hostname }}:{{ matrix_appservice_discord_postgres_port }}/{{ matrix_appservice_discord_postgres_db_name }}'
+
 
 # Tells whether the bot should make use of "Privileged Gateway Intents".
 #
diff --git a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
index 57e59df0..86a5fe51 100644
--- a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
+++ b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
@@ -1,26 +1,23 @@
 ---
 
 - block:
-  - set_fact:
-      matrix_appservice_discord_sqlite_db_path: "{{ matrix_appservice_discord_data_path }}/{{ matrix_appservice_discord_database_filename_name }}"
-
   - name: Check if an SQLite database already exists
     stat:
-      path: "{{ matrix_appservice_discord_sqlite_db_path }}"
-    register: matrix_appservice_discord_sqlite_db_path_stat_result
+      path: "{{ matrix_appservice_discord_sqlite_database_path_local }}"
+    register: matrix_appservice_discord_sqlite_database_path_local_stat_result
 
   - name: Fail if an SQLite database already exists when using Postgres
     fail:
       msg: >-
         matrix_appservice_discord_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_appservice_discord_data_path }}/{{ matrix_appservice_discord_database_filename_name }}.
+        However, we've discovered an existing SQLite database in {{ matrix_appservice_discord_sqlite_database_path_local }}.
         It appears that you've been using this bridge with the SQLite engine until now.
         To continue using SQLite, opt into it explicitly: add `matrix_appservice_discord_database_engine: sqlite` to your vars.yml file and re-run this same command.
         Alternatively, to migrate your existing SQLite database to Postgres:
         1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_discord_sqlite_db_path }} postgres_db_connection_string={{ matrix_appservice_discord_database_connString }}'`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_discord_sqlite_database_path_local }} postgres_db_connection_string={{ matrix_appservice_discord_database_connString }}'`)
         3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_appservice_discord_sqlite_db_path_stat_result.stat.exists"
+    when: "matrix_appservice_discord_sqlite_database_path_local_stat_result.stat.exists"
   when: "matrix_appservice_discord_database_engine == 'postgres'"
 
 - name: Ensure Appservice Discord image is pulled

From c0edacbefd60d5bfd136c448ff3fdf76da645731 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 11:17:12 +0200
Subject: [PATCH 062/179] Fix matrix_postgres_additional_databases syntax
 problems

---
 group_vars/matrix_servers | 106 +++++++++++++++++++-------------------
 1 file changed, 53 insertions(+), 53 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index efb29d4a..763fccd8 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -955,84 +955,84 @@ matrix_postgres_additional_databases: |
       'password': matrix_appservice_discord_postgres_password,
     }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_postgres_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_appservice_slack_database_db_name
-      'username': matrix_appservice_slack_database_username
-      'password': matrix_appservice_slack_database_password
-    }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres' else [])
+      'name': matrix_appservice_slack_database_db_name,
+      'username': matrix_appservice_slack_database_username,
+      'password': matrix_appservice_slack_database_password,
+    }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_appservice_irc_database_db_name
-      'username': matrix_appservice_irc_database_username
-      'password': matrix_appservice_irc_database_password
-    }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres' else [])
+      'name': matrix_appservice_irc_database_db_name,
+      'username': matrix_appservice_irc_database_username,
+      'password': matrix_appservice_irc_database_password,
+    }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_facebook_database_db_name
-      'username': matrix_mautrix_facebook_database_username
-      'password': matrix_mautrix_facebook_database_password
-    }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_database_hostname == 'matrix-postgres' else [])
+      'name': matrix_mautrix_facebook_postgres_db_name,
+      'username': matrix_mautrix_facebook_postgres_username,
+      'password': matrix_mautrix_facebook_postgres_password,
+    }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_postgres_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_hangouts_database_db_name
-      'username': matrix_mautrix_hangouts_database_username
-      'password': matrix_mautrix_hangouts_database_password
-    }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres' else [])
+      'name': matrix_mautrix_hangouts_database_db_name,
+      'username': matrix_mautrix_hangouts_database_username,
+      'password': matrix_mautrix_hangouts_database_password,
+    }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_telegram_database_db_name
-      'username': matrix_mautrix_telegram_database_username
-      'password': matrix_mautrix_telegram_database_password
-    }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_telegram_facebook_database_hostname == 'matrix-postgres' else [])
+      'name': matrix_mautrix_telegram_database_db_name,
+      'username': matrix_mautrix_telegram_database_username,
+      'password': matrix_mautrix_telegram_database_password,
+    }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_telegram_facebook_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_whatsapp_database_db_name
-      'username': matrix_mautrix_whatsapp_database_username
-      'password': matrix_mautrix_whatsapp_database_password
-    }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres' else [])
+      'name': matrix_mautrix_whatsapp_database_db_name,
+      'username': matrix_mautrix_whatsapp_database_username,
+      'password': matrix_mautrix_whatsapp_database_password,
+    }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres') else [])
     + ([{
       'name': 'matrix_bridge_sms',
       'username': 'matrix_bridge_sms',
       'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid,
     }] if matrix_sms_bridge_enabled else [])
     + ([{
-      'name': matrix_mx_puppet_skype_database_db_name
-      'username': matrix_mx_puppet_skype_database_username
-      'password': matrix_mx_puppet_skype_database_password
-    }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres' else [])
+      'name': matrix_mx_puppet_skype_database_db_name,
+      'username': matrix_mx_puppet_skype_database_username,
+      'password': matrix_mx_puppet_skype_database_password,
+    }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mx_puppet_slack_database_db_name
-      'username': matrix_mx_puppet_slack_database_username
-      'password': matrix_mx_puppet_slack_database_password
-    }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres' else [])
+      'name': matrix_mx_puppet_slack_database_db_name,
+      'username': matrix_mx_puppet_slack_database_username,
+      'password': matrix_mx_puppet_slack_database_password,
+    }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_twitter_database_db_name
-      'username': matrix_mx_puppet_twitter_database_username
-      'password': matrix_mx_puppet_twitter_database_password
-    }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres' else [])
+      'name': matrix_mx_puppet_twitter_database_db_name,
+      'username': matrix_mx_puppet_twitter_database_username,
+      'password': matrix_mx_puppet_twitter_database_password,
+    }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_instagram_database_db_name
-      'username': matrix_mx_puppet_instagram_database_username
-      'password': matrix_mx_puppet_instagram_database_password
+      'name': matrix_mx_puppet_instagram_database_db_name,
+      'username': matrix_mx_puppet_instagram_database_username,
+      'password': matrix_mx_puppet_instagram_database_password,
     }] if (matrix_mx_puppet_instagram_enabled
            and matrix_mx_puppet_instagram_database_engine == 'postgres'
-           and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres' else [])
+           and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_discord_database_db_name
-      'username': matrix_mx_puppet_discord_database_username
-      'password': matrix_mx_puppet_discord_database_password
+      'name': matrix_mx_puppet_discord_database_db_name,
+      'username': matrix_mx_puppet_discord_database_username,
+      'password': matrix_mx_puppet_discord_database_password,
     }] if (matrix_mx_puppet_discord_enabled
            and matrix_mx_puppet_discord_database_engine == 'postgres'
-           and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres' else [])
+           and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_steam_database_db_name
-      'username': matrix_mx_puppet_steam_database_username
-      'password': matrix_mx_puppet_steam_database_password
+      'name': matrix_mx_puppet_steam_database_db_name,
+      'username': matrix_mx_puppet_steam_database_username,
+      'password': matrix_mx_puppet_steam_database_password,
     }] if (matrix_mx_puppet_steam_enabled
            and matrix_mx_puppet_steam_database_engine == 'postgres'
-           and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres' else [])
+           and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_dimension_database_db_name
-      'username': matrix_dimension_database_username
-      'password': matrix_dimension_database_password
+      'name': matrix_dimension_database_db_name,
+      'username': matrix_dimension_database_username,
+      'password': matrix_dimension_database_password,
     }] if (matrix_dimension_enabled
            and matrix_dimension_database_engine == 'postgres'
-           and matrix_dimension_database_hostname == 'matrix-postgres' else [])
-  }}
+           and matrix_dimension_database_hostname == 'matrix-postgres') else [])
+   }}
 
 ######################################################################
 #

From 2a99e84b5bfd9f3988566e498b2fe729baa81c75 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 11:19:18 +0200
Subject: [PATCH 063/179] Rename Postgres-related variables
 (`matrix*database_*` -> `matrix*postgres_*`)

`matrix_SERVICE_database_engine` remains as it is - the only one with a
`_database_` infix. Postgres-related ones move to `_postgres_`.
---
 group_vars/matrix_servers | 72 +++++++++++++++++++--------------------
 1 file changed, 36 insertions(+), 36 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 763fccd8..5e3695ca 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -955,14 +955,14 @@ matrix_postgres_additional_databases: |
       'password': matrix_appservice_discord_postgres_password,
     }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_postgres_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_appservice_slack_database_db_name,
-      'username': matrix_appservice_slack_database_username,
-      'password': matrix_appservice_slack_database_password,
+      'name': matrix_appservice_slack_postgres_db_name,
+      'username': matrix_appservice_slack_postgres_username,
+      'password': matrix_appservice_slack_postgres_password,
     }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_appservice_irc_database_db_name,
-      'username': matrix_appservice_irc_database_username,
-      'password': matrix_appservice_irc_database_password,
+      'name': matrix_appservice_irc_postgres_db_name,
+      'username': matrix_appservice_irc_postgres_username,
+      'password': matrix_appservice_irc_postgres_password,
     }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_mautrix_facebook_postgres_db_name,
@@ -970,19 +970,19 @@ matrix_postgres_additional_databases: |
       'password': matrix_mautrix_facebook_postgres_password,
     }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_postgres_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_hangouts_database_db_name,
-      'username': matrix_mautrix_hangouts_database_username,
-      'password': matrix_mautrix_hangouts_database_password,
+      'name': matrix_mautrix_hangouts_postgres_db_name,
+      'username': matrix_mautrix_hangouts_postgres_username,
+      'password': matrix_mautrix_hangouts_postgres_password,
     }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_telegram_database_db_name,
-      'username': matrix_mautrix_telegram_database_username,
-      'password': matrix_mautrix_telegram_database_password,
+      'name': matrix_mautrix_telegram_postgres_db_name,
+      'username': matrix_mautrix_telegram_postgres_username,
+      'password': matrix_mautrix_telegram_postgres_password,
     }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_telegram_facebook_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_whatsapp_database_db_name,
-      'username': matrix_mautrix_whatsapp_database_username,
-      'password': matrix_mautrix_whatsapp_database_password,
+      'name': matrix_mautrix_whatsapp_postgres_db_name,
+      'username': matrix_mautrix_whatsapp_postgres_username,
+      'password': matrix_mautrix_whatsapp_postgres_password,
     }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres') else [])
     + ([{
       'name': 'matrix_bridge_sms',
@@ -990,45 +990,45 @@ matrix_postgres_additional_databases: |
       'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid,
     }] if matrix_sms_bridge_enabled else [])
     + ([{
-      'name': matrix_mx_puppet_skype_database_db_name,
-      'username': matrix_mx_puppet_skype_database_username,
-      'password': matrix_mx_puppet_skype_database_password,
+      'name': matrix_mx_puppet_skype_postgres_db_name,
+      'username': matrix_mx_puppet_skype_postgres_username,
+      'password': matrix_mx_puppet_skype_postgres_password,
     }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mx_puppet_slack_database_db_name,
-      'username': matrix_mx_puppet_slack_database_username,
-      'password': matrix_mx_puppet_slack_database_password,
+      'name': matrix_mx_puppet_slack_postgres_db_name,
+      'username': matrix_mx_puppet_slack_postgres_username,
+      'password': matrix_mx_puppet_slack_postgres_password,
     }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_twitter_database_db_name,
-      'username': matrix_mx_puppet_twitter_database_username,
-      'password': matrix_mx_puppet_twitter_database_password,
+      'name': matrix_mx_puppet_twitter_postgres_db_name,
+      'username': matrix_mx_puppet_twitter_postgres_username,
+      'password': matrix_mx_puppet_twitter_postgres_password,
     }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_instagram_database_db_name,
-      'username': matrix_mx_puppet_instagram_database_username,
-      'password': matrix_mx_puppet_instagram_database_password,
+      'name': matrix_mx_puppet_instagram_postgres_db_name,
+      'username': matrix_mx_puppet_instagram_postgres_username,
+      'password': matrix_mx_puppet_instagram_postgres_password,
     }] if (matrix_mx_puppet_instagram_enabled
            and matrix_mx_puppet_instagram_database_engine == 'postgres'
            and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_discord_database_db_name,
-      'username': matrix_mx_puppet_discord_database_username,
-      'password': matrix_mx_puppet_discord_database_password,
+      'name': matrix_mx_puppet_discord_postgres_db_name,
+      'username': matrix_mx_puppet_discord_postgres_username,
+      'password': matrix_mx_puppet_discord_postgres_password,
     }] if (matrix_mx_puppet_discord_enabled
            and matrix_mx_puppet_discord_database_engine == 'postgres'
            and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_steam_database_db_name,
-      'username': matrix_mx_puppet_steam_database_username,
-      'password': matrix_mx_puppet_steam_database_password,
+      'name': matrix_mx_puppet_steam_postgres_db_name,
+      'username': matrix_mx_puppet_steam_postgres_username,
+      'password': matrix_mx_puppet_steam_postgres_password,
     }] if (matrix_mx_puppet_steam_enabled
            and matrix_mx_puppet_steam_database_engine == 'postgres'
            and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_dimension_database_db_name,
-      'username': matrix_dimension_database_username,
-      'password': matrix_dimension_database_password,
+      'name': matrix_dimension_postgres_db_name,
+      'username': matrix_dimension_postgres_username,
+      'password': matrix_dimension_postgres_password,
     }] if (matrix_dimension_enabled
            and matrix_dimension_database_engine == 'postgres'
            and matrix_dimension_database_hostname == 'matrix-postgres') else [])

From 5dba0c038b6c19a9b430210cfaaa7538e383cef9 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 11:47:00 +0200
Subject: [PATCH 064/179] Make --tags=import-generic-sqlite-db commands not
 pass a sensitive connection string around

Instead of passing the connection string, we can now pass a name of a
variable, which contains a connection string.

Both are supported for having extra flexibility.
---
 .../tasks/setup_install.yml                   |  2 +-
 .../tasks/import_generic_sqlite_db.yml        | 27 ++++++++++++++-----
 2 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
index 86a5fe51..a9d8da4c 100644
--- a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
+++ b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
@@ -15,7 +15,7 @@
         To continue using SQLite, opt into it explicitly: add `matrix_appservice_discord_database_engine: sqlite` to your vars.yml file and re-run this same command.
         Alternatively, to migrate your existing SQLite database to Postgres:
         1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_discord_sqlite_database_path_local }} postgres_db_connection_string={{ matrix_appservice_discord_database_connString }}'`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_discord_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_appservice_discord_database_connString'`)
         3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
     when: "matrix_appservice_discord_sqlite_database_path_local_stat_result.stat.exists"
   when: "matrix_appservice_discord_database_engine == 'postgres'"
diff --git a/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml b/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml
index f2798a73..a42c6f55 100644
--- a/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml
+++ b/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml
@@ -12,13 +12,6 @@
     msg: "The `sqlite_database_path` variable needs to be provided to this playbook, via --extra-vars"
   when: "sqlite_database_path is not defined or sqlite_database_path.startswith('<')"
 
-- name: Fail if playbook called incorrectly
-  fail:
-    msg: >-
-      The `postgres_db_connection_string` variable needs to be provided to this playbook, via `--extra-vars`.
-      Example: `--extra-vars="postgres_db_connection_string=postgresql://username:password@localhost:<port>/database_name`"
-  when: "postgres_db_connection_string is not defined or not postgres_db_connection_string.startswith('postgresql://')"
-
 - name: Check if the provided SQLite database file exists
   stat:
     path: "{{ sqlite_database_path }}"
@@ -29,6 +22,26 @@
     msg: "File cannot be found on the server at {{ sqlite_database_path }}"
   when: "not sqlite_database_path_stat_result.stat.exists"
 
+# We either expect `postgres_db_connection_string` specifying a full Postgres database connection string,
+# or `postgres_connection_string_variable_name`, specifying a name of a variable, which contains a valid connection string.
+
+- block:
+    - name: Fail if postgres_connection_string_variable_name points to an undefined variable
+      fail: msg="postgres_connection_string_variable_name is defined, but there is no variable with the name `{{ postgres_connection_string_variable_name }}`"
+      when: "postgres_connection_string_variable_name not in vars"
+
+    - name: Get Postgres connection string from variable
+      set_fact:
+        postgres_db_connection_string: "{{ lookup('vars', postgres_connection_string_variable_name) }}"
+  when: 'postgres_connection_string_variable_name is defined'
+
+- name: Fail if playbook called incorrectly
+  fail:
+    msg: >-
+      Either a `postgres_db_connection_string` variable or a `postgres_connection_string_variable_name` needs to be provided to this playbook, via `--extra-vars`.
+      Example: `--extra-vars="postgres_db_connection_string=postgresql://username:password@localhost:<port>/database_name"` or `--extra-vars="postgres_connection_string_variable_name=matrix_appservice_discord_database_connString"`
+  when: "postgres_db_connection_string is not defined or not postgres_db_connection_string.startswith('postgresql://')"
+
 
 # Defaults
 

From 2d99ade72f09c9f25b718f0cda690319d9bea526 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 11:50:42 +0200
Subject: [PATCH 065/179] Add (Postgres + SQLite) support to
 matrix-mautrix-facebook bridge

---
 .../defaults/main.yml                         | 29 +++++++++++++++++++
 .../tasks/setup_install.yml                   | 21 ++++++++++++++
 .../templates/config.yaml.j2                  |  2 +-
 .../matrix-mautrix-facebook.service.j2        |  1 +
 4 files changed, 52 insertions(+), 1 deletion(-)

diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
index 1547ad71..95adf354 100644
--- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
@@ -32,6 +32,35 @@ matrix_mautrix_facebook_systemd_wanted_services_list: []
 matrix_mautrix_facebook_appservice_token: ''
 matrix_mautrix_facebook_homeserver_token: ''
 
+
+
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_mautrix_facebook_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_mautrix_facebook_postgres_*` variables
+matrix_mautrix_facebook_database_engine: 'sqlite'
+
+matrix_mautrix_facebook_sqlite_database_path_local: "{{ matrix_mautrix_facebook_data_path }}/mautrix-facebook.db"
+matrix_mautrix_facebook_sqlite_database_path_in_container: "/data/mautrix-facebook.db"
+
+matrix_mautrix_facebook_postgres_username: 'matrix_mautrix_facebook'
+matrix_mautrix_facebook_postgres_password: 'some-password'
+matrix_mautrix_facebook_postgres_hostname: 'matrix-postgres'
+matrix_mautrix_facebook_postgres_port: 5432
+matrix_mautrix_facebook_postgres_db_name: 'matrix_mautrix_facebook'
+
+matrix_mautrix_facebook_postgres_connection_string: 'postgresql://{{ matrix_mautrix_facebook_postgres_username }}:{{ matrix_mautrix_facebook_postgres_password }}@{{ matrix_mautrix_facebook_postgres_hostname }}:{{ matrix_mautrix_facebook_postgres_port }}/{{ matrix_mautrix_facebook_postgres_db_name }}'
+
+matrix_mautrix_facebook_appservice_database: "{{
+	{
+		'sqlite': matrix_mautrix_facebook_sqlite_database_path_in_container,
+		'postgres': matrix_mautrix_facebook_postgres_connection_string,
+	}[matrix_mautrix_facebook_database_engine]
+}}"
+
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mautrix_facebook_login_shared_secret: ''
 
diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
index 638854e2..ef814108 100644
--- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
@@ -1,5 +1,26 @@
 ---
 
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_mautrix_facebook_sqlite_database_path_local }}"
+    register: matrix_mautrix_facebook_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_mautrix_facebook_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mautrix_facebook_sqlite_database_path_local }}.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mautrix_facebook_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_facebook_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_facebook_postgres_connection_string'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mautrix_facebook_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mautrix_facebook_database_engine == 'postgres'"
+
+
 # If the matrix-synapse role is not used, `matrix_synapse_role_executed` won't exist.
 # We don't want to fail in such cases.
 - name: Fail if matrix-synapse role already executed
diff --git a/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2
index 304571c1..09287362 100644
--- a/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2
@@ -27,7 +27,7 @@ appservice:
     # Format examples:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
-    database: sqlite://matrix_bridge_facebook:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_bridge_facebook') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_bridge_facebook
+    database: {{ matrix_mautrix_facebook_appservice_database|to_json }}
 
     # Public part of web server for out-of-Matrix interaction with the bridge.
     public:
diff --git a/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 b/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2
index 18184650..caa52eb5 100644
--- a/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2
+++ b/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2
@@ -18,6 +18,7 @@ ExecStartPre={{ matrix_host_command_docker }} run --rm --name matrix-mautrix-fac
 			--log-driver=none \
 			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
 			--cap-drop=ALL \
+			--network={{ matrix_docker_network }} \
 			-v {{ matrix_mautrix_facebook_config_path }}:/config:z \
 			-v {{ matrix_mautrix_facebook_data_path }}:/data:z \
 			{{ matrix_mautrix_facebook_docker_image }} \

From a3406a182bf9590d7ee018867ee4fbd02c6473b5 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 12:04:47 +0200
Subject: [PATCH 066/179] Move some things around

---
 .../defaults/main.yml                           |  2 +-
 .../tasks/setup_install.yml                     | 17 ++++++++---------
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
index 95adf354..d0b39682 100644
--- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
@@ -33,7 +33,6 @@ matrix_mautrix_facebook_appservice_token: ''
 matrix_mautrix_facebook_homeserver_token: ''
 
 
-
 # Database-related configuration fields.
 #
 # To use SQLite, stick to these defaults.
@@ -61,6 +60,7 @@ matrix_mautrix_facebook_appservice_database: "{{
 	}[matrix_mautrix_facebook_database_engine]
 }}"
 
+
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mautrix_facebook_login_shared_secret: ''
 
diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
index ef814108..c3c8d61f 100644
--- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
@@ -1,5 +1,13 @@
 ---
 
+# If the matrix-synapse role is not used, `matrix_synapse_role_executed` won't exist.
+# We don't want to fail in such cases.
+- name: Fail if matrix-synapse role already executed
+  fail:
+    msg: >-
+      The matrix-bridge-mautrix-facebook role needs to execute before the matrix-synapse role.
+  when: "matrix_synapse_role_executed|default(False)"
+
 - block:
   - name: Check if an SQLite database already exists
     stat:
@@ -20,15 +28,6 @@
     when: "matrix_mautrix_facebook_sqlite_database_path_local_stat_result.stat.exists"
   when: "matrix_mautrix_facebook_database_engine == 'postgres'"
 
-
-# If the matrix-synapse role is not used, `matrix_synapse_role_executed` won't exist.
-# We don't want to fail in such cases.
-- name: Fail if matrix-synapse role already executed
-  fail:
-    msg: >-
-      The matrix-bridge-mautrix-facebook role needs to execute before the matrix-synapse role.
-  when: "matrix_synapse_role_executed|default(False)"
-
 - name: Ensure Mautrix Facebook image is pulled
   docker_image:
     name: "{{ matrix_mautrix_facebook_docker_image }}"

From 9bf8ce878e9451f2f3aa24d200d9d10dbf08e8b3 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 12:06:23 +0200
Subject: [PATCH 067/179] Add (Postgres + SQLite) support to
 matrix-mautrix-telegram bridge

I don't use this bridge, so this is completely untested.
---
 .../defaults/main.yml                         | 29 +++++++++++++++++++
 .../tasks/setup_install.yml                   | 20 +++++++++++++
 .../templates/config.yaml.j2                  |  2 +-
 3 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
index b6f4ef5b..d0df6a7d 100644
--- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
@@ -43,6 +43,35 @@ matrix_mautrix_telegram_systemd_wanted_services_list: []
 matrix_mautrix_telegram_appservice_token: ''
 matrix_mautrix_telegram_homeserver_token: ''
 
+
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_mautrix_telegram_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_mautrix_telegram_postgres_*` variables
+matrix_mautrix_telegram_database_engine: 'sqlite'
+
+matrix_mautrix_telegram_sqlite_database_path_local: "{{ matrix_mautrix_telegram_data_path }}/mautrix-telegram.db"
+matrix_mautrix_telegram_sqlite_database_path_in_container: "/data/mautrix-telegram.db"
+
+matrix_mautrix_telegram_postgres_username: 'matrix_mautrix_telegram'
+matrix_mautrix_telegram_postgres_password: 'some-password'
+matrix_mautrix_telegram_postgres_hostname: 'matrix-postgres'
+matrix_mautrix_telegram_postgres_port: 5432
+matrix_mautrix_telegram_postgres_db_name: 'matrix_mautrix_telegram'
+
+matrix_mautrix_telegram_postgres_connection_string: 'postgresql://{{ matrix_mautrix_telegram_postgres_username }}:{{ matrix_mautrix_telegram_postgres_password }}@{{ matrix_mautrix_telegram_postgres_hostname }}:{{ matrix_mautrix_telegram_postgres_port }}/{{ matrix_mautrix_telegram_postgres_db_name }}'
+
+matrix_mautrix_telegram_appservice_database: "{{
+	{
+		'sqlite': matrix_mautrix_telegram_sqlite_database_path_in_container,
+		'postgres': matrix_mautrix_telegram_postgres_connection_string,
+	}[matrix_mautrix_telegram_database_engine]
+}}"
+
+
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mautrix_telegram_login_shared_secret: ''
 
diff --git a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml
index 9dc38ec2..db1a8f83 100644
--- a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml
@@ -8,6 +8,26 @@
       The matrix-bridge-mautrix-telegram role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_mautrix_telegram_sqlite_database_path_local }}"
+    register: matrix_mautrix_telegram_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_mautrix_telegram_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mautrix_telegram_sqlite_database_path_local }}.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mautrix_telegram_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_telegram_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_telegram_postgres_connection_string'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mautrix_telegram_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mautrix_telegram_database_engine == 'postgres'"
+
 - name: Ensure Mautrix Telegram image is pulled
   docker_image:
     name: "{{ matrix_mautrix_telegram_docker_image }}"
diff --git a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2
index 65f17d33..52efba02 100644
--- a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2
@@ -27,7 +27,7 @@ appservice:
     # Format examples:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
-    database: postgres://mautrix_bridge_telegram:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_bridge_telegram') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/mautrix_bridge_telegram
+    database: {{ matrix_mautrix_telegram_appservice_database|to_json }}
 
     # Public part of web server for out-of-Matrix interaction with the bridge.
     # Used for things like login if the user wants to make sure the 2FA password isn't stored in

From 05dd091133ad7833ca544bd4ccda55311c0450c5 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 12:11:41 +0200
Subject: [PATCH 068/179] Rename `_database_hostname` to `_postgres_hostname`

Should have been done during 2a99e84b5bfd9f39, but I've missed it.

Because of this, 9bf8ce878e945 was also incomplete and would have caused
an error.
---
 group_vars/matrix_servers | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 5e3695ca..0bc0c0c5 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -958,12 +958,12 @@ matrix_postgres_additional_databases: |
       'name': matrix_appservice_slack_postgres_db_name,
       'username': matrix_appservice_slack_postgres_username,
       'password': matrix_appservice_slack_postgres_password,
-    }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres') else [])
+    }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_postgres_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_appservice_irc_postgres_db_name,
       'username': matrix_appservice_irc_postgres_username,
       'password': matrix_appservice_irc_postgres_password,
-    }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres') else [])
+    }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_postgres_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_mautrix_facebook_postgres_db_name,
       'username': matrix_mautrix_facebook_postgres_username,
@@ -973,17 +973,17 @@ matrix_postgres_additional_databases: |
       'name': matrix_mautrix_hangouts_postgres_db_name,
       'username': matrix_mautrix_hangouts_postgres_username,
       'password': matrix_mautrix_hangouts_postgres_password,
-    }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_postgres_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_mautrix_telegram_postgres_db_name,
       'username': matrix_mautrix_telegram_postgres_username,
       'password': matrix_mautrix_telegram_postgres_password,
-    }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_telegram_facebook_database_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_telegram_facebook_postgres_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_mautrix_whatsapp_postgres_db_name,
       'username': matrix_mautrix_whatsapp_postgres_username,
       'password': matrix_mautrix_whatsapp_postgres_password,
-    }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_postgres_hostname == 'matrix-postgres') else [])
     + ([{
       'name': 'matrix_bridge_sms',
       'username': 'matrix_bridge_sms',
@@ -993,45 +993,45 @@ matrix_postgres_additional_databases: |
       'name': matrix_mx_puppet_skype_postgres_db_name,
       'username': matrix_mx_puppet_skype_postgres_username,
       'password': matrix_mx_puppet_skype_postgres_password,
-    }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_postgres_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_mx_puppet_slack_postgres_db_name,
       'username': matrix_mx_puppet_slack_postgres_username,
       'password': matrix_mx_puppet_slack_postgres_password,
-    }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_postgres_hostname == 'matrix-postgres') else [])
      + ([{
       'name': matrix_mx_puppet_twitter_postgres_db_name,
       'username': matrix_mx_puppet_twitter_postgres_username,
       'password': matrix_mx_puppet_twitter_postgres_password,
-    }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_postgres_hostname == 'matrix-postgres') else [])
      + ([{
       'name': matrix_mx_puppet_instagram_postgres_db_name,
       'username': matrix_mx_puppet_instagram_postgres_username,
       'password': matrix_mx_puppet_instagram_postgres_password,
     }] if (matrix_mx_puppet_instagram_enabled
            and matrix_mx_puppet_instagram_database_engine == 'postgres'
-           and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres') else [])
+           and matrix_mx_puppet_instagram_postgres_hostname == 'matrix-postgres') else [])
      + ([{
       'name': matrix_mx_puppet_discord_postgres_db_name,
       'username': matrix_mx_puppet_discord_postgres_username,
       'password': matrix_mx_puppet_discord_postgres_password,
     }] if (matrix_mx_puppet_discord_enabled
            and matrix_mx_puppet_discord_database_engine == 'postgres'
-           and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres') else [])
+           and matrix_mx_puppet_discord_postgres_hostname == 'matrix-postgres') else [])
      + ([{
       'name': matrix_mx_puppet_steam_postgres_db_name,
       'username': matrix_mx_puppet_steam_postgres_username,
       'password': matrix_mx_puppet_steam_postgres_password,
     }] if (matrix_mx_puppet_steam_enabled
            and matrix_mx_puppet_steam_database_engine == 'postgres'
-           and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else [])
+           and matrix_mx_puppet_steam_postgres_hostname == 'matrix-postgres') else [])
      + ([{
       'name': matrix_dimension_postgres_db_name,
       'username': matrix_dimension_postgres_username,
       'password': matrix_dimension_postgres_password,
     }] if (matrix_dimension_enabled
            and matrix_dimension_database_engine == 'postgres'
-           and matrix_dimension_database_hostname == 'matrix-postgres') else [])
+           and matrix_dimension_postgres_hostname == 'matrix-postgres') else [])
    }}
 
 ######################################################################

From 6a2dd8088cea10713ffd760d32cc789821ae5ae9 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 12:13:10 +0200
Subject: [PATCH 069/179] Fix typo (matrix_telegram_facebook_postgres_hostname
 -> matrix_mautrix_telegram_postgres_hostname)

Related to 9bf8ce878e945.
---
 group_vars/matrix_servers | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 0bc0c0c5..626498c0 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -978,7 +978,7 @@ matrix_postgres_additional_databases: |
       'name': matrix_mautrix_telegram_postgres_db_name,
       'username': matrix_mautrix_telegram_postgres_username,
       'password': matrix_mautrix_telegram_postgres_password,
-    }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_telegram_facebook_postgres_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_mautrix_telegram_postgres_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_mautrix_whatsapp_postgres_db_name,
       'username': matrix_mautrix_whatsapp_postgres_username,

From 2848322461e451d2fb1985ecf797d87e5765ea03 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 12:19:35 +0200
Subject: [PATCH 070/179] Remove some remaining `_database_password` usage in
 favor of `_postgres_pasword`

The only one that remains is `matrix_synapse_database_password`, but
that's something old and should be dealt with separately in the future
(unless it remains as it is).
---
 group_vars/matrix_servers | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 626498c0..e3f16d3f 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -142,7 +142,7 @@ matrix_appservice_slack_systemd_required_services_list: |
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_appservice_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_appservice_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.slack.db') | to_uuid }}"
+matrix_appservice_slack_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.slack.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -181,7 +181,7 @@ matrix_appservice_irc_homeserver_token: "{{ matrix_synapse_macaroon_secret_key |
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_appservice_irc_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_appservice_irc_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.irc.db') | to_uuid }}"
+matrix_appservice_irc_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.irc.db') | to_uuid }}"
 
 
 ######################################################################
@@ -219,7 +219,7 @@ matrix_mautrix_facebook_bridge_presence: "{{ matrix_synapse_use_presence if matr
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_facebook_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_facebook_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_facebook.db') | to_uuid }}"
+matrix_mautrix_facebook_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_facebook.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -256,7 +256,7 @@ matrix_mautrix_hangouts_login_shared_secret: "{{ matrix_synapse_ext_password_pro
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_hangouts_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_hangouts_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_hangouts.db') | to_uuid }}"
+matrix_mautrix_hangouts_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_hangouts.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -296,7 +296,7 @@ matrix_mautrix_telegram_login_shared_secret: "{{ matrix_synapse_ext_password_pro
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_telegram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_telegram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_telegram.db') | to_uuid }}"
+matrix_mautrix_telegram_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_telegram.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -328,7 +328,7 @@ matrix_mautrix_whatsapp_login_shared_secret: "{{ matrix_synapse_ext_password_pro
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_whatsapp_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_whatsapp_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_whatsapp.db') | to_uuid }}"
+matrix_mautrix_whatsapp_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_whatsapp.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -388,7 +388,7 @@ matrix_mx_puppet_skype_login_shared_secret: "{{ matrix_synapse_ext_password_prov
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_skype_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_skype_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_skype.db') | to_uuid }}"
+matrix_mx_puppet_skype_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_skype.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -423,7 +423,7 @@ matrix_mx_puppet_slack_login_shared_secret: "{{ matrix_synapse_ext_password_prov
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_slack.db') | to_uuid }}"
+matrix_mx_puppet_slack_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_slack.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -459,7 +459,7 @@ matrix_mx_puppet_twitter_container_http_host_bind_port: "{{ '' if matrix_nginx_p
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_twitter_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_twitter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_twitter.db') | to_uuid }}"
+matrix_mx_puppet_twitter_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_twitter.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -494,7 +494,7 @@ matrix_mx_puppet_instagram_login_shared_secret: "{{ matrix_synapse_ext_password_
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_instagram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_instagram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_instagram.db') | to_uuid }}"
+matrix_mx_puppet_instagram_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_instagram.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -528,7 +528,7 @@ matrix_mx_puppet_discord_login_shared_secret: "{{ matrix_synapse_ext_password_pr
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_discord_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_discord.db') | to_uuid }}"
+matrix_mx_puppet_discord_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_discord.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -562,7 +562,7 @@ matrix_mx_puppet_steam_login_shared_secret: "{{ matrix_synapse_ext_password_prov
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_stream_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_steam.db') | to_uuid }}"
+matrix_mx_puppet_steam_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_steam.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -676,7 +676,7 @@ matrix_integration_manager_ui_url: "{{ matrix_dimension_integrations_ui_url if m
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_dimension_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_dimension_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.dimension.db') | to_uuid }}"
+matrix_dimension_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.dimension.db') | to_uuid }}"
 
 ######################################################################
 #

From 6c77eae96935f6a78c6ad170134d0797b18520af Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 12:24:37 +0200
Subject: [PATCH 071/179] Add (Postgres + SQLite) support to
 matrix-mautrix-whatsapp bridge

I don't use this bridge, so this is completely untested.
---
 .../defaults/main.yml                         | 36 +++++++++++++++++++
 .../tasks/setup_install.yml                   | 26 ++++++++++++--
 .../templates/config.yaml.j2                  |  4 +--
 3 files changed, 61 insertions(+), 5 deletions(-)

diff --git a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
index e9929f96..6cabea3e 100644
--- a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
@@ -27,6 +27,42 @@ matrix_mautrix_whatsapp_systemd_wanted_services_list: []
 matrix_mautrix_whatsapp_appservice_token: ''
 matrix_mautrix_whatsapp_homeserver_token: ''
 
+
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_mautrix_whatsapp_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_mautrix_whatsapp_postgres_*` variables
+matrix_mautrix_whatsapp_database_engine: 'sqlite'
+
+matrix_mautrix_whatsapp_sqlite_database_path_local: "{{ matrix_mautrix_whatsapp_data_path }}/mautrix-whatsapp.db"
+matrix_mautrix_whatsapp_sqlite_database_path_in_container: "/data/mautrix-whatsapp.db"
+
+matrix_mautrix_whatsapp_postgres_username: 'matrix_mautrix_whatsapp'
+matrix_mautrix_whatsapp_postgres_password: 'some-password'
+matrix_mautrix_whatsapp_postgres_hostname: 'matrix-postgres'
+matrix_mautrix_whatsapp_postgres_port: 5432
+matrix_mautrix_whatsapp_postgres_db_name: 'matrix_mautrix_whatsapp'
+
+matrix_mautrix_whatsapp_postgres_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_postgres_username }}:{{ matrix_mautrix_whatsapp_postgres_password }}@{{ matrix_mautrix_whatsapp_postgres_hostname }}:{{ matrix_mautrix_whatsapp_postgres_port }}/{{ matrix_mautrix_whatsapp_postgres_db_name }}'
+
+matrix_mautrix_whatsapp_appservice_database_type: "{{
+	{
+		'sqlite': 'sqlite3',
+		'postgres':'postgres',
+	}[matrix_mautrix_whatsapp_database_engine]
+}}"
+
+matrix_mautrix_whatsapp_appservice_database_uri: "{{
+	{
+		'sqlite': matrix_mautrix_whatsapp_sqlite_database_path_in_container,
+		'postgres': matrix_mautrix_whatsapp_postgres_connection_string,
+	}[matrix_mautrix_whatsapp_database_engine]
+}}"
+
+
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mautrix_whatsapp_login_shared_secret: ''
 
diff --git a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml
index 6bd87bbe..15453e8c 100644
--- a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml
@@ -8,6 +8,26 @@
       The matrix-bridge-mautrix-whatsapp role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_mautrix_whatsapp_sqlite_database_path_local }}"
+    register: matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_mautrix_whatsapp_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mautrix_whatsapp_sqlite_database_path_local }}.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mautrix_whatsapp_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_whatsapp_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_whatsapp_postgres_connection_string'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mautrix_whatsapp_database_engine == 'postgres'"
+
 - name: Ensure Mautrix Whatsapp image is pulled
   docker_image:
     name: "{{ matrix_mautrix_whatsapp_docker_image }}"
@@ -26,12 +46,12 @@
     - "{{ matrix_mautrix_whatsapp_base_path }}"
     - "{{ matrix_mautrix_whatsapp_config_path }}"
     - "{{ matrix_mautrix_whatsapp_data_path }}"
-    
+
 - name: Check if an old database file exists
   stat:
     path: "{{ matrix_mautrix_whatsapp_base_path }}/mautrix-whatsapp.db"
   register: matrix_mautrix_whatsapp_stat_database
-  
+
 - name: Check if an old matrix state file exists
   stat:
     path: "{{ matrix_mautrix_whatsapp_base_path }}/mx-state.json"
@@ -48,7 +68,7 @@
 - name: (Data relocation) Move mautrix-whatsapp database file to ./data directory
   command: "mv {{ matrix_mautrix_whatsapp_base_path }}/mautrix-whatsapp.db {{ matrix_mautrix_whatsapp_data_path }}/mautrix-whatsapp.db"
   when: "matrix_mautrix_whatsapp_stat_database.stat.exists"
-  
+
 - name: (Data relocation) Move mautrix-whatsapp mx-state file to ./data directory
   command: "mv {{ matrix_mautrix_whatsapp_base_path }}/mx-state.json {{ matrix_mautrix_whatsapp_data_path }}/mx-state.json"
   when: "matrix_mautrix_whatsapp_stat_mx_state.stat.exists"
diff --git a/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2
index 93956049..89216695 100644
--- a/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2
@@ -19,11 +19,11 @@ appservice:
     # Database config.
     database:
         # The database type. "sqlite3" and "postgres" are supported.
-        type: sqlite3
+        type: {{ matrix_mautrix_whatsapp_appservice_database_type|to_json }}
         # The database URI.
         #   SQLite: File name is enough. https://github.com/mattn/go-sqlite3#connection-string
         #   Postgres: Connection string. For example, postgres://user:password@host/database
-        uri: postgres://matrix_bridge_whatsapp@{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_bridge_whatsapp') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_bridge_whatsapp
+        uri: {{ matrix_mautrix_whatsapp_appservice_database_uri|to_json }}
         # Maximum number of connections. Mostly relevant for Postgres.
         max_open_conns: 20
         max_idle_conns: 2

From 43d6ff2af83abf0610c73b507eb4d7acae045052 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 12:30:10 +0200
Subject: [PATCH 072/179] Fix sqlite usage for
 mautrix-facebook/mautrix-telegram

Regression since 2d99ade72f and 9bf8ce878e94, respectively.

When SQLite is to be used, these bridges expect an `sqlite://`
connection string, and not a plain file name (path), like Appservice
Discord and mautrix-whatsapp do.
---
 roles/matrix-bridge-mautrix-facebook/defaults/main.yml | 2 +-
 roles/matrix-bridge-mautrix-telegram/defaults/main.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
index d0b39682..97584997 100644
--- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
@@ -55,7 +55,7 @@ matrix_mautrix_facebook_postgres_connection_string: 'postgresql://{{ matrix_maut
 
 matrix_mautrix_facebook_appservice_database: "{{
 	{
-		'sqlite': matrix_mautrix_facebook_sqlite_database_path_in_container,
+		'sqlite': ('sqlite://' + matrix_mautrix_facebook_sqlite_database_path_in_container),
 		'postgres': matrix_mautrix_facebook_postgres_connection_string,
 	}[matrix_mautrix_facebook_database_engine]
 }}"
diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
index d0df6a7d..9ba9be09 100644
--- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
@@ -66,7 +66,7 @@ matrix_mautrix_telegram_postgres_connection_string: 'postgresql://{{ matrix_maut
 
 matrix_mautrix_telegram_appservice_database: "{{
 	{
-		'sqlite': matrix_mautrix_telegram_sqlite_database_path_in_container,
+		'sqlite': ('sqlite://' + matrix_mautrix_telegram_sqlite_database_path_in_container),
 		'postgres': matrix_mautrix_telegram_postgres_connection_string,
 	}[matrix_mautrix_telegram_database_engine]
 }}"

From ce21ea3640283889be6ed7c526116ca6c47ca921 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 12:34:59 +0200
Subject: [PATCH 073/179] Add (Postgres + SQLite) support to
 matrix-mautrix-hangouts bridge

I don't use this bridge, so this is completely untested.
---
 .../defaults/main.yml                         | 29 +++++++++++++++++++
 .../tasks/setup_install.yml                   | 20 +++++++++++++
 .../templates/config.yaml.j2                  |  2 +-
 .../matrix-mautrix-hangouts.service.j2        |  1 +
 4 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
index ab670052..af0fe061 100644
--- a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
@@ -39,6 +39,35 @@ matrix_mautrix_hangouts_systemd_wanted_services_list: []
 matrix_mautrix_hangouts_appservice_token: ''
 matrix_mautrix_hangouts_homeserver_token: ''
 
+
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_mautrix_hangouts_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_mautrix_hangouts_postgres_*` variables
+matrix_mautrix_hangouts_database_engine: 'sqlite'
+
+matrix_mautrix_hangouts_sqlite_database_path_local: "{{ matrix_mautrix_hangouts_data_path }}/mautrix-hangouts.db"
+matrix_mautrix_hangouts_sqlite_database_path_in_container: "/data/mautrix-hangouts.db"
+
+matrix_mautrix_hangouts_postgres_username: 'matrix_mautrix_hangouts'
+matrix_mautrix_hangouts_postgres_password: 'some-password'
+matrix_mautrix_hangouts_postgres_hostname: 'matrix-postgres'
+matrix_mautrix_hangouts_postgres_port: 5432
+matrix_mautrix_hangouts_postgres_db_name: 'matrix_mautrix_hangouts'
+
+matrix_mautrix_hangouts_postgres_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_postgres_username }}:{{ matrix_mautrix_hangouts_postgres_password }}@{{ matrix_mautrix_hangouts_postgres_hostname }}:{{ matrix_mautrix_hangouts_postgres_port }}/{{ matrix_mautrix_hangouts_postgres_db_name }}'
+
+matrix_mautrix_hangouts_appservice_database: "{{
+	{
+		'sqlite': ('sqlite://' + matrix_mautrix_hangouts_sqlite_database_path_in_container),
+		'postgres': matrix_mautrix_hangouts_postgres_connection_string,
+	}[matrix_mautrix_hangouts_database_engine]
+}}"
+
+
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mautrix_hangouts_login_shared_secret: ''
 
diff --git a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml
index eec5e006..9e7bfbba 100644
--- a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml
@@ -8,6 +8,26 @@
       The matrix-bridge-mautrix-hangouts role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_mautrix_hangouts_sqlite_database_path_local }}"
+    register: matrix_mautrix_hangouts_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_mautrix_hangouts_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mautrix_hangouts_sqlite_database_path_local }}.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mautrix_hangouts_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_hangouts_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_hangouts_postgres_connection_string'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mautrix_hangouts_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mautrix_hangouts_database_engine == 'postgres'"
+
 - name: Ensure Mautrix Hangouts image is pulled
   docker_image:
     name: "{{ matrix_mautrix_hangouts_docker_image }}"
diff --git a/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2
index e4e59ad2..cc2ca90b 100644
--- a/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2
@@ -27,7 +27,7 @@ appservice:
     # Format examples:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
-    database: postgres://mautrix_bridge_hangouts:{{ matrix_additional_databases | selectattr('name', 'equalto', 'mautrix_bridge_hangouts') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/mautrix_bridge_hangouts
+    database: {{ matrix_mautrix_hangouts_appservice_database|to_json }}
 
     # The unique ID of this appservice.
     id: hangouts
diff --git a/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2 b/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2
index 26280da3..39559190 100644
--- a/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2
+++ b/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2
@@ -18,6 +18,7 @@ ExecStartPre={{ matrix_host_command_docker }} run --rm --name matrix-mautrix-han
 			--log-driver=none \
 			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
 			--cap-drop=ALL \
+			--network={{ matrix_docker_network }} \
 			-v {{ matrix_mautrix_hangouts_config_path }}:/config:z \
 			-v {{ matrix_mautrix_hangouts_data_path }}:/data:z \
 			{{ matrix_mautrix_hangouts_docker_image }} \

From 087dbe4ddc80ba6308e7ee98391ea475354e8860 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 13:02:47 +0200
Subject: [PATCH 074/179] Rename `matrix_*_postgres_*` back to
 `matrix_*_database_*`

I was thinking that it makes sense to be more specific,
and using `_postgres_` also separated these variables
from the `_database_` variables that ended up in bridge configuration.

However, @jdreichmann makes a good point
(https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/740#discussion_r542281102)
that we don't need to be so specific and can allow for other engines (like MySQL) to use these variables.
---
 group_vars/matrix_servers                     | 112 +++++++++---------
 .../defaults/main.yml                         |  10 +-
 .../defaults/main.yml                         |  10 +-
 .../defaults/main.yml                         |  10 +-
 .../defaults/main.yml                         |  10 +-
 .../defaults/main.yml                         |  10 +-
 6 files changed, 81 insertions(+), 81 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index e3f16d3f..acf133e1 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -70,7 +70,7 @@ matrix_appservice_discord_homeserver_token: "{{ matrix_synapse_macaroon_secret_k
 # We only make this use Postgres if our own Postgres server is enabled.
 # It's only then (for now) that we can automatically create the necessary database and user for this service.
 matrix_appservice_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_appservice_discord_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.discord.db') | to_uuid }}"
+matrix_appservice_discord_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.discord.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -142,7 +142,7 @@ matrix_appservice_slack_systemd_required_services_list: |
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_appservice_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_appservice_slack_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.slack.db') | to_uuid }}"
+matrix_appservice_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.slack.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -181,7 +181,7 @@ matrix_appservice_irc_homeserver_token: "{{ matrix_synapse_macaroon_secret_key |
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_appservice_irc_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_appservice_irc_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.irc.db') | to_uuid }}"
+matrix_appservice_irc_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.irc.db') | to_uuid }}"
 
 
 ######################################################################
@@ -219,7 +219,7 @@ matrix_mautrix_facebook_bridge_presence: "{{ matrix_synapse_use_presence if matr
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_facebook_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_facebook_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_facebook.db') | to_uuid }}"
+matrix_mautrix_facebook_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_facebook.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -256,7 +256,7 @@ matrix_mautrix_hangouts_login_shared_secret: "{{ matrix_synapse_ext_password_pro
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_hangouts_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_hangouts_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_hangouts.db') | to_uuid }}"
+matrix_mautrix_hangouts_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_hangouts.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -296,7 +296,7 @@ matrix_mautrix_telegram_login_shared_secret: "{{ matrix_synapse_ext_password_pro
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_telegram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_telegram_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_telegram.db') | to_uuid }}"
+matrix_mautrix_telegram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_telegram.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -328,7 +328,7 @@ matrix_mautrix_whatsapp_login_shared_secret: "{{ matrix_synapse_ext_password_pro
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_whatsapp_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_whatsapp_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_whatsapp.db') | to_uuid }}"
+matrix_mautrix_whatsapp_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_whatsapp.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -388,7 +388,7 @@ matrix_mx_puppet_skype_login_shared_secret: "{{ matrix_synapse_ext_password_prov
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_skype_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_skype_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_skype.db') | to_uuid }}"
+matrix_mx_puppet_skype_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_skype.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -423,7 +423,7 @@ matrix_mx_puppet_slack_login_shared_secret: "{{ matrix_synapse_ext_password_prov
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_slack_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_slack.db') | to_uuid }}"
+matrix_mx_puppet_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_slack.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -459,7 +459,7 @@ matrix_mx_puppet_twitter_container_http_host_bind_port: "{{ '' if matrix_nginx_p
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_twitter_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_twitter_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_twitter.db') | to_uuid }}"
+matrix_mx_puppet_twitter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_twitter.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -494,7 +494,7 @@ matrix_mx_puppet_instagram_login_shared_secret: "{{ matrix_synapse_ext_password_
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_instagram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_instagram_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_instagram.db') | to_uuid }}"
+matrix_mx_puppet_instagram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_instagram.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -528,7 +528,7 @@ matrix_mx_puppet_discord_login_shared_secret: "{{ matrix_synapse_ext_password_pr
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_discord_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_discord.db') | to_uuid }}"
+matrix_mx_puppet_discord_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_discord.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -562,7 +562,7 @@ matrix_mx_puppet_steam_login_shared_secret: "{{ matrix_synapse_ext_password_prov
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_stream_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_steam_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_steam.db') | to_uuid }}"
+matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_steam.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -676,7 +676,7 @@ matrix_integration_manager_ui_url: "{{ matrix_dimension_integrations_ui_url if m
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_dimension_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_dimension_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.dimension.db') | to_uuid }}"
+matrix_dimension_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.dimension.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -950,39 +950,39 @@ matrix_postgres_db_name: "homeserver"
 matrix_postgres_additional_databases: |
   {{
     ([{
-      'name': matrix_appservice_discord_postgres_db_name,
-      'username': matrix_appservice_discord_postgres_username,
-      'password': matrix_appservice_discord_postgres_password,
+      'name': matrix_appservice_discord_database_db_name,
+      'username': matrix_appservice_discord_database_username,
+      'password': matrix_appservice_discord_database_password,
     }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_postgres_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_appservice_slack_postgres_db_name,
-      'username': matrix_appservice_slack_postgres_username,
-      'password': matrix_appservice_slack_postgres_password,
+      'name': matrix_appservice_slack_database_db_name,
+      'username': matrix_appservice_slack_database_username,
+      'password': matrix_appservice_slack_database_password,
     }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_postgres_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_appservice_irc_postgres_db_name,
-      'username': matrix_appservice_irc_postgres_username,
-      'password': matrix_appservice_irc_postgres_password,
+      'name': matrix_appservice_irc_database_db_name,
+      'username': matrix_appservice_irc_database_username,
+      'password': matrix_appservice_irc_database_password,
     }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_postgres_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_facebook_postgres_db_name,
-      'username': matrix_mautrix_facebook_postgres_username,
-      'password': matrix_mautrix_facebook_postgres_password,
+      'name': matrix_mautrix_facebook_database_db_name,
+      'username': matrix_mautrix_facebook_database_username,
+      'password': matrix_mautrix_facebook_database_password,
     }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_postgres_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_hangouts_postgres_db_name,
-      'username': matrix_mautrix_hangouts_postgres_username,
-      'password': matrix_mautrix_hangouts_postgres_password,
+      'name': matrix_mautrix_hangouts_database_db_name,
+      'username': matrix_mautrix_hangouts_database_username,
+      'password': matrix_mautrix_hangouts_database_password,
     }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_postgres_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_telegram_postgres_db_name,
-      'username': matrix_mautrix_telegram_postgres_username,
-      'password': matrix_mautrix_telegram_postgres_password,
+      'name': matrix_mautrix_telegram_database_db_name,
+      'username': matrix_mautrix_telegram_database_username,
+      'password': matrix_mautrix_telegram_database_password,
     }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_mautrix_telegram_postgres_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_whatsapp_postgres_db_name,
-      'username': matrix_mautrix_whatsapp_postgres_username,
-      'password': matrix_mautrix_whatsapp_postgres_password,
+      'name': matrix_mautrix_whatsapp_database_db_name,
+      'username': matrix_mautrix_whatsapp_database_username,
+      'password': matrix_mautrix_whatsapp_database_password,
     }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_postgres_hostname == 'matrix-postgres') else [])
     + ([{
       'name': 'matrix_bridge_sms',
@@ -990,45 +990,45 @@ matrix_postgres_additional_databases: |
       'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid,
     }] if matrix_sms_bridge_enabled else [])
     + ([{
-      'name': matrix_mx_puppet_skype_postgres_db_name,
-      'username': matrix_mx_puppet_skype_postgres_username,
-      'password': matrix_mx_puppet_skype_postgres_password,
+      'name': matrix_mx_puppet_skype_database_db_name,
+      'username': matrix_mx_puppet_skype_database_username,
+      'password': matrix_mx_puppet_skype_database_password,
     }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_postgres_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mx_puppet_slack_postgres_db_name,
-      'username': matrix_mx_puppet_slack_postgres_username,
-      'password': matrix_mx_puppet_slack_postgres_password,
+      'name': matrix_mx_puppet_slack_database_db_name,
+      'username': matrix_mx_puppet_slack_database_username,
+      'password': matrix_mx_puppet_slack_database_password,
     }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_postgres_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_twitter_postgres_db_name,
-      'username': matrix_mx_puppet_twitter_postgres_username,
-      'password': matrix_mx_puppet_twitter_postgres_password,
+      'name': matrix_mx_puppet_twitter_database_db_name,
+      'username': matrix_mx_puppet_twitter_database_username,
+      'password': matrix_mx_puppet_twitter_database_password,
     }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_postgres_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_instagram_postgres_db_name,
-      'username': matrix_mx_puppet_instagram_postgres_username,
-      'password': matrix_mx_puppet_instagram_postgres_password,
+      'name': matrix_mx_puppet_instagram_database_db_name,
+      'username': matrix_mx_puppet_instagram_database_username,
+      'password': matrix_mx_puppet_instagram_database_password,
     }] if (matrix_mx_puppet_instagram_enabled
            and matrix_mx_puppet_instagram_database_engine == 'postgres'
            and matrix_mx_puppet_instagram_postgres_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_discord_postgres_db_name,
-      'username': matrix_mx_puppet_discord_postgres_username,
-      'password': matrix_mx_puppet_discord_postgres_password,
+      'name': matrix_mx_puppet_discord_database_db_name,
+      'username': matrix_mx_puppet_discord_database_username,
+      'password': matrix_mx_puppet_discord_database_password,
     }] if (matrix_mx_puppet_discord_enabled
            and matrix_mx_puppet_discord_database_engine == 'postgres'
            and matrix_mx_puppet_discord_postgres_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_steam_postgres_db_name,
-      'username': matrix_mx_puppet_steam_postgres_username,
-      'password': matrix_mx_puppet_steam_postgres_password,
+      'name': matrix_mx_puppet_steam_database_db_name,
+      'username': matrix_mx_puppet_steam_database_username,
+      'password': matrix_mx_puppet_steam_database_password,
     }] if (matrix_mx_puppet_steam_enabled
            and matrix_mx_puppet_steam_database_engine == 'postgres'
            and matrix_mx_puppet_steam_postgres_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_dimension_postgres_db_name,
-      'username': matrix_dimension_postgres_username,
-      'password': matrix_dimension_postgres_password,
+      'name': matrix_dimension_database_db_name,
+      'username': matrix_dimension_database_username,
+      'password': matrix_dimension_database_password,
     }] if (matrix_dimension_enabled
            and matrix_dimension_database_engine == 'postgres'
            and matrix_dimension_postgres_hostname == 'matrix-postgres') else [])
diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml
index fb517cd5..57a88867 100644
--- a/roles/matrix-bridge-appservice-discord/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml
@@ -53,16 +53,16 @@ matrix_appservice_discord_database_engine: 'sqlite'
 matrix_appservice_discord_sqlite_database_path_local: "{{ matrix_appservice_discord_data_path }}/discord.db"
 matrix_appservice_discord_sqlite_database_path_in_container: "/data/discord.db"
 
-matrix_appservice_discord_postgres_username: 'matrix_appservice_discord'
-matrix_appservice_discord_postgres_password: 'some-password'
+matrix_appservice_discord_database_username: 'matrix_appservice_discord'
+matrix_appservice_discord_database_password: 'some-password'
 matrix_appservice_discord_postgres_hostname: 'matrix-postgres'
-matrix_appservice_discord_postgres_port: 5432
-matrix_appservice_discord_postgres_db_name: 'matrix_appservice_discord'
+matrix_appservice_discord_database_port: 5432
+matrix_appservice_discord_database_db_name: 'matrix_appservice_discord'
 
 # These 2 variables are what actually ends up in the bridge configuration.
 # It's best if you don't change them directly, but rather redefine the sub-variables that constitute them.
 matrix_appservice_discord_database_filename: "{{ matrix_appservice_discord_sqlite_database_path_in_container }}"
-matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_postgres_username }}:{{ matrix_appservice_discord_postgres_password }}@{{ matrix_appservice_discord_postgres_hostname }}:{{ matrix_appservice_discord_postgres_port }}/{{ matrix_appservice_discord_postgres_db_name }}'
+matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_username }}:{{ matrix_appservice_discord_database_password }}@{{ matrix_appservice_discord_postgres_hostname }}:{{ matrix_appservice_discord_database_port }}/{{ matrix_appservice_discord_database_db_name }}'
 
 
 # Tells whether the bot should make use of "Privileged Gateway Intents".
diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
index 97584997..246c32fe 100644
--- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
@@ -45,13 +45,13 @@ matrix_mautrix_facebook_database_engine: 'sqlite'
 matrix_mautrix_facebook_sqlite_database_path_local: "{{ matrix_mautrix_facebook_data_path }}/mautrix-facebook.db"
 matrix_mautrix_facebook_sqlite_database_path_in_container: "/data/mautrix-facebook.db"
 
-matrix_mautrix_facebook_postgres_username: 'matrix_mautrix_facebook'
-matrix_mautrix_facebook_postgres_password: 'some-password'
+matrix_mautrix_facebook_database_username: 'matrix_mautrix_facebook'
+matrix_mautrix_facebook_database_password: 'some-password'
 matrix_mautrix_facebook_postgres_hostname: 'matrix-postgres'
-matrix_mautrix_facebook_postgres_port: 5432
-matrix_mautrix_facebook_postgres_db_name: 'matrix_mautrix_facebook'
+matrix_mautrix_facebook_database_port: 5432
+matrix_mautrix_facebook_database_db_name: 'matrix_mautrix_facebook'
 
-matrix_mautrix_facebook_postgres_connection_string: 'postgresql://{{ matrix_mautrix_facebook_postgres_username }}:{{ matrix_mautrix_facebook_postgres_password }}@{{ matrix_mautrix_facebook_postgres_hostname }}:{{ matrix_mautrix_facebook_postgres_port }}/{{ matrix_mautrix_facebook_postgres_db_name }}'
+matrix_mautrix_facebook_postgres_connection_string: 'postgresql://{{ matrix_mautrix_facebook_database_username }}:{{ matrix_mautrix_facebook_database_password }}@{{ matrix_mautrix_facebook_postgres_hostname }}:{{ matrix_mautrix_facebook_database_port }}/{{ matrix_mautrix_facebook_database_db_name }}'
 
 matrix_mautrix_facebook_appservice_database: "{{
 	{
diff --git a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
index af0fe061..04280ac4 100644
--- a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
@@ -52,13 +52,13 @@ matrix_mautrix_hangouts_database_engine: 'sqlite'
 matrix_mautrix_hangouts_sqlite_database_path_local: "{{ matrix_mautrix_hangouts_data_path }}/mautrix-hangouts.db"
 matrix_mautrix_hangouts_sqlite_database_path_in_container: "/data/mautrix-hangouts.db"
 
-matrix_mautrix_hangouts_postgres_username: 'matrix_mautrix_hangouts'
-matrix_mautrix_hangouts_postgres_password: 'some-password'
+matrix_mautrix_hangouts_database_username: 'matrix_mautrix_hangouts'
+matrix_mautrix_hangouts_database_password: 'some-password'
 matrix_mautrix_hangouts_postgres_hostname: 'matrix-postgres'
-matrix_mautrix_hangouts_postgres_port: 5432
-matrix_mautrix_hangouts_postgres_db_name: 'matrix_mautrix_hangouts'
+matrix_mautrix_hangouts_database_port: 5432
+matrix_mautrix_hangouts_database_db_name: 'matrix_mautrix_hangouts'
 
-matrix_mautrix_hangouts_postgres_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_postgres_username }}:{{ matrix_mautrix_hangouts_postgres_password }}@{{ matrix_mautrix_hangouts_postgres_hostname }}:{{ matrix_mautrix_hangouts_postgres_port }}/{{ matrix_mautrix_hangouts_postgres_db_name }}'
+matrix_mautrix_hangouts_postgres_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_database_username }}:{{ matrix_mautrix_hangouts_database_password }}@{{ matrix_mautrix_hangouts_postgres_hostname }}:{{ matrix_mautrix_hangouts_database_port }}/{{ matrix_mautrix_hangouts_database_db_name }}'
 
 matrix_mautrix_hangouts_appservice_database: "{{
 	{
diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
index 9ba9be09..4f584128 100644
--- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
@@ -56,13 +56,13 @@ matrix_mautrix_telegram_database_engine: 'sqlite'
 matrix_mautrix_telegram_sqlite_database_path_local: "{{ matrix_mautrix_telegram_data_path }}/mautrix-telegram.db"
 matrix_mautrix_telegram_sqlite_database_path_in_container: "/data/mautrix-telegram.db"
 
-matrix_mautrix_telegram_postgres_username: 'matrix_mautrix_telegram'
-matrix_mautrix_telegram_postgres_password: 'some-password'
+matrix_mautrix_telegram_database_username: 'matrix_mautrix_telegram'
+matrix_mautrix_telegram_database_password: 'some-password'
 matrix_mautrix_telegram_postgres_hostname: 'matrix-postgres'
-matrix_mautrix_telegram_postgres_port: 5432
-matrix_mautrix_telegram_postgres_db_name: 'matrix_mautrix_telegram'
+matrix_mautrix_telegram_database_port: 5432
+matrix_mautrix_telegram_database_db_name: 'matrix_mautrix_telegram'
 
-matrix_mautrix_telegram_postgres_connection_string: 'postgresql://{{ matrix_mautrix_telegram_postgres_username }}:{{ matrix_mautrix_telegram_postgres_password }}@{{ matrix_mautrix_telegram_postgres_hostname }}:{{ matrix_mautrix_telegram_postgres_port }}/{{ matrix_mautrix_telegram_postgres_db_name }}'
+matrix_mautrix_telegram_postgres_connection_string: 'postgresql://{{ matrix_mautrix_telegram_database_username }}:{{ matrix_mautrix_telegram_database_password }}@{{ matrix_mautrix_telegram_postgres_hostname }}:{{ matrix_mautrix_telegram_database_port }}/{{ matrix_mautrix_telegram_database_db_name }}'
 
 matrix_mautrix_telegram_appservice_database: "{{
 	{
diff --git a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
index 6cabea3e..357f5892 100644
--- a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
@@ -40,13 +40,13 @@ matrix_mautrix_whatsapp_database_engine: 'sqlite'
 matrix_mautrix_whatsapp_sqlite_database_path_local: "{{ matrix_mautrix_whatsapp_data_path }}/mautrix-whatsapp.db"
 matrix_mautrix_whatsapp_sqlite_database_path_in_container: "/data/mautrix-whatsapp.db"
 
-matrix_mautrix_whatsapp_postgres_username: 'matrix_mautrix_whatsapp'
-matrix_mautrix_whatsapp_postgres_password: 'some-password'
+matrix_mautrix_whatsapp_database_username: 'matrix_mautrix_whatsapp'
+matrix_mautrix_whatsapp_database_password: 'some-password'
 matrix_mautrix_whatsapp_postgres_hostname: 'matrix-postgres'
-matrix_mautrix_whatsapp_postgres_port: 5432
-matrix_mautrix_whatsapp_postgres_db_name: 'matrix_mautrix_whatsapp'
+matrix_mautrix_whatsapp_database_port: 5432
+matrix_mautrix_whatsapp_database_db_name: 'matrix_mautrix_whatsapp'
 
-matrix_mautrix_whatsapp_postgres_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_postgres_username }}:{{ matrix_mautrix_whatsapp_postgres_password }}@{{ matrix_mautrix_whatsapp_postgres_hostname }}:{{ matrix_mautrix_whatsapp_postgres_port }}/{{ matrix_mautrix_whatsapp_postgres_db_name }}'
+matrix_mautrix_whatsapp_postgres_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_database_username }}:{{ matrix_mautrix_whatsapp_database_password }}@{{ matrix_mautrix_whatsapp_postgres_hostname }}:{{ matrix_mautrix_whatsapp_database_port }}/{{ matrix_mautrix_whatsapp_database_db_name }}'
 
 matrix_mautrix_whatsapp_appservice_database_type: "{{
 	{

From 5d70bc1376f30c4453beffe0c23ccb4c3004635c Mon Sep 17 00:00:00 2001
From: transcaffeine <transcaffeine@finallycoffee.eu>
Date: Mon, 14 Dec 2020 12:38:51 +0100
Subject: [PATCH 075/179] add postgres support for mx-puppet-* with fallback to
 sqlite in role and migration notice

---
 .../defaults/main.yml                         | 15 +++++++++++++
 .../tasks/validate_config.yml                 | 20 +++++++++++++++++
 .../templates/config.yaml.j2                  | 11 ++++++----
 .../defaults/main.yml                         | 14 ++++++++++++
 .../tasks/validate_config.yml                 | 21 ++++++++++++++++++
 .../templates/config.yaml.j2                  |  7 ++++--
 .../defaults/main.yml                         | 15 +++++++++++++
 .../tasks/validate_config.yml                 | 20 +++++++++++++++++
 .../templates/config.yaml.j2                  |  7 ++++--
 .../defaults/main.yml                         | 15 +++++++++++++
 .../tasks/validate_config.yml                 | 20 +++++++++++++++++
 .../templates/config.yaml.j2                  |  7 ++++--
 .../defaults/main.yml                         | 14 ++++++++++++
 .../tasks/validate_config.yml                 | 22 +++++++++++++++++++
 .../templates/config.yaml.j2                  |  7 ++++--
 .../defaults/main.yml                         | 15 +++++++++++++
 .../tasks/validate_config.yml                 | 22 +++++++++++++++++++
 .../templates/config.yaml.j2                  |  7 ++++--
 18 files changed, 245 insertions(+), 14 deletions(-)

diff --git a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
index 268bbf8f..5a65f33e 100644
--- a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
@@ -55,6 +55,21 @@ matrix_mx_puppet_discord_homeserver_token: ''
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mx_puppet_discord_login_shared_secret: ''
 
+# Database configuration
+matrix_mx_puppet_discord_database_engine: 'sqlite'
+matrix_mx_puppet_discord_database_username: matrix_mx_puppet_discord
+matrix_mx_puppet_discord_database_password: ~
+matrix_mx_puppet_discord_database_hostname: 'matrix-postgres'
+matrix_mx_puppet_discord_database_port: 5432
+matrix_mx_puppet_discord_database_name: matrix_mx_puppet_discord
+matrix_mx_puppet_discord_database_file: /data/database.db
+matrix_mx_puppet_discord_database_connString: >-2
+  {%- if matrix_mx_puppet_discord_database_engine == 'postgres' -%}
+  postgresql://{{ matrix_mx_puppet_discord_database_username }}:{{ matrix_mx_puppet_discord_database_password }}@{{ matrix_mx_puppet_discord_database_hostname }}:{{ matrix_mx_puppet_discord_database_port }}/{{ matrix_mx_puppet_discord_database_name }}?sslmode=disable
+  {%- else if matrix_mx_puppet_discord_database_engine == 'sqlite' -%}
+  {{ matrix_mx_puppet_discord_database_engine }}://{{ matrix_mx_puppet_discord_database_file }}
+  {%- endif -%}
+
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
 #
diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml
index c253eda2..8db47ed1 100644
--- a/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml
@@ -8,3 +8,23 @@
   with_items:
     - "matrix_mx_puppet_discord_appservice_token"
     - "matrix_mx_puppet_discord_homeserver_token"
+
+- block:
+  - name: Check if a SQLite database already exists
+    stat:
+      path: "{{ matrix_mx_puppet_discord_data_path }}/database.db"
+    register: matrix_mx_puppet_discord_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-2
+        matrix_mx_puppet_discord_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_discord_data_path }}/database.db.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_discord_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_discord_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_discord_database_connString'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mx_puppet_discord_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mx_puppet_discord_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
index 88c591e2..c24c7e54 100644
--- a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
@@ -100,15 +100,18 @@ namePatterns:
   group: :name
 
 database:
+{% if matrix_mx_puppet_slack_discord_database_engine == 'sqlite' %}
+  # Use SQLite3 as a database backend
+  # The name of the database file
+  filename: /data/database.db
+{% else %}
   # Use Postgres as a database backend
   # If set, will be used instead of SQLite3
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: "postgres://matrix_puppet_discord:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_discord') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_discord?sslmode=disable"
-  # Use SQLite3 as a database backend
-  # The name of the database file
-  #filename: /data/database.db
+  connString: {{ matrix_mx_puppet_discord_database_connString|to_json }}
+{% endif %}
 
 logging:
   # Log level of console output
diff --git a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
index 21cde53f..c30f9b8b 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
@@ -46,6 +46,20 @@ matrix_mx_puppet_instagram_homeserver_token: ''
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mx_puppet_instagram_login_shared_secret: ''
 
+matrix_mx_puppet_instagram_database_engine: sqlite
+matrix_mx_puppet_instagram_database_username: matrix_mx_puppet_instagram
+matrix_mx_puppet_instagram_database_password: ~
+matrix_mx_puppet_instagram_database_hostname: 'matrix-postgres'
+matrix_mx_puppet_instagram_database_port: 5432
+matrix_mx_puppet_instagram_database_name: matrix_mx_puppet_instagram
+matrix_mx_puppet_instagram_database_file: /data/database.db
+matrix_mx_puppet_instagram_database_connString: >-2
+  {%- if matrix_mx_puppet_instagram_database_engine == 'postgres' -%}
+  postgresql://{{ matrix_mx_puppet_instagram_database_username }}:{{ matrix_mx_puppet_instagram_database_password }}@{{ matrix_mx_puppet_instagram_database_hostname }}:{{ matrix_mx_puppet_instagram_database_port }}/{{ matrix_mx_puppet_instagram_database_name }}?sslmode=disable
+  {%- else if matrix_mx_puppet_instagram_database_engine == 'sqlite' -%}
+  {{ matrix_mx_puppet_instagram_database_engine }}://{{ matrix_mx_puppet_instagram_database_file }}
+  {%- endif -%}
+
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
 #
diff --git a/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml
index b6d9d994..7000a8a1 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml
@@ -8,3 +8,24 @@
   with_items:
     - "matrix_mx_puppet_instagram_appservice_token"
     - "matrix_mx_puppet_instagram_homeserver_token"
+
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_mx_puppet_instagram_data_path }}/database.db"
+    register: matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result
+
+   - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_mx_puppet_instagram_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_instagram_data_path }}/database.db.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_instagram_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_instagram_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_instagram_database_connString'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mx_puppet_instagram_database_engine == 'postgres'"
+
diff --git a/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2
index 2d189195..d89c5bdb 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2
@@ -44,15 +44,18 @@ provisioning:
   apiPrefix: /_matrix/provision
 
 database:
+{% if matrix_mx_puppet_instagram_database_engine == 'postgres' %}
   # Use Postgres as a database backend
   # If set, will be used instead of SQLite3
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: "postgres://matrix_puppet_instagram:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_instagram') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_instagram?sslmode=disable"
+  connString: {{ matrix_mx_puppet_instagram_database_connString | to_json }}
+{% else %}
   # Use SQLite3 as a database backend
   # The name of the database file
-  #filename: /data/database.db
+  filename: {{ matrix_mx_puppet_instagram_database_file }}
+{% endif %}
 
 logging:
   # Log level of console output
diff --git a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
index 0c16a8ba..3b7448c6 100644
--- a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
@@ -53,6 +53,21 @@ matrix_mx_puppet_skype_homeserver_token: ''
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mx_puppet_skype_login_shared_secret: ''
 
+# Database configuration, role default is `sqlite` but playbook default is `postgres`
+matrix_mx_puppet_skype_database_engine: sqlite
+matrix_mx_puppet_skype_database_username: matrix_mx_puppet_skype
+matrix_mx_puppet_skype_database_password: ~
+matrix_mx_puppet_skype_database_hostname: 'matrix-postgres'
+matrix_mx_puppet_skype_database_port: 5432
+matrix_mx_puppet_skype_database_name: matrix_mx_puppet_skype
+matrix_mx_puppet_skype_database_file: /data/database.db
+matrix_mx_puppet_skype_database_connString: >-2
+  {%- if matrix_mx_puppet_skype_database_engine == 'postgres' -%}
+  postgresql://{{ matrix_mx_puppet_skype_database_username }}:{{ matrix_mx_puppet_skype_database_password }}@{{ matrix_mx_puppet_skype_database_hostname }}:{{ matrix_mx_puppet_skype_database_port }}/{{ matrix_mx_puppet_skype_database_name }}?sslmode=disable
+  {%- else if matrix_mx_puppet_skype_database_engine == 'sqlite' -%}
+  {{ matrix_mx_puppet_skype_database_engine }}://{{ matrix_mx_puppet_skype_database_file }}
+  {%- endif -%}
+
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
 #
diff --git a/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml
index 7ed433b1..c7100e51 100644
--- a/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml
@@ -8,3 +8,23 @@
   with_items:
     - "matrix_mx_puppet_skype_appservice_token"
     - "matrix_mx_puppet_skype_homeserver_token"
+
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_mx_puppet_skype_data_path }}/database.db"
+    register: matrix_mx_puppet_skype_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_mx_puppet_skype_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_skype_data_path }}/database.db.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_skype_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_skype_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_skype_database_connString'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mx_puppet_skype_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mx_puppet_skype_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2
index a54ca758..a32523b1 100644
--- a/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2
@@ -68,15 +68,18 @@ logging:
       #  - Store
 
 database:
+{% if matrix_mx_puppet_skype_database_engine == 'postgres' %}
   # Use Postgres as a database backend
   # If set, will be used instead of SQLite3
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: "postgres://matrix_puppet_skype:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_skype') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_skype?sslmode=disable"
+  connString: {{ matrix_mx_puppet_skype_database_connString | to_json }}
+{% else %}
   # Use SQLite3 as a database backend
   # The name of the database file
-  #filename: /data/database.db
+  filename: {{ matrix_mx_puppet_skype_database_file }}
+{% endif %}
 
 provisioning:
   # Regex of Matrix IDs allowed to use the puppet bridge
diff --git a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
index 96ab7833..5f91beb8 100644
--- a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
@@ -57,6 +57,21 @@ matrix_mx_puppet_slack_homeserver_token: ''
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mx_puppet_slack_login_shared_secret: ''
 
+# Database configuration, role uses 'sqlite' per default but playbook sets up postgres by default
+matrix_mx_puppet_slack_database_engine: sqlite
+matrix_mx_puppet_slack_database_username: matrix_mx_puppet_slack
+matrix_mx_puppet_slack_database_password: ~
+matrix_mx_puppet_slack_database_hostname: 'matrix-postgres'
+matrix_mx_puppet_slack_database_port: 5432
+matrix_mx_puppet_slack_database_name: matrix_mx_puppet_slack
+matrix_mx_puppet_slack_database_file: /data/database.db
+matrix_mx_puppet_slack_database_connString: >-2
+  {%- if matrix_mx_puppet_slack_database_engine == 'postgres' -%}
+  postgresql://{{ matrix_mx_puppet_slack_database_username }}:{{ matrix_mx_puppet_slack_database_password }}@{{ matrix_mx_puppet_slack_database_hostname }}:{{ matrix_mx_puppet_slack_database_port }}/{{ matrix_mx_puppet_slack_database_name }}?sslmode=disable
+  {%- else if matrix_mx_puppet_slack_database_engine == 'sqlite' -%}
+  {{ matrix_mx_puppet_slack_database_engine }}://{{ matrix_mx_puppet_slack_database_file }}
+  {%- endif -%}
+
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
 #
diff --git a/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml
index 3a0bca11..bc722ee6 100644
--- a/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml
@@ -8,3 +8,23 @@
   with_items:
     - "matrix_mx_puppet_slack_appservice_token"
     - "matrix_mx_puppet_slack_homeserver_token"
+
+- block:
+  - name: Check if sqlite database already exists
+    stat:
+      path: "{{ matrix_mx_puppet_slack_data_path }}/database.db"
+    register: matrix_mx_puppet_slack_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_mx_puppet_slack_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_slack_data_path }}/database.db.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_slack_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_slack_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_slack_database_connString'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mx_puppet_slack_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mx_puppet_slack_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2
index 7f1b21d7..58c5c11a 100644
--- a/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2
@@ -58,15 +58,18 @@ provisioning:
   apiPrefix: /_matrix/provision
 
 database:
+{% if matrix_mx_puppet_slack_database_engine == 'postgres' %}
   # Use Postgres as a database backend
   # If set, will be used instead of SQLite3
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: "postgres://matrix_puppet_slack:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_slack') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_slack?sslmode=disable"
+  connString: {{ matrix_mx_puppet_slack_database_connString | to_json }}
+{% else %}
   # Use SQLite3 as a database backend
   # The name of the database file
-  #filename: /data/database.db
+  filename: {{ matrix_mx_puppet_slack_database_file }}
+{% endif %}
 
 logging:
   # Log level of console output
diff --git a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
index b5eb8473..be343db4 100644
--- a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
@@ -55,6 +55,20 @@ matrix_mx_puppet_steam_homeserver_token: ''
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mx_puppet_steam_login_shared_secret: ''
 
+matrix_mx_puppet_steam_database_engine: sqlite
+matrix_mx_puppet_steam_database_username: matrix_mx_puppet_steam
+matrix_mx_puppet_steam_database_password: ~
+matrix_mx_puppet_steam_database_hostname: 'matrix-postgres'
+matrix_mx_puppet_steam_database_port: 5432
+matrix_mx_puppet_steam_database_name: matrix_mx_puppet_steam
+matrix_mx_puppet_steam_database_file: /data/database.db
+matrix_mx_puppet_steam_database_connString: >-2
+  {%- if matrix_mx_puppet_steam_database_engine == 'postgres' -%}
+  postgresql://{{ matrix_mx_puppet_steam_database_username }}:{{ matrix_mx_puppet_steam_database_password }}@{{ matrix_mx_puppet_steam_database_hostname }}:{{ matrix_mx_puppet_steam_database_port }}/{{ matrix_mx_puppet_steam_database_name }}?sslmode=disable
+  {%- else if matrix_mx_puppet_steam_database_engine == 'sqlite' -%}
+  {{ matrix_mx_puppet_steam_database_engine }}://{{ matrix_mx_puppet_steam_database_file }}
+  {%- endif -%}
+
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
 #
diff --git a/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml
index a8bc6a42..50ce15d9 100644
--- a/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml
@@ -8,3 +8,25 @@
   with_items:
     - "matrix_mx_puppet_steam_appservice_token"
     - "matrix_mx_puppet_steam_homeserver_token"
+
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_mx_puppet_steam_data_path }}"
+    register: matrix_mx_puppet_steam_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_mx_puppet_steam_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_steam_data_path }}/database.db.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_steam_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_steam_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_steam_database_connString'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mx_puppet_steam_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mx_puppet_steam_database_engine == 'postgres'"
+
+
diff --git a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2
index 14f86319..8eda278d 100644
--- a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2
@@ -61,15 +61,18 @@ selfService:
   blacklist: {{ matrix_mx_puppet_steam_provisioning_blacklist|to_json }}
 
 database:
+{% if matrix_mx_puppet_steam_database_engine == 'postgres' %}
   # Use Postgres as a database backend
   # If set, will be used instead of SQLite3
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: "postgres://matrix_puppet_steam:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_steam') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_steam?sslmode=disable"
+  connString: {{ matrix_mx_puppet_steam_database_connString | to_json }}
+{% else %}
   # Use SQLite3 as a database backend
   # The name of the database file
-  #filename: /data/database.db
+  filename: {{ matrix_mx_puppet_steam_database_file }}
+{% endif %}
 
 logging:
   # Log level of console output
diff --git a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
index 13438a21..f536029c 100644
--- a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
@@ -60,6 +60,21 @@ matrix_mx_puppet_twitter_homeserver_token: ''
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mx_puppet_twitter_login_shared_secret: ''
 
+# Database configuration
+matrix_mx_puppet_twitter_database_engine: sqlite
+matrix_mx_puppet_twitter_database_username: matrix:mx_puppet_twitter
+matrix_mx_puppet_twitter_database_password: ~
+matrix_mx_puppet_twitter_database_hostname: 'matrix-postgres'
+matrix_mx_puppet_twitter_database_port: 5432
+matrix_mx_puppet_twitter_database_name: matrix_mx_puppet_twitter
+matrix_mx_puppet_twitter_database_file: /data/database.db
+matrix_mx_puppet_twitter_database_connString: >-2
+  {%- if matrix_mx_puppet_twitter_database_engine == 'postgres' -%}
+  postgresql://{{ matrix_mx_puppet_twitter_database_username }}:{{ matrix_mx_puppet_twitter_database_password }}@{{ matrix_mx_puppet_twitter_database_hostname }}:{{ matrix_mx_puppet_twitter_database_port }}/{{ matrix_mx_puppet_twitter_database_name }}?sslmode=disable
+  {%- else if matrix_mx_puppet_twitter_database_engine == 'sqlite' -%}
+  {{ matrix_mx_puppet_twitter_database_engine }}://{{ matrix_mx_puppet_twitter_database_file }}
+  {%- endif -%}
+
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
 #
diff --git a/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml
index d13a39e1..0b3bd8f8 100644
--- a/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml
@@ -8,3 +8,25 @@
   with_items:
     - "matrix_mx_puppet_twitter_appservice_token"
     - "matrix_mx_puppet_twitter_homeserver_token"
+
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_mx_puppet_twitter_sqlite_data_path }}/database.db"
+    register: matrix_mx_puppet_twitter_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_mx_puppet_twitter_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_twitter_data_path }}/database.db.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_twitter_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_twitter_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_twitter_database_connString'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mx_puppet_twitter_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mx_puppet_twitter_database_engine == 'postgres'"
+
+
diff --git a/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2
index 853d23fc..cecb171d 100644
--- a/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2
@@ -54,15 +54,18 @@ provisioning:
   apiPrefix: /_matrix/provision
 
 database:
+{% if matrix_mx_puppet_twitter_database_engine == 'postgres' %}
   # Use Postgres as a database backend
   # If set, will be used instead of SQLite3
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: "postgres://matrix_puppet_twitter:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_twitter') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_twitter?sslmode=disable"
+  connString: {{ matrix_mx_puppet_twitter_database_connString | to_json }}
+{% else %}
   # Use SQLite3 as a database backend
   # The name of the database file
-  #filename: /data/database.db
+  filename: {{ matrix_mx_puppet_twitter_database_file }}
+{% endif %}
 
 logging:
   # Log level of console output

From b9a04a7f953e504a3e6798eeebcc786f51bf9788 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 14:42:18 +0200
Subject: [PATCH 076/179] Rename some remaining `matrix_*_postgres_*` vars back
 to `matrix_*_database_*`

Looks like there are some that I missed in 087dbe4ddc80ba6
---
 group_vars/matrix_servers                     | 28 +++++++++----------
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  6 ++--
 .../tasks/setup_install.yml                   |  2 +-
 .../defaults/main.yml                         |  6 ++--
 .../tasks/setup_install.yml                   |  2 +-
 .../defaults/main.yml                         |  6 ++--
 .../tasks/setup_install.yml                   |  2 +-
 .../defaults/main.yml                         |  6 ++--
 .../tasks/setup_install.yml                   |  2 +-
 10 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index acf133e1..ad1e8ce3 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -953,37 +953,37 @@ matrix_postgres_additional_databases: |
       'name': matrix_appservice_discord_database_db_name,
       'username': matrix_appservice_discord_database_username,
       'password': matrix_appservice_discord_database_password,
-    }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_postgres_hostname == 'matrix-postgres') else [])
+    }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_database_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_appservice_slack_database_db_name,
       'username': matrix_appservice_slack_database_username,
       'password': matrix_appservice_slack_database_password,
-    }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_postgres_hostname == 'matrix-postgres') else [])
+    }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_appservice_irc_database_db_name,
       'username': matrix_appservice_irc_database_username,
       'password': matrix_appservice_irc_database_password,
-    }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_postgres_hostname == 'matrix-postgres') else [])
+    }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_mautrix_facebook_database_db_name,
       'username': matrix_mautrix_facebook_database_username,
       'password': matrix_mautrix_facebook_database_password,
-    }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_postgres_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_database_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_mautrix_hangouts_database_db_name,
       'username': matrix_mautrix_hangouts_database_username,
       'password': matrix_mautrix_hangouts_database_password,
-    }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_postgres_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_mautrix_telegram_database_db_name,
       'username': matrix_mautrix_telegram_database_username,
       'password': matrix_mautrix_telegram_database_password,
-    }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_mautrix_telegram_postgres_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_mautrix_telegram_database_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_mautrix_whatsapp_database_db_name,
       'username': matrix_mautrix_whatsapp_database_username,
       'password': matrix_mautrix_whatsapp_database_password,
-    }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_postgres_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres') else [])
     + ([{
       'name': 'matrix_bridge_sms',
       'username': 'matrix_bridge_sms',
@@ -993,45 +993,45 @@ matrix_postgres_additional_databases: |
       'name': matrix_mx_puppet_skype_database_db_name,
       'username': matrix_mx_puppet_skype_database_username,
       'password': matrix_mx_puppet_skype_database_password,
-    }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_postgres_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_mx_puppet_slack_database_db_name,
       'username': matrix_mx_puppet_slack_database_username,
       'password': matrix_mx_puppet_slack_database_password,
-    }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_postgres_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres') else [])
      + ([{
       'name': matrix_mx_puppet_twitter_database_db_name,
       'username': matrix_mx_puppet_twitter_database_username,
       'password': matrix_mx_puppet_twitter_database_password,
-    }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_postgres_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres') else [])
      + ([{
       'name': matrix_mx_puppet_instagram_database_db_name,
       'username': matrix_mx_puppet_instagram_database_username,
       'password': matrix_mx_puppet_instagram_database_password,
     }] if (matrix_mx_puppet_instagram_enabled
            and matrix_mx_puppet_instagram_database_engine == 'postgres'
-           and matrix_mx_puppet_instagram_postgres_hostname == 'matrix-postgres') else [])
+           and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres') else [])
      + ([{
       'name': matrix_mx_puppet_discord_database_db_name,
       'username': matrix_mx_puppet_discord_database_username,
       'password': matrix_mx_puppet_discord_database_password,
     }] if (matrix_mx_puppet_discord_enabled
            and matrix_mx_puppet_discord_database_engine == 'postgres'
-           and matrix_mx_puppet_discord_postgres_hostname == 'matrix-postgres') else [])
+           and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres') else [])
      + ([{
       'name': matrix_mx_puppet_steam_database_db_name,
       'username': matrix_mx_puppet_steam_database_username,
       'password': matrix_mx_puppet_steam_database_password,
     }] if (matrix_mx_puppet_steam_enabled
            and matrix_mx_puppet_steam_database_engine == 'postgres'
-           and matrix_mx_puppet_steam_postgres_hostname == 'matrix-postgres') else [])
+           and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else [])
      + ([{
       'name': matrix_dimension_database_db_name,
       'username': matrix_dimension_database_username,
       'password': matrix_dimension_database_password,
     }] if (matrix_dimension_enabled
            and matrix_dimension_database_engine == 'postgres'
-           and matrix_dimension_postgres_hostname == 'matrix-postgres') else [])
+           and matrix_dimension_database_hostname == 'matrix-postgres') else [])
    }}
 
 ######################################################################
diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml
index 57a88867..e2f97b85 100644
--- a/roles/matrix-bridge-appservice-discord/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml
@@ -55,14 +55,14 @@ matrix_appservice_discord_sqlite_database_path_in_container: "/data/discord.db"
 
 matrix_appservice_discord_database_username: 'matrix_appservice_discord'
 matrix_appservice_discord_database_password: 'some-password'
-matrix_appservice_discord_postgres_hostname: 'matrix-postgres'
+matrix_appservice_discord_database_hostname: 'matrix-postgres'
 matrix_appservice_discord_database_port: 5432
 matrix_appservice_discord_database_db_name: 'matrix_appservice_discord'
 
 # These 2 variables are what actually ends up in the bridge configuration.
 # It's best if you don't change them directly, but rather redefine the sub-variables that constitute them.
 matrix_appservice_discord_database_filename: "{{ matrix_appservice_discord_sqlite_database_path_in_container }}"
-matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_username }}:{{ matrix_appservice_discord_database_password }}@{{ matrix_appservice_discord_postgres_hostname }}:{{ matrix_appservice_discord_database_port }}/{{ matrix_appservice_discord_database_db_name }}'
+matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_username }}:{{ matrix_appservice_discord_database_password }}@{{ matrix_appservice_discord_database_hostname }}:{{ matrix_appservice_discord_database_port }}/{{ matrix_appservice_discord_database_db_name }}'
 
 
 # Tells whether the bot should make use of "Privileged Gateway Intents".
diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
index 246c32fe..c02197a5 100644
--- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
@@ -47,16 +47,16 @@ matrix_mautrix_facebook_sqlite_database_path_in_container: "/data/mautrix-facebo
 
 matrix_mautrix_facebook_database_username: 'matrix_mautrix_facebook'
 matrix_mautrix_facebook_database_password: 'some-password'
-matrix_mautrix_facebook_postgres_hostname: 'matrix-postgres'
+matrix_mautrix_facebook_database_hostname: 'matrix-postgres'
 matrix_mautrix_facebook_database_port: 5432
 matrix_mautrix_facebook_database_db_name: 'matrix_mautrix_facebook'
 
-matrix_mautrix_facebook_postgres_connection_string: 'postgresql://{{ matrix_mautrix_facebook_database_username }}:{{ matrix_mautrix_facebook_database_password }}@{{ matrix_mautrix_facebook_postgres_hostname }}:{{ matrix_mautrix_facebook_database_port }}/{{ matrix_mautrix_facebook_database_db_name }}'
+matrix_mautrix_facebook_database_connection_string: 'postgresql://{{ matrix_mautrix_facebook_database_username }}:{{ matrix_mautrix_facebook_database_password }}@{{ matrix_mautrix_facebook_database_hostname }}:{{ matrix_mautrix_facebook_database_port }}/{{ matrix_mautrix_facebook_database_db_name }}'
 
 matrix_mautrix_facebook_appservice_database: "{{
 	{
 		'sqlite': ('sqlite://' + matrix_mautrix_facebook_sqlite_database_path_in_container),
-		'postgres': matrix_mautrix_facebook_postgres_connection_string,
+		'postgres': matrix_mautrix_facebook_database_connection_string,
 	}[matrix_mautrix_facebook_database_engine]
 }}"
 
diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
index c3c8d61f..1a0ff70c 100644
--- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
@@ -23,7 +23,7 @@
         To continue using SQLite, opt into it explicitly: add `matrix_mautrix_facebook_database_engine: sqlite` to your vars.yml file and re-run this same command.
         Alternatively, to migrate your existing SQLite database to Postgres:
         1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_facebook_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_facebook_postgres_connection_string'`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_facebook_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_facebook_database_connection_string'`)
         3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
     when: "matrix_mautrix_facebook_sqlite_database_path_local_stat_result.stat.exists"
   when: "matrix_mautrix_facebook_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
index 04280ac4..5d38289b 100644
--- a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
@@ -54,16 +54,16 @@ matrix_mautrix_hangouts_sqlite_database_path_in_container: "/data/mautrix-hangou
 
 matrix_mautrix_hangouts_database_username: 'matrix_mautrix_hangouts'
 matrix_mautrix_hangouts_database_password: 'some-password'
-matrix_mautrix_hangouts_postgres_hostname: 'matrix-postgres'
+matrix_mautrix_hangouts_database_hostname: 'matrix-postgres'
 matrix_mautrix_hangouts_database_port: 5432
 matrix_mautrix_hangouts_database_db_name: 'matrix_mautrix_hangouts'
 
-matrix_mautrix_hangouts_postgres_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_database_username }}:{{ matrix_mautrix_hangouts_database_password }}@{{ matrix_mautrix_hangouts_postgres_hostname }}:{{ matrix_mautrix_hangouts_database_port }}/{{ matrix_mautrix_hangouts_database_db_name }}'
+matrix_mautrix_hangouts_database_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_database_username }}:{{ matrix_mautrix_hangouts_database_password }}@{{ matrix_mautrix_hangouts_database_hostname }}:{{ matrix_mautrix_hangouts_database_port }}/{{ matrix_mautrix_hangouts_database_db_name }}'
 
 matrix_mautrix_hangouts_appservice_database: "{{
 	{
 		'sqlite': ('sqlite://' + matrix_mautrix_hangouts_sqlite_database_path_in_container),
-		'postgres': matrix_mautrix_hangouts_postgres_connection_string,
+		'postgres': matrix_mautrix_hangouts_database_connection_string,
 	}[matrix_mautrix_hangouts_database_engine]
 }}"
 
diff --git a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml
index 9e7bfbba..02c34357 100644
--- a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml
@@ -23,7 +23,7 @@
         To continue using SQLite, opt into it explicitly: add `matrix_mautrix_hangouts_database_engine: sqlite` to your vars.yml file and re-run this same command.
         Alternatively, to migrate your existing SQLite database to Postgres:
         1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_hangouts_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_hangouts_postgres_connection_string'`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_hangouts_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_hangouts_database_connection_string'`)
         3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
     when: "matrix_mautrix_hangouts_sqlite_database_path_local_stat_result.stat.exists"
   when: "matrix_mautrix_hangouts_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
index 4f584128..fcdf95db 100644
--- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
@@ -58,16 +58,16 @@ matrix_mautrix_telegram_sqlite_database_path_in_container: "/data/mautrix-telegr
 
 matrix_mautrix_telegram_database_username: 'matrix_mautrix_telegram'
 matrix_mautrix_telegram_database_password: 'some-password'
-matrix_mautrix_telegram_postgres_hostname: 'matrix-postgres'
+matrix_mautrix_telegram_database_hostname: 'matrix-postgres'
 matrix_mautrix_telegram_database_port: 5432
 matrix_mautrix_telegram_database_db_name: 'matrix_mautrix_telegram'
 
-matrix_mautrix_telegram_postgres_connection_string: 'postgresql://{{ matrix_mautrix_telegram_database_username }}:{{ matrix_mautrix_telegram_database_password }}@{{ matrix_mautrix_telegram_postgres_hostname }}:{{ matrix_mautrix_telegram_database_port }}/{{ matrix_mautrix_telegram_database_db_name }}'
+matrix_mautrix_telegram_database_connection_string: 'postgresql://{{ matrix_mautrix_telegram_database_username }}:{{ matrix_mautrix_telegram_database_password }}@{{ matrix_mautrix_telegram_database_hostname }}:{{ matrix_mautrix_telegram_database_port }}/{{ matrix_mautrix_telegram_database_db_name }}'
 
 matrix_mautrix_telegram_appservice_database: "{{
 	{
 		'sqlite': ('sqlite://' + matrix_mautrix_telegram_sqlite_database_path_in_container),
-		'postgres': matrix_mautrix_telegram_postgres_connection_string,
+		'postgres': matrix_mautrix_telegram_database_connection_string,
 	}[matrix_mautrix_telegram_database_engine]
 }}"
 
diff --git a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml
index db1a8f83..d898c590 100644
--- a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml
@@ -23,7 +23,7 @@
         To continue using SQLite, opt into it explicitly: add `matrix_mautrix_telegram_database_engine: sqlite` to your vars.yml file and re-run this same command.
         Alternatively, to migrate your existing SQLite database to Postgres:
         1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_telegram_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_telegram_postgres_connection_string'`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_telegram_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_telegram_database_connection_string'`)
         3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
     when: "matrix_mautrix_telegram_sqlite_database_path_local_stat_result.stat.exists"
   when: "matrix_mautrix_telegram_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
index 357f5892..7e198b36 100644
--- a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
@@ -42,11 +42,11 @@ matrix_mautrix_whatsapp_sqlite_database_path_in_container: "/data/mautrix-whatsa
 
 matrix_mautrix_whatsapp_database_username: 'matrix_mautrix_whatsapp'
 matrix_mautrix_whatsapp_database_password: 'some-password'
-matrix_mautrix_whatsapp_postgres_hostname: 'matrix-postgres'
+matrix_mautrix_whatsapp_database_hostname: 'matrix-postgres'
 matrix_mautrix_whatsapp_database_port: 5432
 matrix_mautrix_whatsapp_database_db_name: 'matrix_mautrix_whatsapp'
 
-matrix_mautrix_whatsapp_postgres_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_database_username }}:{{ matrix_mautrix_whatsapp_database_password }}@{{ matrix_mautrix_whatsapp_postgres_hostname }}:{{ matrix_mautrix_whatsapp_database_port }}/{{ matrix_mautrix_whatsapp_database_db_name }}'
+matrix_mautrix_whatsapp_database_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_database_username }}:{{ matrix_mautrix_whatsapp_database_password }}@{{ matrix_mautrix_whatsapp_database_hostname }}:{{ matrix_mautrix_whatsapp_database_port }}/{{ matrix_mautrix_whatsapp_database_db_name }}'
 
 matrix_mautrix_whatsapp_appservice_database_type: "{{
 	{
@@ -58,7 +58,7 @@ matrix_mautrix_whatsapp_appservice_database_type: "{{
 matrix_mautrix_whatsapp_appservice_database_uri: "{{
 	{
 		'sqlite': matrix_mautrix_whatsapp_sqlite_database_path_in_container,
-		'postgres': matrix_mautrix_whatsapp_postgres_connection_string,
+		'postgres': matrix_mautrix_whatsapp_database_connection_string,
 	}[matrix_mautrix_whatsapp_database_engine]
 }}"
 
diff --git a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml
index 15453e8c..9521e0a5 100644
--- a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml
@@ -23,7 +23,7 @@
         To continue using SQLite, opt into it explicitly: add `matrix_mautrix_whatsapp_database_engine: sqlite` to your vars.yml file and re-run this same command.
         Alternatively, to migrate your existing SQLite database to Postgres:
         1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_whatsapp_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_whatsapp_postgres_connection_string'`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_whatsapp_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_whatsapp_database_connection_string'`)
         3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
     when: "matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result.stat.exists"
   when: "matrix_mautrix_whatsapp_database_engine == 'postgres'"

From aa828ff9f64415a404b3d7bb2a70690fea968d02 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 14:50:04 +0200
Subject: [PATCH 077/179] Separate matrix-reminder-bot install/uninstall tasks

---
 .../tasks/main.yml                            |  9 +-
 .../tasks/setup.yml                           | 88 -------------------
 .../tasks/setup_install.yml                   | 41 +++++++++
 .../tasks/setup_uninstall.yml                 | 35 ++++++++
 4 files changed, 84 insertions(+), 89 deletions(-)
 delete mode 100644 roles/matrix-bot-matrix-reminder-bot/tasks/setup.yml
 create mode 100644 roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml
 create mode 100644 roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml

diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/main.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/main.yml
index ade3d191..fc2afddb 100644
--- a/roles/matrix-bot-matrix-reminder-bot/tasks/main.yml
+++ b/roles/matrix-bot-matrix-reminder-bot/tasks/main.yml
@@ -8,7 +8,14 @@
     - setup-all
     - setup-bot-matrix-reminder-bot
 
-- import_tasks: "{{ role_path }}/tasks/setup.yml"
+- import_tasks: "{{ role_path }}/tasks/setup_install.yml"
+  when: "run_setup|bool and matrix_bot_matrix_reminder_bot_enabled|bool"
+  tags:
+    - setup-all
+    - setup-bot-matrix-reminder-bot
+
+- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml"
+  when: "run_setup|bool and not matrix_bot_matrix_reminder_bot_enabled|bool"
   tags:
     - setup-all
     - setup-bot-matrix-reminder-bot
diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup.yml
deleted file mode 100644
index bc211e31..00000000
--- a/roles/matrix-bot-matrix-reminder-bot/tasks/setup.yml
+++ /dev/null
@@ -1,88 +0,0 @@
----
-
-#
-# Tasks related to setting up matrix-reminder-bot
-#
-
-- name: Ensure matrix-reminder-bot paths exist
-  file:
-    path: "{{ item.path }}"
-    state: directory
-    mode: 0750
-    owner: "{{ matrix_user_username }}"
-    group: "{{ matrix_user_groupname }}"
-  with_items:
-    - { path: "{{ matrix_bot_matrix_reminder_bot_config_path }}", when: true }
-    - { path: "{{ matrix_bot_matrix_reminder_bot_data_path }}", when: true }
-    - { path: "{{ matrix_bot_matrix_reminder_bot_data_store_path }}", when: true }
-  when: matrix_bot_matrix_reminder_bot_enabled|bool and item.when
-
-- name: Ensure matrix-reminder-bot image is pulled
-  docker_image:
-    name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}"
-    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
-    force_source: "{{ matrix_bot_matrix_reminder_bot_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
-    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_bot_matrix_reminder_bot_docker_image_force_pull }}"
-  when: matrix_bot_matrix_reminder_bot_enabled|bool
-
-- name: Ensure matrix-reminder-bot config installed
-  copy:
-    content: "{{ matrix_bot_matrix_reminder_bot_configuration|to_nice_yaml }}"
-    dest: "{{ matrix_bot_matrix_reminder_bot_config_path }}/config.yaml"
-    mode: 0644
-    owner: "{{ matrix_user_username }}"
-    group: "{{ matrix_user_groupname }}"
-  when: matrix_bot_matrix_reminder_bot_enabled|bool
-
-- name: Ensure matrix-matrix-reminder-bot.service installed
-  template:
-    src: "{{ role_path }}/templates/systemd/matrix-bot-matrix-reminder-bot.service.j2"
-    dest: "{{ matrix_systemd_path }}/matrix-bot-matrix-reminder-bot.service"
-    mode: 0644
-  register: matrix_bot_matrix_reminder_bot_systemd_service_result
-  when: matrix_bot_matrix_reminder_bot_enabled|bool
-
-- name: Ensure systemd reloaded after matrix-matrix-reminder-bot.service installation
-  service:
-    daemon_reload: yes
-  when: "matrix_bot_matrix_reminder_bot_enabled|bool and matrix_bot_matrix_reminder_bot_systemd_service_result.changed"
-
-#
-# Tasks related to getting rid of matrix-reminder-bot (if it was previously enabled)
-#
-
-- name: Check existence of matrix-matrix-reminder-bot service
-  stat:
-    path: "{{ matrix_systemd_path }}/matrix-matrix-reminder-bot.service"
-  register: matrix_bot_matrix_reminder_bot_service_stat
-
-- name: Ensure matrix-matrix-reminder-bot is stopped
-  service:
-    name: matrix-matrix-reminder-bot
-    state: stopped
-    daemon_reload: yes
-  register: stopping_result
-  when: "not matrix_bot_matrix_reminder_bot_enabled|bool and matrix_bot_matrix_reminder_bot_service_stat.stat.exists"
-
-- name: Ensure matrix-matrix-reminder-bot.service doesn't exist
-  file:
-    path: "{{ matrix_systemd_path }}/matrix-matrix-reminder-bot.service"
-    state: absent
-  when: "not matrix_bot_matrix_reminder_bot_enabled|bool and matrix_bot_matrix_reminder_bot_service_stat.stat.exists"
-
-- name: Ensure systemd reloaded after matrix-matrix-reminder-bot.service removal
-  service:
-    daemon_reload: yes
-  when: "not matrix_bot_matrix_reminder_bot_enabled|bool and matrix_bot_matrix_reminder_bot_service_stat.stat.exists"
-
-- name: Ensure Matrix matrix-reminder-bot paths don't exist
-  file:
-    path: "{{ matrix_bot_matrix_reminder_bot_base_path }}"
-    state: absent
-  when: "not matrix_bot_matrix_reminder_bot_enabled|bool"
-
-- name: Ensure matrix-reminder-bot Docker image doesn't exist
-  docker_image:
-    name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}"
-    state: absent
-  when: "not matrix_bot_matrix_reminder_bot_enabled|bool"
diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml
new file mode 100644
index 00000000..366c39c3
--- /dev/null
+++ b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml
@@ -0,0 +1,41 @@
+---
+
+- name: Ensure matrix-reminder-bot paths exist
+  file:
+    path: "{{ item.path }}"
+    state: directory
+    mode: 0750
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+  with_items:
+    - { path: "{{ matrix_bot_matrix_reminder_bot_config_path }}", when: true }
+    - { path: "{{ matrix_bot_matrix_reminder_bot_data_path }}", when: true }
+    - { path: "{{ matrix_bot_matrix_reminder_bot_data_store_path }}", when: true }
+  when: "item.when|bool"
+
+- name: Ensure matrix-reminder-bot image is pulled
+  docker_image:
+    name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_bot_matrix_reminder_bot_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_bot_matrix_reminder_bot_docker_image_force_pull }}"
+
+- name: Ensure matrix-reminder-bot config installed
+  copy:
+    content: "{{ matrix_bot_matrix_reminder_bot_configuration|to_nice_yaml }}"
+    dest: "{{ matrix_bot_matrix_reminder_bot_config_path }}/config.yaml"
+    mode: 0644
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+
+- name: Ensure matrix-bot-matrix-reminder-bot.service installed
+  template:
+    src: "{{ role_path }}/templates/systemd/matrix-bot-matrix-reminder-bot.service.j2"
+    dest: "{{ matrix_systemd_path }}/matrix-bot-matrix-reminder-bot.service"
+    mode: 0644
+  register: matrix_bot_matrix_reminder_bot_systemd_service_result
+
+- name: Ensure systemd reloaded after matrix-bot-matrix-reminder-bot.service installation
+  service:
+    daemon_reload: yes
+  when: "matrix_bot_matrix_reminder_bot_systemd_service_result.changed|bool"
diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml
new file mode 100644
index 00000000..744f474d
--- /dev/null
+++ b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml
@@ -0,0 +1,35 @@
+---
+
+- name: Check existence of matrix-matrix-reminder-bot service
+  stat:
+    path: "{{ matrix_systemd_path }}/matrix-bot-matrix-reminder-bot.service"
+  register: matrix_bot_matrix_reminder_bot_service_stat
+
+- name: Ensure matrix-matrix-reminder-bot is stopped
+  service:
+    name: matrix-matrix-reminder-bot
+    state: stopped
+    daemon_reload: yes
+  register: stopping_result
+  when: "matrix_bot_matrix_reminder_bot_service_stat.stat.exists|bool"
+
+- name: Ensure matrix-bot-matrix-reminder-bot.service doesn't exist
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-bot-matrix-reminder-bot.service"
+    state: absent
+  when: "matrix_bot_matrix_reminder_bot_service_stat.stat.exists|bool"
+
+- name: Ensure systemd reloaded after matrix-bot-matrix-reminder-bot.service removal
+  service:
+    daemon_reload: yes
+  when: "matrix_bot_matrix_reminder_bot_service_stat.stat.exists|bool"
+
+- name: Ensure Matrix matrix-reminder-bot paths don't exist
+  file:
+    path: "{{ matrix_bot_matrix_reminder_bot_base_path }}"
+    state: absent
+
+- name: Ensure matrix-reminder-bot Docker image doesn't exist
+  docker_image:
+    name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}"
+    state: absent

From dde1c9f899230813ca70e657c4d7873bb890f6e4 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 14:53:35 +0200
Subject: [PATCH 078/179] Fix indentation causing YAML syntax error

---
 .../tasks/validate_config.yml                                  | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml
index 7000a8a1..c2115cbb 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml
@@ -15,7 +15,7 @@
       path: "{{ matrix_mx_puppet_instagram_data_path }}/database.db"
     register: matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result
 
-   - name: Fail if an SQLite database already exists when using Postgres
+  - name: Fail if an SQLite database already exists when using Postgres
     fail:
       msg: >-
         matrix_mx_puppet_instagram_database_engine has been set to `postgres` (which is our new default now).
@@ -28,4 +28,3 @@
         3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
     when: "matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result.stat.exists"
   when: "matrix_mx_puppet_instagram_database_engine == 'postgres'"
-

From e3a0c9addac73580da23f1e386a67ba14d7461bf Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 15:02:11 +0200
Subject: [PATCH 079/179] Add (Postgres + SQLite) support to
 matrix-reminder-bot

This has been tested and appears to work.
---
 group_vars/matrix_servers                     | 10 +++++++
 .../defaults/main.yml                         | 28 +++++++++++++++++++
 .../tasks/setup_install.yml                   | 20 +++++++++++++
 .../templates/config.yaml.j2                  |  2 +-
 4 files changed, 59 insertions(+), 1 deletion(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index ad1e8ce3..d6d6cc4a 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -580,6 +580,10 @@ matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key
 # We don't enable bots by default.
 matrix_bot_matrix_reminder_bot_enabled: false
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_bot_matrix_reminder_bot_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_bot_matrix_reminder_bot_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'reminder.bot.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bot-matrix-reminder-bot
@@ -949,6 +953,12 @@ matrix_postgres_db_name: "homeserver"
 
 matrix_postgres_additional_databases: |
   {{
+    ([{
+      'name': matrix_bot_matrix_reminder_bot_database_db_name,
+      'username': matrix_bot_matrix_reminder_bot_database_username,
+      'password': matrix_bot_matrix_reminder_bot_database_password,
+    }] if (matrix_bot_matrix_reminder_bot_enabled and matrix_bot_matrix_reminder_bot_database_engine == 'postgres' and matrix_bot_matrix_reminder_bot_database_hostname == 'matrix-postgres') else [])
+    +
     ([{
       'name': matrix_appservice_discord_database_db_name,
       'username': matrix_appservice_discord_database_username,
diff --git a/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml b/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml
index 33028147..70e7b2d2 100644
--- a/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml
+++ b/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml
@@ -21,6 +21,34 @@ matrix_bot_matrix_reminder_bot_systemd_required_services_list: ['docker.service'
 matrix_bot_matrix_reminder_bot_systemd_wanted_services_list: []
 
 
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_bot_matrix_reminder_bot_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_bot_matrix_reminder_bot_database_*` variables
+matrix_bot_matrix_reminder_bot_database_engine: 'sqlite'
+
+matrix_bot_matrix_reminder_bot_sqlite_database_path_local: "{{ matrix_bot_matrix_reminder_bot_data_path }}/bot.db"
+matrix_bot_matrix_reminder_bot_sqlite_database_path_in_container: "/data/bot.db"
+
+matrix_bot_matrix_reminder_bot_database_username: 'matrix_reminder_bot'
+matrix_bot_matrix_reminder_bot_database_password: 'some-password'
+matrix_bot_matrix_reminder_bot_database_hostname: 'matrix-postgres'
+matrix_bot_matrix_reminder_bot_database_port: 5432
+matrix_bot_matrix_reminder_bot_database_db_name: 'matrix_reminder_bot'
+
+matrix_bot_matrix_reminder_bot_database_connection_string: 'postgres://{{ matrix_bot_matrix_reminder_bot_database_username }}:{{ matrix_bot_matrix_reminder_bot_database_password }}@{{ matrix_bot_matrix_reminder_bot_database_hostname }}:{{ matrix_bot_matrix_reminder_bot_database_port }}/{{ matrix_bot_matrix_reminder_bot_database_db_name }}'
+
+matrix_bot_matrix_reminder_bot_storage_database: "{{
+	{
+		'sqlite': ('sqlite://' + matrix_bot_matrix_reminder_bot_sqlite_database_path_in_container),
+		'postgres': matrix_bot_matrix_reminder_bot_database_connection_string,
+	}[matrix_bot_matrix_reminder_bot_database_engine]
+}}"
+
+
 # The bot's username. This user needs to be created manually beforehand.
 # Also see `matrix_bot_matrix_reminder_bot_user_password`.
 matrix_bot_matrix_reminder_bot_matrix_user_id_localpart: "bot.matrix-reminder-bot"
diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml
index 366c39c3..4fbafa3a 100644
--- a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml
+++ b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml
@@ -1,5 +1,25 @@
 ---
 
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }}"
+    register: matrix_bot_matrix_reminder_bot_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_bot_matrix_reminder_bot_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }}.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_bot_matrix_reminder_bot_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_bot_matrix_reminder_bot_database_connection_string'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_bot_matrix_reminder_bot_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_bot_matrix_reminder_bot_database_engine == 'postgres'"
+
 - name: Ensure matrix-reminder-bot paths exist
   file:
     path: "{{ item.path }}"
diff --git a/roles/matrix-bot-matrix-reminder-bot/templates/config.yaml.j2 b/roles/matrix-bot-matrix-reminder-bot/templates/config.yaml.j2
index e5ba5651..59643958 100644
--- a/roles/matrix-bot-matrix-reminder-bot/templates/config.yaml.j2
+++ b/roles/matrix-bot-matrix-reminder-bot/templates/config.yaml.j2
@@ -23,7 +23,7 @@ storage:
   # For Postgres, this would look like:
   #     database: "postgres://username:password@localhost/dbname?sslmode=disable"
   #database: "postgres://matrix-reminder-bot:remindme@localhost/matrix-reminder-bot?sslmode=disable"
-  database: "sqlite:///data/bot.db"
+  database: {{ matrix_bot_matrix_reminder_bot_storage_database|to_json }}
   # The path to a directory for internal bot storage
   # containing encryption keys, sync tokens, etc.
   store_path: "/data/store"

From 54da61f81b56e4c5906d09cd7dbbf0fb35092948 Mon Sep 17 00:00:00 2001
From: transcaffeine <transcaffeine@finallycoffee.eu>
Date: Mon, 14 Dec 2020 14:02:51 +0100
Subject: [PATCH 080/179] add postgres support mx-appservice-[slack|irc] with
 fallback to nedb in role and migration notice

---
 .../defaults/main.yml                         | 14 ++++++++++++
 .../tasks/validate_config.yml                 | 20 +++++++++++++++++
 .../templates/config.yaml.j2                  |  4 ++--
 .../defaults/main.yml                         | 15 +++++++++++++
 .../tasks/validate_config.yml                 | 22 ++++++++++++++++++-
 .../templates/config.yaml.j2                  |  6 +++--
 6 files changed, 76 insertions(+), 5 deletions(-)

diff --git a/roles/matrix-bridge-appservice-irc/defaults/main.yml b/roles/matrix-bridge-appservice-irc/defaults/main.yml
index 7ff2d46a..a1c76253 100644
--- a/roles/matrix-bridge-appservice-irc/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-irc/defaults/main.yml
@@ -16,6 +16,20 @@ matrix_appservice_irc_homeserver_domain: '{{ matrix_domain }}'
 matrix_appservice_irc_homeserver_enablePresence: true
 matrix_appservice_irc_appservice_address: 'http://matrix-appservice-irc:9999'
 
+matrix_appservice_irc_database_engine: nedb
+matrix_appservice_irc_database_username: matrix_appservice_irc
+matrix_appservice_irc_database_password: ~
+matrix_appservice_irc_database_hostname: 'matrix-postgres'
+matrix_appservice_irc_database_port: 5432
+matrix_appservice_irc_database_name: matrix_appservice_irc
+matrix_appservice_irc_databasa
+matrix_appservice_irc_database_connString: >-2
+  {%- if matrix_appservice_irc_database_engine == 'postgres' -%}
+  postgresql://{{ matrix_appservice_irc_database_username }}:{{ matrix_appservice_irc_database_password }}@{{ matrix_appservice_irc_database_hostname }}:{{ matrix_appservice_irc_database_port }}/{{ matrix_appservice_irc_database_name }}?sslmode=disable
+  {%- else if matrix_appservice_irc_database_engine == 'nedb' -%}
+  {{ matrix_appservice_irc_database_engine }}://{{ matrix_appservice_irc_database_file }}
+  {%- endif -%}
+
 matrix_appservice_irc_ircService_servers: []
 
 # Example of `matrix_appservice_irc_ircService_servers` with one server (and all its options):
diff --git a/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml b/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml
index bd08427c..e0e932bc 100644
--- a/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml
+++ b/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml
@@ -33,3 +33,23 @@
   when: "item.old in vars"
   with_items:
     - {'old': 'matrix_appservice_irc_container_expose_client_server_api_port', 'new': '<superseded by matrix_appservice_irc_container_http_host_bind_port>'}
+
+- block:
+  - name: Check if a neDB database already exists
+    stat:
+      path: "{{ matrix_appservice_irc_data_path }}/"
+    register: matrix_appservice_irc_nedb_stat_result
+
+  - name: Fail if an neDB database already exists when using Postgres
+    fail:
+      msg: >-2
+        matrix_appservice_irc_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing neDB database in {{ matrix_appservice_irc_data_path }}/.
+        It appears that you've been using this bridge with the neDB engine until now.
+        To continue using neDB, opt into it explicitly: add `matrix_appservice_irc_database_engine: nedb` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing neDB database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_irc_data_path }} postgres_connection_string_variable_name=matrix_appservice_irc_database_connString'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_appservice_irc_nedb_stat_result.stat.exists"
+  when: "matrix_appservice_irc_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2
index 83b71835..0da28403 100644
--- a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2
+++ b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2
@@ -127,8 +127,8 @@ advanced:
 # Use an external database to store bridge state.
 database:
   # database engine (must be 'postgres' or 'nedb'). Default: nedb
-  engine: "postgres"
+  engine: {{ matrix_appservice_irc_database_engine }}
   # Either a PostgreSQL connection string, or a path to the NeDB storage directory.
   # For postgres, it must start with postgres://
   # For NeDB, it must start with nedb://. The path is relative to the project directory.
-  connectionString: "postgres://matrix_appservice_irc:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_appservice_irc') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_irc"
+  connectionString: {{ matrix_appservice_irc_database_connString | to_json }}
diff --git a/roles/matrix-bridge-appservice-slack/defaults/main.yml b/roles/matrix-bridge-appservice-slack/defaults/main.yml
index f2cfd8f9..e2127c29 100644
--- a/roles/matrix-bridge-appservice-slack/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-slack/defaults/main.yml
@@ -45,6 +45,21 @@ matrix_appservice_slack_appservice_token: ''
 matrix_appservice_slack_homeserver_token: ''
 matrix_appservice_slack_id_token: ''
 
+matrix_appservice_slack_database_engine: nedb
+matrix_appservice_slack_database_username: matrix_appservice_slack
+matrix_appservice_slack_database_password: ~
+matrix_appservice_slack_database_hostname: 'matrix-postgres'
+matrix_appservice_slack_database_port: 5432
+matrix_appservice_slack_database_name: matrix_appservice_slack
+matrix_appservice_slack_database_file: /data
+matrix_appservice_slack_database_connString: >-2
+  {%- if matrix_appservice_slack_database_engine == 'postgres' -%}
+  postgresql://{{ matrix_appservice_slack_database_username }}:{{ matrix_appservice_slack_database_password }}@{{ matrix_appservice_slack_database_hostname }}:{{ matrix_appservice_slack_database_port }}/{{ matrix_appservice_slack_database_name }}?sslmode=disable
+  {%- else if matrix_appservice_slack_database_engine == 'nedb' -%}
+  {{ matrix_appservice_slack_database_engine }}://{{ matrix_appservice_slack_database_file }}
+  {%- endif -%}
+
+
 matrix_appservice_slack_configuration_yaml: "{{ lookup('template', 'templates/config.yaml.j2') }}"
 
 matrix_appservice_slack_configuration_extension_yaml: |
diff --git a/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml b/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml
index 5da5f947..5421b112 100644
--- a/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml
+++ b/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml
@@ -9,4 +9,24 @@
     - "matrix_appservice_slack_control_room_id"
     - "matrix_appservice_slack_appservice_token"
     - "matrix_appservice_slack_homeserver_token"
-    - "matrix_appservice_slack_id_token"
\ No newline at end of file
+    - "matrix_appservice_slack_id_token"
+
+- block:
+  - name: Check if a neDB database already exists
+    stat:
+      path: "{{ matrix_appservice_slack_data_path }}/"
+    register: matrix_appservice_slack_nedb_stat_result
+
+  - name: Fail if an neDB database already exists when using Postgres
+    fail:
+      msg: >-2
+        matrix_appservice_slack_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing neDB database in {{ matrix_appservice_slack_data_path }}/.
+        It appears that you've been using this bridge with the neDB engine until now.
+        To continue using neDB, opt into it explicitly: add `matrix_appservice_slack_database_engine: nedb` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing neDB database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_slack_data_path }} postgres_connection_string_variable_name=matrix_appservice_slack_database_connString'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_appservice_slack_nedb_stat_result.stat.exists"
+  when: "matrix_appservice_slack_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2
index 6c491134..0ae13738 100644
--- a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2
+++ b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2
@@ -9,9 +9,11 @@ homeserver:
     url: "{{ matrix_appservice_slack_homeserver_url }}"
     server_name: "{{ matrix_domain }}"
 
+{% if matrix_appservice_slack_database_engine == 'nedb' %}
 dbdir: "/data"
+{% endif %}
 db:
-    engine: "postgres"
-    connectionString: "postgresql://matrix_appservice_slack:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_appservice_slack') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_slack"
+    engine: "{{ matrix_appservice_slack_database_engine }}"
+    connectionString: {{ matrix_appservice_slack_database_connString | to_json }}
 
 matrix_admin_room: "{{ matrix_appservice_slack_control_room_id }}"

From bc376c2fb2f2da5172dfe6ef130317921a23070f Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 15:55:35 +0200
Subject: [PATCH 081/179] Add database migration utility to matrix-postgres
 role

---
 .../tasks/util/migrate_db_to_postgres.yml     | 111 ++++++++++++++++++
 1 file changed, 111 insertions(+)
 create mode 100644 roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml

diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
new file mode 100644
index 00000000..ec635f99
--- /dev/null
+++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
@@ -0,0 +1,111 @@
+---
+
+- name: Fail if Postgres not enabled
+  fail:
+    msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot migrate."
+  when: "not matrix_postgres_enabled|bool"
+
+- name: Fail if util called incorrectly (missing matrix_postgres_db_migration_request)
+  fail:
+    msg: "The `matrix_postgres_db_migration_request` variable needs to be provided to this util."
+  when: "matrix_postgres_db_migration_request is not defined"
+
+- name: Fail if util called incorrectly (invalid matrix_postgres_db_migration_request)
+  fail:
+    msg: "The `matrix_postgres_db_migration_request` variable needs to contain `{{ item }}`."
+  with_items:
+    - src
+    - dst
+    - caller
+    - engine_variable_name
+    - systemd_services_to_stop
+  when: "item not in matrix_postgres_db_migration_request"
+
+- name: Check if the provided source database file exists
+  stat:
+    path: "{{ matrix_postgres_db_migration_request.src }}"
+  register: matrix_postgres_db_migration_request_src_stat_result
+
+- name: Fail if provided source database file doesn't exist
+  fail:
+    msg: "File cannot be found on the server at {{ matrix_postgres_db_migration_request.src }}"
+  when: "not matrix_postgres_db_migration_request_src_stat_result.stat.exists"
+
+- name: Fail if we cannot migrate on the current architecture ({{ matrix_architecture }})
+  fail:
+    msg: >-
+      {{ matrix_postgres_db_migration_request.engine_variable_name }} has been set to `postgres` (which is our new default now).
+      However, we've discovered an existing file-based database ({{ matrix_postgres_db_migration_request.engine_old_default }}) in {{ matrix_postgres_db_migration_request.src }}.
+      It appears that you've been using this bridge with a file-based database engine until now.
+      To continue using {{ matrix_postgres_db_migration_request.engine_old_default }}, opt into it explicitly: add `{{ matrix_postgres_db_migration_request.engine_variable_name }}: {{ matrix_postgres_db_migration_request.engine_old_default }}` to your vars.yml file and re-run this same command.
+      We'd normally auto-migrate you to Postgres, but we can't do it on the {{ matrix_architecture }} architecture. Our pgloader container image only supports amd64 (for now).
+      Learn more here: https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/740
+  when: "matrix_architecture != 'amd64'"
+
+
+# Defaults
+
+- name: Set postgres_start_wait_time, if not provided
+  set_fact:
+    postgres_start_wait_time: 15
+  when: "postgres_start_wait_time|default('') == ''"
+
+# Actual import work
+
+# matrix-postgres is most likely started already
+- name: Ensure matrix-postgres is started
+  service:
+    name: matrix-postgres
+    state: started
+    daemon_reload: yes
+  register: matrix_postgres_service_start_result
+
+- name: Wait a bit, so that Postgres can start
+  wait_for:
+    timeout: "{{ postgres_start_wait_time }}"
+  delegate_to: 127.0.0.1
+  become: false
+  when: "matrix_postgres_service_start_result.changed|bool"
+
+# We only stop services here, leaving it to the caller to start them later.
+#
+# We can't start them, because they probably need to be reconfigured too (changing the configuration from using SQLite to Postgres, etc.),
+# before starting.
+#
+# Since the caller will be starting them, it might make sense to leave stopping to it as well.
+# However, we don't do it, because it's simpler having it here, and it also gets to happen only if we'll be doing an import.
+# If we bailed out (somewhere above), nothing would have gotten stopped. It's nice to leave this running in such cases.
+- name: Ensure systemd services stopped
+  service:
+    name: "{{ item }}"
+    state: stopped
+  with_items: "{{ matrix_postgres_db_migration_request.systemd_services_to_stop }}"
+
+- name: Import SQLite database from {{ matrix_postgres_db_migration_request.src }} into Postgres
+  command:
+    cmd: >-
+      {{ matrix_host_command_docker }} run
+      --rm
+      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
+      --cap-drop=ALL
+      --network={{ matrix_docker_network }}
+      --mount type=bind,src={{ matrix_postgres_db_migration_request.src }},dst=/in.db,ro
+      --entrypoint=/bin/sh
+      {{ matrix_postgres_pgloader_docker_image }}
+      -c
+      'pgloader /in.db {{ matrix_postgres_db_migration_request.dst }}'
+
+- name: Archive SQLite database ({{ matrix_postgres_db_migration_request.src }} -> {{ matrix_postgres_db_migration_request.src }}.backup)
+  command:
+    cmd: "mv {{ matrix_postgres_db_migration_request.src }} {{ matrix_postgres_db_migration_request.src }}.backup"
+
+- name: Inject result
+  set_fact:
+    matrix_playbook_runtime_results: |
+      {{
+        matrix_playbook_runtime_results|default([])
+        +
+        [
+          "NOTE: Your {{ matrix_postgres_db_migration_request.engine_variable_name }} database file has been imported into Postgres. The original file has been moved from `{{ matrix_postgres_db_migration_request.src }}` to `{{ matrix_postgres_db_migration_request.src }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file."
+        ]
+      }}

From dc7850e83cc0d23f0f5b6085e8e974f6e89f0e0e Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 16:03:03 +0200
Subject: [PATCH 082/179] Fix wording and variable names a bit

---
 .../tasks/util/migrate_db_to_postgres.yml            | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
index ec635f99..60539a91 100644
--- a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
+++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
@@ -35,9 +35,9 @@
   fail:
     msg: >-
       {{ matrix_postgres_db_migration_request.engine_variable_name }} has been set to `postgres` (which is our new default now).
-      However, we've discovered an existing file-based database ({{ matrix_postgres_db_migration_request.engine_old_default }}) in {{ matrix_postgres_db_migration_request.src }}.
+      However, we've discovered an existing file-based database ({{ matrix_postgres_db_migration_request.engine_old }}) in {{ matrix_postgres_db_migration_request.src }}.
       It appears that you've been using this bridge with a file-based database engine until now.
-      To continue using {{ matrix_postgres_db_migration_request.engine_old_default }}, opt into it explicitly: add `{{ matrix_postgres_db_migration_request.engine_variable_name }}: {{ matrix_postgres_db_migration_request.engine_old_default }}` to your vars.yml file and re-run this same command.
+      To continue using {{ matrix_postgres_db_migration_request.engine_old }}, opt into it explicitly: add `{{ matrix_postgres_db_migration_request.engine_variable_name }}: {{ matrix_postgres_db_migration_request.engine_old }}` to your vars.yml file and re-run this same command.
       We'd normally auto-migrate you to Postgres, but we can't do it on the {{ matrix_architecture }} architecture. Our pgloader container image only supports amd64 (for now).
       Learn more here: https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/740
   when: "matrix_architecture != 'amd64'"
@@ -75,13 +75,13 @@
 # Since the caller will be starting them, it might make sense to leave stopping to it as well.
 # However, we don't do it, because it's simpler having it here, and it also gets to happen only if we'll be doing an import.
 # If we bailed out (somewhere above), nothing would have gotten stopped. It's nice to leave this running in such cases.
-- name: Ensure systemd services stopped
+- name: Ensure systemd services blocking the database import are stopped
   service:
     name: "{{ item }}"
     state: stopped
   with_items: "{{ matrix_postgres_db_migration_request.systemd_services_to_stop }}"
 
-- name: Import SQLite database from {{ matrix_postgres_db_migration_request.src }} into Postgres
+- name: Import {{ matrix_postgres_db_migration_request.engine_old }} database from {{ matrix_postgres_db_migration_request.src }} into Postgres
   command:
     cmd: >-
       {{ matrix_host_command_docker }} run
@@ -95,7 +95,7 @@
       -c
       'pgloader /in.db {{ matrix_postgres_db_migration_request.dst }}'
 
-- name: Archive SQLite database ({{ matrix_postgres_db_migration_request.src }} -> {{ matrix_postgres_db_migration_request.src }}.backup)
+- name: Archive {{ matrix_postgres_db_migration_request.engine_old }} database ({{ matrix_postgres_db_migration_request.src }} -> {{ matrix_postgres_db_migration_request.src }}.backup)
   command:
     cmd: "mv {{ matrix_postgres_db_migration_request.src }} {{ matrix_postgres_db_migration_request.src }}.backup"
 
@@ -106,6 +106,6 @@
         matrix_playbook_runtime_results|default([])
         +
         [
-          "NOTE: Your {{ matrix_postgres_db_migration_request.engine_variable_name }} database file has been imported into Postgres. The original file has been moved from `{{ matrix_postgres_db_migration_request.src }}` to `{{ matrix_postgres_db_migration_request.src }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file."
+          "NOTE: Your {{ matrix_postgres_db_migration_request.engine_old }} database file has been imported into Postgres. The original database file has been moved from `{{ matrix_postgres_db_migration_request.src }}` to `{{ matrix_postgres_db_migration_request.src }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file."
         ]
       }}

From ac37091d01c4b38f59c1a57c7b83305bb335ae6d Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 16:03:32 +0200
Subject: [PATCH 083/179] Enable automatic (SQLite -> Postgres) migration for
 matrix-reminder-bot

---
 .../tasks/setup_install.yml                   | 46 ++++++++++++-------
 1 file changed, 29 insertions(+), 17 deletions(-)

diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml
index 4fbafa3a..fea1e00d 100644
--- a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml
+++ b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml
@@ -1,23 +1,29 @@
 ---
 
+- set_fact:
+    matrix_bot_matrix_reminder_bot_requires_restart: false
+
 - block:
-  - name: Check if an SQLite database already exists
-    stat:
-      path: "{{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }}"
-    register: matrix_bot_matrix_reminder_bot_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-
-        matrix_bot_matrix_reminder_bot_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }}.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_bot_matrix_reminder_bot_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_bot_matrix_reminder_bot_database_connection_string'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_bot_matrix_reminder_bot_sqlite_database_path_local_stat_result.stat.exists"
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }}"
+      register: matrix_bot_matrix_reminder_bot_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }}"
+              dst: "{{ matrix_bot_matrix_reminder_bot_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_bot_matrix_reminder_bot_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-bot-matrix-reminder-bot.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_bot_matrix_reminder_bot_requires_restart: true
+      when: "matrix_bot_matrix_reminder_bot_sqlite_database_path_local_stat_result.stat.exists|bool"
   when: "matrix_bot_matrix_reminder_bot_database_engine == 'postgres'"
 
 - name: Ensure matrix-reminder-bot paths exist
@@ -59,3 +65,9 @@
   service:
     daemon_reload: yes
   when: "matrix_bot_matrix_reminder_bot_systemd_service_result.changed|bool"
+
+- name: Ensure matrix-bot-matrix-reminder-bot.service restarted, if necessary
+  service:
+    name: "matrix-bot-matrix-reminder-bot.service"
+    state: restarted
+  when: "matrix_bot_matrix_reminder_bot_requires_restart|bool"

From c1431b28f0384002843b5a059f4fe11caa3d8a57 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 16:13:57 +0200
Subject: [PATCH 084/179] Make use of
 matrix_postgres_db_migration_request.caller

---
 roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
index 60539a91..b77ce4b0 100644
--- a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
+++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
@@ -34,7 +34,7 @@
 - name: Fail if we cannot migrate on the current architecture ({{ matrix_architecture }})
   fail:
     msg: >-
-      {{ matrix_postgres_db_migration_request.engine_variable_name }} has been set to `postgres` (which is our new default now).
+      {{ matrix_postgres_db_migration_request.engine_variable_name }} (part of {{ matrix_postgres_db_migration_request.caller }}) has been set to `postgres` (which is our new default now).
       However, we've discovered an existing file-based database ({{ matrix_postgres_db_migration_request.engine_old }}) in {{ matrix_postgres_db_migration_request.src }}.
       It appears that you've been using this bridge with a file-based database engine until now.
       To continue using {{ matrix_postgres_db_migration_request.engine_old }}, opt into it explicitly: add `{{ matrix_postgres_db_migration_request.engine_variable_name }}: {{ matrix_postgres_db_migration_request.engine_old }}` to your vars.yml file and re-run this same command.

From bbc08722c5ba4c02e39f747e0631696022316321 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 16:14:23 +0200
Subject: [PATCH 085/179] Enable automatic (SQLite -> Postgres) migration for
 matrix-mautrix-facebook

---
 .../tasks/setup_install.yml                   | 46 ++++++++++++-------
 1 file changed, 29 insertions(+), 17 deletions(-)

diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
index 1a0ff70c..7f310446 100644
--- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
@@ -8,24 +8,30 @@
       The matrix-bridge-mautrix-facebook role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+- set_fact:
+    matrix_mautrix_facebook_requires_restart: false
+
 - block:
-  - name: Check if an SQLite database already exists
-    stat:
-      path: "{{ matrix_mautrix_facebook_sqlite_database_path_local }}"
-    register: matrix_mautrix_facebook_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-
-        matrix_mautrix_facebook_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_mautrix_facebook_sqlite_database_path_local }}.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_mautrix_facebook_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_facebook_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_facebook_database_connection_string'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_mautrix_facebook_sqlite_database_path_local_stat_result.stat.exists"
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_mautrix_facebook_sqlite_database_path_local }}"
+      register: matrix_mautrix_facebook_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_mautrix_facebook_sqlite_database_path_local }}"
+              dst: "{{ matrix_mautrix_facebook_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_mautrix_facebook_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-mautrix-facebook.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_mautrix_facebook_requires_restart: true
+      when: "matrix_mautrix_facebook_sqlite_database_path_local_stat_result.stat.exists|bool"
   when: "matrix_mautrix_facebook_database_engine == 'postgres'"
 
 - name: Ensure Mautrix Facebook image is pulled
@@ -114,3 +120,9 @@
   service:
     daemon_reload: yes
   when: "matrix_mautrix_facebook_systemd_service_result.changed"
+
+- name: Ensure matrix-mautrix-facebook.service restarted, if necessary
+  service:
+    name: "matrix-mautrix-facebook.service"
+    state: restarted
+  when: "matrix_mautrix_facebook_requires_restart|bool"

From 3ba852026663407c0db3707ee9a659e850186728 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 16:18:38 +0200
Subject: [PATCH 086/179] Enable automatic (SQLite -> Postgres) migration for
 matrix-mautrix-hangouts

---
 .../tasks/setup_install.yml                   | 46 ++++++++++++-------
 1 file changed, 29 insertions(+), 17 deletions(-)

diff --git a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml
index 02c34357..15af9626 100644
--- a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml
@@ -8,24 +8,30 @@
       The matrix-bridge-mautrix-hangouts role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+- set_fact:
+    matrix_mautrix_hangouts_requires_restart: false
+
 - block:
-  - name: Check if an SQLite database already exists
-    stat:
-      path: "{{ matrix_mautrix_hangouts_sqlite_database_path_local }}"
-    register: matrix_mautrix_hangouts_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-
-        matrix_mautrix_hangouts_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_mautrix_hangouts_sqlite_database_path_local }}.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_mautrix_hangouts_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_hangouts_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_hangouts_database_connection_string'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_mautrix_hangouts_sqlite_database_path_local_stat_result.stat.exists"
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_mautrix_hangouts_sqlite_database_path_local }}"
+      register: matrix_mautrix_hangouts_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_mautrix_hangouts_sqlite_database_path_local }}"
+              dst: "{{ matrix_mautrix_hangouts_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_mautrix_hangouts_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-mautrix-hangouts.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_mautrix_hangouts_requires_restart: true
+      when: "matrix_mautrix_hangouts_sqlite_database_path_local_stat_result.stat.exists|bool"
   when: "matrix_mautrix_hangouts_database_engine == 'postgres'"
 
 - name: Ensure Mautrix Hangouts image is pulled
@@ -113,3 +119,9 @@
   service:
     daemon_reload: yes
   when: "matrix_mautrix_hangouts_systemd_service_result.changed"
+
+- name: Ensure matrix-mautrix-hangouts.service restarted, if necessary
+  service:
+    name: "matrix-mautrix-hangouts.service"
+    state: restarted
+  when: "matrix_mautrix_hangouts_requires_restart|bool"

From 86a8091768bbe8871212761c99de5aa734462039 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 16:19:54 +0200
Subject: [PATCH 087/179] Enable automatic (SQLite -> Postgres) migration for
 matrix-mautrix-telegram

---
 .../tasks/setup_install.yml                   | 46 ++++++++++++-------
 1 file changed, 29 insertions(+), 17 deletions(-)

diff --git a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml
index d898c590..6a37974a 100644
--- a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml
@@ -8,24 +8,30 @@
       The matrix-bridge-mautrix-telegram role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+- set_fact:
+    matrix_mautrix_telegram_requires_restart: false
+
 - block:
-  - name: Check if an SQLite database already exists
-    stat:
-      path: "{{ matrix_mautrix_telegram_sqlite_database_path_local }}"
-    register: matrix_mautrix_telegram_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-
-        matrix_mautrix_telegram_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_mautrix_telegram_sqlite_database_path_local }}.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_mautrix_telegram_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_telegram_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_telegram_database_connection_string'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_mautrix_telegram_sqlite_database_path_local_stat_result.stat.exists"
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_mautrix_telegram_sqlite_database_path_local }}"
+      register: matrix_mautrix_telegram_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_mautrix_telegram_sqlite_database_path_local }}"
+              dst: "{{ matrix_mautrix_telegram_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_mautrix_telegram_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-mautrix-telegram.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_mautrix_telegram_requires_restart: true
+      when: "matrix_mautrix_telegram_sqlite_database_path_local_stat_result.stat.exists|bool"
   when: "matrix_mautrix_telegram_database_engine == 'postgres'"
 
 - name: Ensure Mautrix Telegram image is pulled
@@ -91,3 +97,9 @@
   service:
     daemon_reload: yes
   when: "matrix_mautrix_telegram_systemd_service_result.changed"
+
+- name: Ensure matrix-mautrix-telegram.service restarted, if necessary
+  service:
+    name: "matrix-mautrix-telegram.service"
+    state: restarted
+  when: "matrix_mautrix_telegram_requires_restart|bool"

From 13f84e2ad588668f26010e8f8de07f35c6527e97 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 16:21:01 +0200
Subject: [PATCH 088/179] Enable automatic (SQLite -> Postgres) migration for
 matrix-mautrix-whatsapp

---
 .../tasks/setup_install.yml                   | 46 ++++++++++++-------
 1 file changed, 29 insertions(+), 17 deletions(-)

diff --git a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml
index 9521e0a5..8d894a84 100644
--- a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml
@@ -8,24 +8,30 @@
       The matrix-bridge-mautrix-whatsapp role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+- set_fact:
+    matrix_mautrix_whatsapp_requires_restart: false
+
 - block:
-  - name: Check if an SQLite database already exists
-    stat:
-      path: "{{ matrix_mautrix_whatsapp_sqlite_database_path_local }}"
-    register: matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-
-        matrix_mautrix_whatsapp_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_mautrix_whatsapp_sqlite_database_path_local }}.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_mautrix_whatsapp_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_whatsapp_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_whatsapp_database_connection_string'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result.stat.exists"
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_mautrix_whatsapp_sqlite_database_path_local }}"
+      register: matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_mautrix_whatsapp_sqlite_database_path_local }}"
+              dst: "{{ matrix_mautrix_whatsapp_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_mautrix_whatsapp_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-mautrix-whatsapp.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_mautrix_whatsapp_requires_restart: true
+      when: "matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result.stat.exists|bool"
   when: "matrix_mautrix_whatsapp_database_engine == 'postgres'"
 
 - name: Ensure Mautrix Whatsapp image is pulled
@@ -100,3 +106,9 @@
   service:
     daemon_reload: yes
   when: "matrix_mautrix_whatsapp_systemd_service_result.changed"
+
+- name: Ensure matrix-mautrix-whatsapp.service restarted, if necessary
+  service:
+    name: "matrix-mautrix-whatsapp.service"
+    state: restarted
+  when: "matrix_mautrix_whatsapp_requires_restart|bool"

From cba973d6b555b9288e8791eecb601c4e80aba4f6 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 16:24:56 +0200
Subject: [PATCH 089/179] Enable automatic (SQLite -> Postgres) migration for
 matrix-appservice-discord

---
 .../tasks/setup_install.yml                   | 46 ++++++++++++-------
 1 file changed, 29 insertions(+), 17 deletions(-)

diff --git a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
index a9d8da4c..8bf50e28 100644
--- a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
+++ b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
@@ -1,23 +1,29 @@
 ---
 
+- set_fact:
+    matrix_appservice_discord_requires_restart: false
+
 - block:
-  - name: Check if an SQLite database already exists
-    stat:
-      path: "{{ matrix_appservice_discord_sqlite_database_path_local }}"
-    register: matrix_appservice_discord_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-
-        matrix_appservice_discord_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_appservice_discord_sqlite_database_path_local }}.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_appservice_discord_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_discord_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_appservice_discord_database_connString'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_appservice_discord_sqlite_database_path_local_stat_result.stat.exists"
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_appservice_discord_sqlite_database_path_local }}"
+      register: matrix_appservice_discord_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_appservice_discord_sqlite_database_path_local }}"
+              dst: "{{ matrix_appservice_discord_database_connString }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_appservice_discord_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-appservice-discord.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_appservice_discord_requires_restart: true
+      when: "matrix_appservice_discord_sqlite_database_path_local_stat_result.stat.exists|bool"
   when: "matrix_appservice_discord_database_engine == 'postgres'"
 
 - name: Ensure Appservice Discord image is pulled
@@ -100,3 +106,9 @@
   service:
     daemon_reload: yes
   when: "matrix_appservice_discord_systemd_service_result.changed"
+
+- name: Ensure matrix-appservice-discord.service restarted, if necessary
+  service:
+    name: "matrix-appservice-discord.service"
+    state: restarted
+  when: "matrix_appservice_discord_requires_restart|bool"

From 7248eb3c11d77303dbc395f03c2aeb8ed85e4639 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 16:25:44 +0200
Subject: [PATCH 090/179] Fix syntax error in
 roles/matrix-bridge-appservice-irc/defaults/main.yml

---
 roles/matrix-bridge-appservice-irc/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-bridge-appservice-irc/defaults/main.yml b/roles/matrix-bridge-appservice-irc/defaults/main.yml
index a1c76253..bf23b0ab 100644
--- a/roles/matrix-bridge-appservice-irc/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-irc/defaults/main.yml
@@ -22,7 +22,7 @@ matrix_appservice_irc_database_password: ~
 matrix_appservice_irc_database_hostname: 'matrix-postgres'
 matrix_appservice_irc_database_port: 5432
 matrix_appservice_irc_database_name: matrix_appservice_irc
-matrix_appservice_irc_databasa
+
 matrix_appservice_irc_database_connString: >-2
   {%- if matrix_appservice_irc_database_engine == 'postgres' -%}
   postgresql://{{ matrix_appservice_irc_database_username }}:{{ matrix_appservice_irc_database_password }}@{{ matrix_appservice_irc_database_hostname }}:{{ matrix_appservice_irc_database_port }}/{{ matrix_appservice_irc_database_name }}?sslmode=disable

From 0ca48f3532d211df9e44c67d9661a95420701660 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 16:57:51 +0200
Subject: [PATCH 091/179] Separate matrix-ma1sd install/uninstall tasks

---
 roles/matrix-ma1sd/tasks/main.yml             |  9 ++-
 .../{setup_ma1sd.yml => setup_install.yml}    | 57 ++-----------------
 roles/matrix-ma1sd/tasks/setup_uninstall.yml  | 35 ++++++++++++
 3 files changed, 48 insertions(+), 53 deletions(-)
 rename roles/matrix-ma1sd/tasks/{setup_ma1sd.yml => setup_install.yml} (69%)
 create mode 100644 roles/matrix-ma1sd/tasks/setup_uninstall.yml

diff --git a/roles/matrix-ma1sd/tasks/main.yml b/roles/matrix-ma1sd/tasks/main.yml
index f5ac34d6..0b8a114e 100644
--- a/roles/matrix-ma1sd/tasks/main.yml
+++ b/roles/matrix-ma1sd/tasks/main.yml
@@ -8,7 +8,14 @@
     - setup-all
     - setup-ma1sd
 
-- import_tasks: "{{ role_path }}/tasks/setup_ma1sd.yml"
+- import_tasks: "{{ role_path }}/tasks/setup_install.yml"
+  when: "run_setup|bool and matrix_ma1sd_enabled|bool"
+  tags:
+    - setup-all
+    - setup-ma1sd
+
+- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml"
+  when: "run_setup|bool and not matrix_ma1sd_enabled|bool"
   tags:
     - setup-all
     - setup-ma1sd
diff --git a/roles/matrix-ma1sd/tasks/setup_ma1sd.yml b/roles/matrix-ma1sd/tasks/setup_install.yml
similarity index 69%
rename from roles/matrix-ma1sd/tasks/setup_ma1sd.yml
rename to roles/matrix-ma1sd/tasks/setup_install.yml
index 46acb428..aa11ae7a 100644
--- a/roles/matrix-ma1sd/tasks/setup_ma1sd.yml
+++ b/roles/matrix-ma1sd/tasks/setup_install.yml
@@ -1,9 +1,5 @@
 ---
 
-#
-# Tasks related to setting up ma1sd
-#
-
 - name: Ensure ma1sd paths exist
   file:
     path: "{{ item.path }}"
@@ -15,10 +11,9 @@
     - { path: "{{ matrix_ma1sd_config_path }}", when: true }
     - { path: "{{ matrix_ma1sd_data_path }}", when: true }
     - { path: "{{ matrix_ma1sd_docker_src_files_path }}", when: "{{ matrix_ma1sd_container_image_self_build }}"}
-  when: matrix_ma1sd_enabled|bool and item.when
+  when: "item.when|bool"
 
 - import_tasks: "{{ role_path }}/tasks/migrate_mxisd.yml"
-  when: matrix_ma1sd_enabled|bool
 
 - name: Ensure ma1sd image is pulled
   docker_image:
@@ -26,7 +21,7 @@
     source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
     force_source: "{{ matrix_ma1sd_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
     force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_ma1sd_docker_image_force_pull }}"
-  when: matrix_ma1sd_enabled|bool and not matrix_ma1sd_container_image_self_build
+  when: "not matrix_ma1sd_container_image_self_build|bool"
 
 - block:
   - name: Ensure gradle is installed for self-building (Debian)
@@ -72,7 +67,7 @@
       repository: "{{ matrix_ma1sd_docker_image }}"
       force_tag: yes
       source: local
-  when: "matrix_ma1sd_enabled|bool and matrix_ma1sd_container_image_self_build"
+  when: "matrix_ma1sd_container_image_self_build|bool"
 
 - name: Ensure ma1sd config installed
   copy:
@@ -81,7 +76,6 @@
     mode: 0644
     owner: "{{ matrix_user_username }}"
     group: "{{ matrix_user_groupname }}"
-  when: matrix_ma1sd_enabled|bool
 
 - name: Ensure custom templates are installed if any
   copy:
@@ -95,7 +89,7 @@
     - {value: "{{ matrix_ma1sd_threepid_medium_email_custom_session_validation_template }}", location: 'validate-template.eml'}
     - {value: "{{ matrix_ma1sd_threepid_medium_email_custom_unbind_fraudulent_template }}", location: 'unbind-fraudulent.eml'}
     - {value: "{{ matrix_ma1sd_threepid_medium_email_custom_matrixid_template }}", location: 'mxid-template.eml'}
-  when: "matrix_ma1sd_enabled|bool and matrix_ma1sd_threepid_medium_email_custom_templates_enabled|bool and item.value"
+  when: "matrix_ma1sd_threepid_medium_email_custom_templates_enabled|bool and item.value"
 
 - name: Ensure matrix-ma1sd.service installed
   template:
@@ -103,49 +97,8 @@
     dest: "{{ matrix_systemd_path }}/matrix-ma1sd.service"
     mode: 0644
   register: matrix_ma1sd_systemd_service_result
-  when: matrix_ma1sd_enabled|bool
 
 - name: Ensure systemd reloaded after matrix-ma1sd.service installation
   service:
     daemon_reload: yes
-  when: "matrix_ma1sd_enabled|bool and matrix_ma1sd_systemd_service_result.changed"
-
-#
-# Tasks related to getting rid of ma1sd (if it was previously enabled)
-#
-
-- name: Check existence of matrix-ma1sd service
-  stat:
-    path: "{{ matrix_systemd_path }}/matrix-ma1sd.service"
-  register: matrix_ma1sd_service_stat
-
-- name: Ensure matrix-ma1sd is stopped
-  service:
-    name: matrix-ma1sd
-    state: stopped
-    daemon_reload: yes
-  register: stopping_result
-  when: "not matrix_ma1sd_enabled|bool and matrix_ma1sd_service_stat.stat.exists"
-
-- name: Ensure matrix-ma1sd.service doesn't exist
-  file:
-    path: "{{ matrix_systemd_path }}/matrix-ma1sd.service"
-    state: absent
-  when: "not matrix_ma1sd_enabled|bool and matrix_ma1sd_service_stat.stat.exists"
-
-- name: Ensure systemd reloaded after matrix-ma1sd.service removal
-  service:
-    daemon_reload: yes
-  when: "not matrix_ma1sd_enabled|bool and matrix_ma1sd_service_stat.stat.exists"
-
-- name: Ensure Matrix ma1sd paths don't exist
-  file:
-    path: "{{ matrix_ma1sd_base_path }}"
-    state: absent
-  when: "not matrix_ma1sd_enabled|bool"
-
-- name: Ensure ma1sd Docker image doesn't exist
-  docker_image:
-    name: "{{ matrix_ma1sd_docker_image }}"
-    state: absent
-  when: "not matrix_ma1sd_enabled|bool"
+  when: "matrix_ma1sd_systemd_service_result.changed|bool"
diff --git a/roles/matrix-ma1sd/tasks/setup_uninstall.yml b/roles/matrix-ma1sd/tasks/setup_uninstall.yml
new file mode 100644
index 00000000..b36ab508
--- /dev/null
+++ b/roles/matrix-ma1sd/tasks/setup_uninstall.yml
@@ -0,0 +1,35 @@
+---
+
+- name: Check existence of matrix-ma1sd service
+  stat:
+    path: "{{ matrix_systemd_path }}/matrix-ma1sd.service"
+  register: matrix_ma1sd_service_stat
+
+- name: Ensure matrix-ma1sd is stopped
+  service:
+    name: matrix-ma1sd
+    state: stopped
+    daemon_reload: yes
+  register: stopping_result
+  when: "matrix_ma1sd_service_stat.stat.exists|bool"
+
+- name: Ensure matrix-ma1sd.service doesn't exist
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-ma1sd.service"
+    state: absent
+  when: "matrix_ma1sd_service_stat.stat.exists|bool"
+
+- name: Ensure systemd reloaded after matrix-ma1sd.service removal
+  service:
+    daemon_reload: yes
+  when: "matrix_ma1sd_service_stat.stat.exists|bool"
+
+- name: Ensure Matrix ma1sd paths don't exist
+  file:
+    path: "{{ matrix_ma1sd_base_path }}"
+    state: absent
+
+- name: Ensure ma1sd Docker image doesn't exist
+  docker_image:
+    name: "{{ matrix_ma1sd_docker_image }}"
+    state: absent

From af3ea67bbaa699ad6f03db6633f1dd1902ab9290 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 17:16:25 +0200
Subject: [PATCH 092/179] Add (SQLite + Postgres) support and automatic
 migration to matrix-ma1sd

---
 group_vars/matrix_servers                     | 10 ++++++
 roles/matrix-ma1sd/defaults/main.yml          | 22 ++++++++++++
 roles/matrix-ma1sd/tasks/setup_install.yml    | 36 +++++++++++++++++++
 roles/matrix-ma1sd/templates/ma1sd.yaml.j2    | 16 +++++++--
 .../tasks/util/migrate_db_to_postgres.yml     |  2 +-
 5 files changed, 82 insertions(+), 4 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index d6d6cc4a..735d1324 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -835,6 +835,10 @@ matrix_ma1sd_systemd_wanted_services_list: |
     (['matrix-mailer.service'] if matrix_mailer_enabled else [])
   }}
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_ma1sd_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_ma1sd_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'ma1sd.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-ma1sd
@@ -953,6 +957,12 @@ matrix_postgres_db_name: "homeserver"
 
 matrix_postgres_additional_databases: |
   {{
+    ([{
+      'name': matrix_ma1sd_database_db_name,
+      'username': matrix_ma1sd_database_username,
+      'password': matrix_ma1sd_database_password,
+    }] if (matrix_ma1sd_enabled and matrix_ma1sd_database_engine == 'postgres' and matrix_ma1sd_database_hostname == 'matrix-postgres') else [])
+    +
     ([{
       'name': matrix_bot_matrix_reminder_bot_database_db_name,
       'username': matrix_bot_matrix_reminder_bot_database_username,
diff --git a/roles/matrix-ma1sd/defaults/main.yml b/roles/matrix-ma1sd/defaults/main.yml
index f7a0782d..42e36e92 100644
--- a/roles/matrix-ma1sd/defaults/main.yml
+++ b/roles/matrix-ma1sd/defaults/main.yml
@@ -39,6 +39,28 @@ matrix_ma1sd_systemd_wanted_services_list: []
 # Enabling this is discouraged. Learn more here: https://github.com/ma1uta/ma1sd/blob/master/docs/features/identity.md#lookups
 matrix_ma1sd_matrixorg_forwarding_enabled: false
 
+
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_ma1sd_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_ma1sd_postgres_*` variables
+matrix_ma1sd_database_engine: 'sqlite'
+
+matrix_ma1sd_sqlite_database_path_local: "{{ matrix_ma1sd_data_path }}/ma1sd.db"
+matrix_ma1sd_sqlite_database_path_in_container: "/var/ma1sd/ma1sd.db"
+
+matrix_ma1sd_database_username: 'matrix_ma1sd'
+matrix_ma1sd_database_password: 'some-password'
+matrix_ma1sd_database_hostname: 'matrix-postgres'
+matrix_ma1sd_database_port: 5432
+matrix_ma1sd_database_db_name: 'matrix_ma1sd'
+
+matrix_ma1sd_database_connection_string: 'postgresql://{{ matrix_ma1sd_database_username }}:{{ matrix_ma1sd_database_password }}@{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_db_name }}'
+
+
 # ma1sd has serveral supported identity stores.
 # One of them is storing identities directly in Synapse's database.
 # Learn more here: https://github.com/ma1uta/ma1sd/blob/master/docs/stores/synapse.md
diff --git a/roles/matrix-ma1sd/tasks/setup_install.yml b/roles/matrix-ma1sd/tasks/setup_install.yml
index aa11ae7a..9ae5f077 100644
--- a/roles/matrix-ma1sd/tasks/setup_install.yml
+++ b/roles/matrix-ma1sd/tasks/setup_install.yml
@@ -15,6 +15,36 @@
 
 - import_tasks: "{{ role_path }}/tasks/migrate_mxisd.yml"
 
+
+# These (SQLite -> Postgres) migration tasks are usually at the top,
+# but we'd like to run them after `migrate_mxisd.yml`, which requires the ma1sd paths to exist.
+- set_fact:
+    matrix_ma1sd_requires_restart: false
+
+- block:
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_ma1sd_sqlite_database_path_local }}"
+      register: matrix_ma1sd_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_ma1sd_sqlite_database_path_local }}"
+              dst: "{{ matrix_ma1sd_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_ma1sd_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-ma1sd.service']
+              pgloader_options: ['--with "quote identifiers"']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_ma1sd_requires_restart: true
+      when: "matrix_ma1sd_sqlite_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_ma1sd_database_engine == 'postgres'"
+
 - name: Ensure ma1sd image is pulled
   docker_image:
     name: "{{ matrix_ma1sd_docker_image }}"
@@ -102,3 +132,9 @@
   service:
     daemon_reload: yes
   when: "matrix_ma1sd_systemd_service_result.changed|bool"
+
+- name: Ensure matrix-ma1sd.service restarted, if necessary
+  service:
+    name: "matrix-ma1sd.service"
+    state: restarted
+  when: "matrix_ma1sd_requires_restart|bool"
diff --git a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
index 84585707..43b4022d 100644
--- a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
+++ b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
@@ -11,9 +11,19 @@ key:
   path: /var/ma1sd/sign.key
 
 storage:
-  provider:
-    sqlite:
-      database: /var/ma1sd/ma1sd.db
+  {% if matrix_ma1sd_database_engine == 'sqlite' %}
+    backend: sqlite
+    provider:
+      sqlite:
+        database: {{ matrix_ma1sd_sqlite_database_path_in_container|to_json }}
+  {% elif matrix_ma1sd_database_engine == 'postgres' %}
+    backend: postgresql
+    provider:
+      postgresql:
+        database: //{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_db_name }}
+        username: {{ matrix_ma1sd_database_username|to_json }}
+        password: {{ matrix_ma1sd_database_password|to_json }}
+  {% endif %}
 
 {% if matrix_ma1sd_dns_overwrite_enabled %}
 dns:
diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
index b77ce4b0..5d917538 100644
--- a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
+++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
@@ -93,7 +93,7 @@
       --entrypoint=/bin/sh
       {{ matrix_postgres_pgloader_docker_image }}
       -c
-      'pgloader /in.db {{ matrix_postgres_db_migration_request.dst }}'
+      'pgloader {{ matrix_postgres_db_migration_request.pgloader_options|default([])|join(' ') }} /in.db {{ matrix_postgres_db_migration_request.dst }}'
 
 - name: Archive {{ matrix_postgres_db_migration_request.engine_old }} database ({{ matrix_postgres_db_migration_request.src }} -> {{ matrix_postgres_db_migration_request.src }}.backup)
   command:

From 3ab7dd1abe07574085da2705b3f79ca6773c4f29 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 17:22:31 +0200
Subject: [PATCH 093/179] Potentially add matrix-postgres.service as required
 for bridges, etc.

---
 group_vars/matrix_servers | 41 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 735d1324..1223d8a6 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -61,6 +61,8 @@ matrix_appservice_discord_systemd_required_services_list: |
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
 matrix_appservice_discord_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'discord.as.token') | to_uuid }}"
@@ -207,6 +209,8 @@ matrix_mautrix_facebook_systemd_required_services_list: |
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
 matrix_mautrix_facebook_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'fb.as.token') | to_uuid }}"
@@ -244,6 +248,8 @@ matrix_mautrix_hangouts_systemd_required_services_list: |
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
 matrix_mautrix_hangouts_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'ho.as.token') | to_uuid }}"
@@ -282,6 +288,8 @@ matrix_mautrix_telegram_systemd_required_services_list: |
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
 matrix_mautrix_telegram_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'telegr.as.token') | to_uuid }}"
@@ -318,6 +326,8 @@ matrix_mautrix_whatsapp_systemd_required_services_list: |
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
 matrix_mautrix_whatsapp_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'whats.as.token') | to_uuid }}"
@@ -378,6 +388,8 @@ matrix_mx_puppet_skype_systemd_required_services_list: |
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
 matrix_mx_puppet_skype_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'skype.as.tok') | to_uuid }}"
@@ -413,6 +425,8 @@ matrix_mx_puppet_slack_systemd_required_services_list: |
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
 matrix_mx_puppet_slack_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxslk.as.tok') | to_uuid }}"
@@ -447,6 +461,8 @@ matrix_mx_puppet_twitter_systemd_required_services_list: |
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
 matrix_mx_puppet_twitter_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxtwt.as.tok') | to_uuid }}"
@@ -484,6 +500,8 @@ matrix_mx_puppet_instagram_systemd_required_services_list: |
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
 matrix_mx_puppet_instagram_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxig.as.tok') | to_uuid }}"
@@ -518,6 +536,8 @@ matrix_mx_puppet_discord_systemd_required_services_list: |
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
 matrix_mx_puppet_discord_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxdsc.as.tok') | to_uuid }}"
@@ -552,6 +572,8 @@ matrix_mx_puppet_steam_systemd_required_services_list: |
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
 matrix_mx_puppet_steam_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxste.as.tok') | to_uuid }}"
@@ -580,6 +602,13 @@ matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key
 # We don't enable bots by default.
 matrix_bot_matrix_reminder_bot_enabled: false
 
+matrix_bot_matrix_reminder_bot_systemd_required_services_list: |
+  {{
+    ['docker.service']
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
+  }}
+
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_bot_matrix_reminder_bot_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
 matrix_bot_matrix_reminder_bot_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'reminder.bot.db') | to_uuid }}"
@@ -678,6 +707,13 @@ matrix_dimension_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_ena
 matrix_integration_manager_rest_url: "{{ matrix_dimension_integrations_rest_url if matrix_dimension_enabled else None }}"
 matrix_integration_manager_ui_url: "{{ matrix_dimension_integrations_ui_url if matrix_dimension_enabled else None }}"
 
+matrix_dimension_systemd_required_services_list: |
+  {{
+    ['docker.service']
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
+  }}
+
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_dimension_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
 matrix_dimension_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.dimension.db') | to_uuid }}"
@@ -826,6 +862,11 @@ matrix_ma1sd_threepid_medium_email_connectors_smtp_tls: 0
 
 matrix_ma1sd_self_check_validate_certificates: "{{ false if matrix_ssl_retrieval_method == 'self-signed' else true }}"
 
+matrix_ma1sd_systemd_required_services_list: |
+  {{
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
+  }}
+
 matrix_ma1sd_systemd_wanted_services_list: |
   {{
     (['matrix-corporal.service'] if matrix_corporal_enabled else ['matrix-synapse.service'])

From 13d8a9b39c1a5c5a225ad2c82d1a876210e341c0 Mon Sep 17 00:00:00 2001
From: transcaffeine <transcaffeine@finallycoffee.eu>
Date: Mon, 14 Dec 2020 16:27:06 +0100
Subject: [PATCH 094/179] hint supported automatic migration nedb->postgres

---
 .../tasks/validate_config.yml                 |   2 +-
 roles/matrix-postgres/tasks/import_nedb.yml   | 117 ++++++++++++++++++
 roles/matrix-postgres/tasks/main.yml          |  16 +++
 3 files changed, 134 insertions(+), 1 deletion(-)
 create mode 100644 roles/matrix-postgres/tasks/import_nedb.yml

diff --git a/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml b/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml
index e0e932bc..9b89a340 100644
--- a/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml
+++ b/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml
@@ -49,7 +49,7 @@
         To continue using neDB, opt into it explicitly: add `matrix_appservice_irc_database_engine: nedb` to your vars.yml file and re-run this same command.
         Alternatively, to migrate your existing neDB database to Postgres:
         1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_irc_data_path }} postgres_connection_string_variable_name=matrix_appservice_irc_database_connString'`)
+        2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-irc-nedb --extra-vars='nedb_database_path={{ matrix_appservice_irc_data_path }} postgres_connection_string_variable_name=matrix_appservice_irc_database_connString'`)
         3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
     when: "matrix_appservice_irc_nedb_stat_result.stat.exists"
   when: "matrix_appservice_irc_database_engine == 'postgres'"
diff --git a/roles/matrix-postgres/tasks/import_nedb.yml b/roles/matrix-postgres/tasks/import_nedb.yml
new file mode 100644
index 00000000..cc1f9d78
--- /dev/null
+++ b/roles/matrix-postgres/tasks/import_nedb.yml
@@ -0,0 +1,117 @@
+---
+
+# Pre-checks
+
+- name: Fail if Postgres not enabled
+  fail:
+    msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot import."
+  when: "not matrix_postgres_enabled|bool"
+
+- name: Fail if playbook called incorrectly
+  fail:
+    msg: "The `nedb_database_path` variable needs to be provided to this playbook, via --extra-vars"
+  when: "nedb_database_path is not defined or nedb_database_path.startswith('<')"
+
+- name: Check if the provided nedb database file exists
+  stat:
+    path: "{{ nedb_database_path }}"
+  register: nedb_database_path_stat_result
+
+- name: Fail if provided SQLite database file doesn't exist
+  fail:
+    msg: "File cannot be found on the server at {{ nedb_database_path }}"
+  when: "not nedb_database_path_stat_result.stat.exists"
+
+# We either expect `postgres_db_connection_string` specifying a full Postgres database connection string,
+# or `postgres_connection_string_variable_name`, specifying a name of a variable, which contains a valid connection string.
+
+- block:
+    - name: Fail if postgres_connection_string_variable_name points to an undefined variable
+      fail: msg="postgres_connection_string_variable_name is defined, but there is no variable with the name `{{ postgres_connection_string_variable_name }}`"
+      when: "postgres_connection_string_variable_name not in vars"
+
+    - name: Get Postgres connection string from variable
+      set_fact:
+        postgres_db_connection_string: "{{ lookup('vars', postgres_connection_string_variable_name) }}"
+  when: 'postgres_connection_string_variable_name is defined'
+
+- name: Fail if playbook called incorrectly
+  fail:
+    msg: >-
+      Either a `postgres_db_connection_string` variable or a `postgres_connection_string_variable_name` needs to be provided to this playbook, via `--extra-vars`.
+      Example: `--extra-vars="postgres_db_connection_string=postgresql://username:password@localhost:<port>/database_name"` or `--extra-vars="postgres_connection_string_variable_name=matrix_appservice_discord_database_connString"`
+  when: "postgres_db_connection_string is not defined or not postgres_db_connection_string.startswith('postgresql://')"
+
+
+# Defaults
+
+- name: Set postgres_start_wait_time, if not provided
+  set_fact:
+    postgres_start_wait_time: 15
+  when: "postgres_start_wait_time|default('') == ''"
+
+
+# Actual import work
+
+- name: Ensure matrix-postgres is started
+  service:
+    name: matrix-postgres
+    state: started
+    daemon_reload: yes
+  register: matrix_postgres_service_start_result
+
+- name: Wait a bit, so that Postgres can start
+  wait_for:
+    timeout: "{{ postgres_start_wait_time }}"
+  delegate_to: 127.0.0.1
+  become: false
+  when: "matrix_postgres_service_start_result.changed|bool"
+
+# See https://github.com/matrix-org/matrix-appservice-irc/wiki/Migrating-from-NEdB-to-PostgreSQL
+- name: Import appservice_irc NeDB database from {{ sqlite_database_path }} into Postgres
+  when: database == 'appservice_irc'
+  command:
+    cmd: >-
+      {{ matrix_host_command_docker }} run
+      --rm
+      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
+      --cap-drop=ALL
+      --network={{ matrix_docker_network }}
+      --mount type=bind,src={{ matrix_appservice_irc_data_path }}:/data:ro
+      --entrypoint=/bin/sh
+      {{ matrix_appservice_irc_docker_image }}
+      -c
+      './scripts/migrate-db-to-pgres.sh -d /data -p passkey.pem -c {{ postgres_db_connection_string }}'
+
+# No migration.sh available, but found this:
+# https://github.com/matrix-org/matrix-appservice-slack/blob/develop/src/scripts/migrateToPostgres.ts
+# Usage should be similar to appservice_irc
+- name: Import appservice_slack NeDB database from {{ sqlite_database_path }} into Postgres
+  when: database == 'appservice_slack'
+  command:
+    cmd: >-
+      {{ matrix_host_command_docker }} run
+      --rm
+      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
+      --cap-drop=ALL
+      --network={{ matrix_docker_network }}
+      --mount type=bind,src={{ matrix_appservice_irc_data_path }}:/data:ro
+      --entrypoint=/bin/sh
+      {{ matrix_appservice_slack_docker_image }}
+      -c
+      'node /lib/scripts/migrate-db-to-pgres.js -d /data -p passkey.pem -c {{ postgres_db_connection_string }}'
+
+- name: Archive NeDB database ({{ sqlite_database_path }} -> {{ sqlite_database_path }}.backup)
+  command:
+    cmd: "mv {{ sqlite_database_path }} {{ sqlite_database_path }}.backup"
+
+- name: Inject result
+  set_fact:
+    matrix_playbook_runtime_results: |
+      {{
+        matrix_playbook_runtime_results|default([])
+        +
+        [
+          "NOTE: Your NeDB database file has been imported into Postgres. The original directory has been moved from `{{ nedb_database_path }}` to `{{ nedb_database_path }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file."
+        ]
+      }}
diff --git a/roles/matrix-postgres/tasks/main.yml b/roles/matrix-postgres/tasks/main.yml
index b9c2ae7c..51801314 100644
--- a/roles/matrix-postgres/tasks/main.yml
+++ b/roles/matrix-postgres/tasks/main.yml
@@ -32,6 +32,22 @@
   tags:
     - import-generic-sqlite-db
 
+# Imports appservice-irc NeDB into postgres
+- import_tasks: "{{ role_path }}/tasks/import_nedb.yml"
+  vars:
+    database: appservice_irc
+  when: run_postgres_import_nedb|bool
+  tags:
+    - import-irc-nedb
+
+# Imports slacks neDB to postgres.
+- import_tasks: "{{ role_path }}/tasks/import_nedb.yml"
+  vars:
+    database: appservice_slack
+  when: run_postgres_import_nedb|bool
+  tags:
+    - import-slack-nedb
+
 - import_tasks: "{{ role_path }}/tasks/upgrade_postgres.yml"
   when: run_postgres_upgrade|bool
   tags:

From 516ccb2b2b3eee4d110a6056302d80dcf902f03a Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 18:12:03 +0200
Subject: [PATCH 095/179] Separate matrix-registration install/uninstall tasks

---
 roles/matrix-registration/tasks/main.yml      |  9 +++-
 .../tasks/{setup.yml => setup_install.yml}    | 50 ++-----------------
 .../tasks/setup_uninstall.yml                 | 30 +++++++++++
 3 files changed, 43 insertions(+), 46 deletions(-)
 rename roles/matrix-registration/tasks/{setup.yml => setup_install.yml} (57%)
 create mode 100644 roles/matrix-registration/tasks/setup_uninstall.yml

diff --git a/roles/matrix-registration/tasks/main.yml b/roles/matrix-registration/tasks/main.yml
index 4a884ccd..4fef6abe 100644
--- a/roles/matrix-registration/tasks/main.yml
+++ b/roles/matrix-registration/tasks/main.yml
@@ -8,7 +8,14 @@
     - setup-all
     - setup-matrix-registration
 
-- import_tasks: "{{ role_path }}/tasks/setup.yml"
+- import_tasks: "{{ role_path }}/tasks/setup_install.yml"
+  when: "run_setup|bool and matrix_registration_enabled|bool"
+  tags:
+    - setup-all
+    - setup-matrix-registration
+
+- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml"
+  when: "run_setup|bool and not matrix_registration_enabled|bool"
   tags:
     - setup-all
     - setup-matrix-registration
diff --git a/roles/matrix-registration/tasks/setup.yml b/roles/matrix-registration/tasks/setup_install.yml
similarity index 57%
rename from roles/matrix-registration/tasks/setup.yml
rename to roles/matrix-registration/tasks/setup_install.yml
index dfe1ba8b..77c35581 100644
--- a/roles/matrix-registration/tasks/setup.yml
+++ b/roles/matrix-registration/tasks/setup_install.yml
@@ -1,9 +1,5 @@
 ---
 
-#
-# Tasks related to setting up matrix-registration
-#
-
 - name: Ensure matrix-registration paths exist
   file:
     path: "{{ item.path }}"
@@ -16,7 +12,7 @@
     - { path: "{{ matrix_registration_config_path }}", when: true }
     - { path: "{{ matrix_registration_data_path }}", when: true }
     - { path: "{{ matrix_registration_docker_src_files_path }}", when: "{{ matrix_registration_container_image_self_build }}"}
-  when: matrix_registration_enabled|bool and item.when
+  when: "item.when|bool"
 
 - name: Ensure matrix-registration image is pulled
   docker_image:
@@ -24,7 +20,7 @@
     source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
     force_source: "{{ matrix_registration_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
     force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_registration_docker_image_force_pull }}"
-  when: "matrix_registration_enabled|bool and not matrix_registration_container_image_self_build|bool"
+  when: "not matrix_registration_container_image_self_build|bool"
 
 - name: Ensure matrix-registration repository is present when self-building
   git:
@@ -33,7 +29,7 @@
     version: "{{ matrix_registration_container_image_self_build_branch }}"
     force: "yes"
   register: matrix_registration_git_pull_results
-  when: "matrix_registration_enabled|bool and matrix_registration_container_image_self_build|bool"
+  when: "matrix_registration_container_image_self_build|bool"
 
 - name: Ensure matrix-registration Docker image is built
   docker_image:
@@ -44,7 +40,7 @@
       dockerfile: Dockerfile
       path: "{{ matrix_registration_docker_src_files_path }}"
       pull: yes
-  when: "matrix_registration_enabled|bool and matrix_registration_container_image_self_build|bool"
+  when: "matrix_registration_container_image_self_build|bool"
 
 - name: Ensure matrix-registration config installed
   copy:
@@ -53,7 +49,6 @@
     mode: 0644
     owner: "{{ matrix_user_username }}"
     group: "{{ matrix_user_groupname }}"
-  when: matrix_registration_enabled|bool
 
 - name: Ensure matrix-registration.service installed
   template:
@@ -61,43 +56,8 @@
     dest: "{{ matrix_systemd_path }}/matrix-registration.service"
     mode: 0644
   register: matrix_registration_systemd_service_result
-  when: matrix_registration_enabled|bool
 
 - name: Ensure systemd reloaded after matrix-registration.service installation
   service:
     daemon_reload: yes
-  when: "matrix_registration_enabled|bool and matrix_registration_systemd_service_result.changed"
-
-#
-# Tasks related to getting rid of matrix-registration (if it was previously enabled)
-#
-
-- name: Check existence of matrix-registration service
-  stat:
-    path: "{{ matrix_systemd_path }}/matrix-registration.service"
-  register: matrix_registration_service_stat
-
-- name: Ensure matrix-registration is stopped
-  service:
-    name: matrix-registration
-    state: stopped
-    daemon_reload: yes
-  register: stopping_result
-  when: "not matrix_registration_enabled|bool and matrix_registration_service_stat.stat.exists"
-
-- name: Ensure matrix-registration.service doesn't exist
-  file:
-    path: "{{ matrix_systemd_path }}/matrix-registration.service"
-    state: absent
-  when: "not matrix_registration_enabled|bool and matrix_registration_service_stat.stat.exists"
-
-- name: Ensure systemd reloaded after matrix-registration.service removal
-  service:
-    daemon_reload: yes
-  when: "not matrix_registration_enabled|bool and matrix_registration_service_stat.stat.exists"
-
-- name: Ensure matrix-registration Docker image doesn't exist
-  docker_image:
-    name: "{{ matrix_registration_docker_image }}"
-    state: absent
-  when: "not matrix_registration_enabled|bool"
+  when: "matrix_registration_systemd_service_result.changed|bool"
diff --git a/roles/matrix-registration/tasks/setup_uninstall.yml b/roles/matrix-registration/tasks/setup_uninstall.yml
new file mode 100644
index 00000000..573f8170
--- /dev/null
+++ b/roles/matrix-registration/tasks/setup_uninstall.yml
@@ -0,0 +1,30 @@
+---
+
+- name: Check existence of matrix-registration service
+  stat:
+    path: "{{ matrix_systemd_path }}/matrix-registration.service"
+  register: matrix_registration_service_stat
+
+- name: Ensure matrix-registration is stopped
+  service:
+    name: matrix-registration
+    state: stopped
+    daemon_reload: yes
+  register: stopping_result
+  when: "matrix_registration_service_stat.stat.exists|bool"
+
+- name: Ensure matrix-registration.service doesn't exist
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-registration.service"
+    state: absent
+  when: "matrix_registration_service_stat.stat.exists|bool"
+
+- name: Ensure systemd reloaded after matrix-registration.service removal
+  service:
+    daemon_reload: yes
+  when: "matrix_registration_service_stat.stat.exists|bool"
+
+- name: Ensure matrix-registration Docker image doesn't exist
+  docker_image:
+    name: "{{ matrix_registration_docker_image }}"
+    state: absent

From 8d7459387895a6a3c14ef4c1a751353c740c5a9d Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 18:58:37 +0200
Subject: [PATCH 096/179] Prepare matrix-registation for (SQLite + Postgres)
 support

Auto-migration and everything seems to work. It's just that
matrix-registration cannot load the Python modules required
for talking to a Postgres database.

Tracked here: https://github.com/ZerataX/matrix-registration/issues/44

Until this gets fixed, we'll continue default to 'sqlite'.
---
 group_vars/matrix_servers                     | 18 +++++++++++
 roles/matrix-registration/defaults/main.yml   | 30 +++++++++++++++++
 .../tasks/setup_install.yml                   | 32 +++++++++++++++++++
 .../tasks/validate_config.yml                 |  7 ++++
 .../templates/config.yaml.j2                  |  2 +-
 5 files changed, 88 insertions(+), 1 deletion(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 1223d8a6..ad62a86e 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -1010,6 +1010,12 @@ matrix_postgres_additional_databases: |
       'password': matrix_bot_matrix_reminder_bot_database_password,
     }] if (matrix_bot_matrix_reminder_bot_enabled and matrix_bot_matrix_reminder_bot_database_engine == 'postgres' and matrix_bot_matrix_reminder_bot_database_hostname == 'matrix-postgres') else [])
     +
+    ([{
+      'name': matrix_registration_database_db_name,
+      'username': matrix_registration_database_username,
+      'password': matrix_registration_database_password,
+    }] if (matrix_registration_enabled and matrix_registration_database_engine == 'postgres' and matrix_registration_database_hostname == 'matrix-postgres') else [])
+    +
     ([{
       'name': matrix_appservice_discord_database_db_name,
       'username': matrix_appservice_discord_database_username,
@@ -1291,6 +1297,18 @@ matrix_registration_api_validate_certs: "{{ false if matrix_ssl_retrieval_method
 
 matrix_registration_container_image_self_build: "{{ matrix_architecture != 'amd64' }}"
 
+matrix_registration_systemd_required_services_list: |
+  {{
+    ['docker.service']
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
+  }}
+
+# We'd like to use 'postgres' if matrix_postgres_enabled, but the container image doesn't seem to support that.
+# Learn more here: https://github.com/ZerataX/matrix-registration/issues/44
+matrix_registration_database_engine: 'sqlite'
+matrix_registration_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx.registr.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-registration
diff --git a/roles/matrix-registration/defaults/main.yml b/roles/matrix-registration/defaults/main.yml
index 95147665..5afe4dee 100644
--- a/roles/matrix-registration/defaults/main.yml
+++ b/roles/matrix-registration/defaults/main.yml
@@ -32,6 +32,36 @@ matrix_registration_systemd_wanted_services_list: []
 # Takes an "<ip>:<port>" or "<port>" value (e.g. "127.0.0.1:8767"), or empty string to not expose.
 matrix_registration_container_http_host_bind_port: ''
 
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_registration_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_registration_postgres_*` variables
+matrix_registration_database_engine: 'sqlite'
+
+matrix_registration_sqlite_database_path_local: "{{ matrix_registration_data_path }}/db.sqlite3"
+matrix_registration_sqlite_database_path_in_container: "/data/db.sqlite3"
+
+matrix_registration_database_username: 'matrix_registration'
+matrix_registration_database_password: 'some-password'
+matrix_registration_database_hostname: 'matrix-postgres'
+matrix_registration_database_port: 5432
+matrix_registration_database_db_name: 'matrix_registration'
+
+matrix_registration_database_connection_string: 'postgresql://{{ matrix_registration_database_username }}:{{ matrix_registration_database_password }}@{{ matrix_registration_database_hostname }}:{{ matrix_registration_database_port }}/{{ matrix_registration_database_db_name }}'
+
+# For some reason, matrix-registraiton expects the `db` field to be like this: `sqlite:////data/db.sqlite3`.
+# (seems like one too many slashes, but..)
+matrix_registration_db: "{{
+	{
+		'sqlite': ('sqlite:///' + matrix_registration_sqlite_database_path_in_container),
+		'postgres': matrix_registration_database_connection_string,
+	}[matrix_registration_database_engine]
+}}"
+
+
 # The path at which Matrix Registration will be exposed on `matrix.DOMAIN`
 # (only applies when matrix-nginx-proxy is used).
 matrix_registration_public_endpoint: /matrix-registration
diff --git a/roles/matrix-registration/tasks/setup_install.yml b/roles/matrix-registration/tasks/setup_install.yml
index 77c35581..489bab8b 100644
--- a/roles/matrix-registration/tasks/setup_install.yml
+++ b/roles/matrix-registration/tasks/setup_install.yml
@@ -1,5 +1,31 @@
 ---
 
+- set_fact:
+    matrix_registration_requires_restart: false
+
+- block:
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_registration_sqlite_database_path_local }}"
+      register: matrix_registration_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_registration_sqlite_database_path_local }}"
+              dst: "{{ matrix_registration_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_registration_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-registration.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_registration_requires_restart: true
+      when: "matrix_registration_sqlite_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_registration_database_engine == 'postgres'"
+
 - name: Ensure matrix-registration paths exist
   file:
     path: "{{ item.path }}"
@@ -61,3 +87,9 @@
   service:
     daemon_reload: yes
   when: "matrix_registration_systemd_service_result.changed|bool"
+
+- name: Ensure matrix-registration.service restarted, if necessary
+  service:
+    name: "matrix-registration.service"
+    state: restarted
+  when: "matrix_registration_requires_restart|bool"
diff --git a/roles/matrix-registration/tasks/validate_config.yml b/roles/matrix-registration/tasks/validate_config.yml
index 90466b46..80293bcb 100644
--- a/roles/matrix-registration/tasks/validate_config.yml
+++ b/roles/matrix-registration/tasks/validate_config.yml
@@ -18,3 +18,10 @@
   when: "item.old in vars"
   with_items:
     - {'old': 'matrix_registration_docker_repo', 'new': 'matrix_registration_container_image_self_build_repo'}
+
+- name: Fail if Postgres usage attempted
+  fail:
+    msg: >
+      matrix-registration doesn't support using Postgres just yet.
+      Learn more here: https://github.com/ZerataX/matrix-registration/issues/44
+  when: "matrix_registration_database_engine == 'postgres'"
diff --git a/roles/matrix-registration/templates/config.yaml.j2 b/roles/matrix-registration/templates/config.yaml.j2
index f3b1c57b..ae0ba5ff 100644
--- a/roles/matrix-registration/templates/config.yaml.j2
+++ b/roles/matrix-registration/templates/config.yaml.j2
@@ -3,7 +3,7 @@ server_name: {{ matrix_registration_server_name|to_json }}
 shared_secret: {{ matrix_registration_shared_secret|to_json }}
 admin_secret: {{ matrix_registration_admin_secret|to_json }}
 riot_instance: {{ matrix_registration_riot_instance|to_json }}
-db: 'sqlite:////data/db.sqlite3'
+db: {{ matrix_registration_db|to_json }}
 host: '0.0.0.0'
 port: 5000
 rate_limit: ["100 per day", "10 per minute"]

From 374f43735aa362bbbb2a9068ef2fd8e81b5ab561 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 20:05:31 +0200
Subject: [PATCH 097/179] Separate matrix-dimension install/uninstall tasks

---
 roles/matrix-dimension/tasks/main.yml         | 10 ++-
 .../tasks/setup_dimension.yml                 | 85 -------------------
 .../matrix-dimension/tasks/setup_install.yml  | 36 ++++++++
 .../tasks/setup_uninstall.yml                 | 35 ++++++++
 4 files changed, 79 insertions(+), 87 deletions(-)
 delete mode 100644 roles/matrix-dimension/tasks/setup_dimension.yml
 create mode 100644 roles/matrix-dimension/tasks/setup_install.yml
 create mode 100644 roles/matrix-dimension/tasks/setup_uninstall.yml

diff --git a/roles/matrix-dimension/tasks/main.yml b/roles/matrix-dimension/tasks/main.yml
index 1888f945..aad55286 100644
--- a/roles/matrix-dimension/tasks/main.yml
+++ b/roles/matrix-dimension/tasks/main.yml
@@ -8,8 +8,14 @@
     - setup-all
     - setup-dimension
 
-- import_tasks: "{{ role_path }}/tasks/setup_dimension.yml"
-  when: run_setup|bool
+- import_tasks: "{{ role_path }}/tasks/setup_install.yml"
+  when: run_setup|bool and matrix_dimension_enabled|bool
+  tags:
+    - setup-all
+    - setup-dimension
+
+- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml"
+  when: run_setup|bool and not matrix_dimension_enabled|bool
   tags:
     - setup-all
     - setup-dimension
diff --git a/roles/matrix-dimension/tasks/setup_dimension.yml b/roles/matrix-dimension/tasks/setup_dimension.yml
deleted file mode 100644
index 2437a547..00000000
--- a/roles/matrix-dimension/tasks/setup_dimension.yml
+++ /dev/null
@@ -1,85 +0,0 @@
----
-
-#
-# Tasks related to setting up the dimension
-#
-
-- name: Ensure Dimension base path exists
-  file:
-    path: "{{ matrix_dimension_base_path }}"
-    state: directory
-    mode: 0770
-    owner: "{{ matrix_user_username }}"
-    group: "{{ matrix_dimension_user_gid }}"
-  when: matrix_dimension_enabled|bool
-
-- name: Ensure Dimension config installed
-  copy:
-    content: "{{ matrix_dimension_configuration|to_nice_yaml }}"
-    dest: "{{ matrix_dimension_base_path }}/config.yaml"
-    mode: 0640
-    owner: "{{ matrix_user_username }}"
-    group: "{{ matrix_dimension_user_gid }}"
-  when: matrix_dimension_enabled|bool
-
-- name: Ensure Dimension image is pulled
-  docker_image:
-    name: "{{ matrix_dimension_docker_image }}"
-    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
-    force_source: "{{ matrix_dimension_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
-    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_dimension_docker_image_force_pull }}"
-  when: matrix_dimension_enabled|bool
-
-- name: Ensure matrix-dimension.service installed
-  template:
-    src: "{{ role_path }}/templates/systemd/matrix-dimension.service.j2"
-    dest: "{{ matrix_systemd_path }}/matrix-dimension.service"
-    mode: 0644
-  register: matrix_dimension_systemd_service_result
-  when: matrix_dimension_enabled|bool
-
-- name: Ensure systemd reloaded after matrix-dimension.service installation
-  service:
-    daemon_reload: yes
-  when: "matrix_dimension_enabled|bool and matrix_dimension_systemd_service_result.changed"
-
-#
-# Tasks related to getting rid of the dimension (if it was previously enabled)
-#
-
-- name: Check existence of matrix-dimension service
-  stat:
-    path: "{{ matrix_systemd_path }}/matrix-dimension.service"
-  register: matrix_dimension_service_stat
-  when: "not matrix_dimension_enabled|bool"
-
-- name: Ensure matrix-dimension is stopped
-  service:
-    name: matrix-dimension
-    state: stopped
-    daemon_reload: yes
-  register: stopping_result
-  when: "not matrix_dimension_enabled|bool and matrix_dimension_service_stat.stat.exists"
-
-- name: Ensure matrix-dimension.service doesn't exist
-  file:
-    path: "{{ matrix_systemd_path }}/matrix-dimension.service"
-    state: absent
-  when: "not matrix_dimension_enabled|bool and matrix_dimension_service_stat.stat.exists"
-
-- name: Ensure systemd reloaded after matrix-dimension.service removal
-  service:
-    daemon_reload: yes
-  when: "not matrix_dimension_enabled|bool and matrix_dimension_service_stat.stat.exists"
-
-- name: Ensure Dimension environment variables path doesn't exist
-  file:
-    path: "{{ matrix_dimension_base_path }}"
-    state: absent
-  when: "not matrix_dimension_enabled|bool"
-
-- name: Ensure Dimension Docker image doesn't exist
-  docker_image:
-    name: "{{ matrix_dimension_docker_image }}"
-    state: absent
-  when: "not matrix_dimension_enabled|bool"
diff --git a/roles/matrix-dimension/tasks/setup_install.yml b/roles/matrix-dimension/tasks/setup_install.yml
new file mode 100644
index 00000000..e118dd62
--- /dev/null
+++ b/roles/matrix-dimension/tasks/setup_install.yml
@@ -0,0 +1,36 @@
+---
+
+- name: Ensure Dimension base path exists
+  file:
+    path: "{{ matrix_dimension_base_path }}"
+    state: directory
+    mode: 0770
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_dimension_user_gid }}"
+
+- name: Ensure Dimension config installed
+  copy:
+    content: "{{ matrix_dimension_configuration|to_nice_yaml }}"
+    dest: "{{ matrix_dimension_base_path }}/config.yaml"
+    mode: 0640
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_dimension_user_gid }}"
+
+- name: Ensure Dimension image is pulled
+  docker_image:
+    name: "{{ matrix_dimension_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_dimension_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_dimension_docker_image_force_pull }}"
+
+- name: Ensure matrix-dimension.service installed
+  template:
+    src: "{{ role_path }}/templates/systemd/matrix-dimension.service.j2"
+    dest: "{{ matrix_systemd_path }}/matrix-dimension.service"
+    mode: 0644
+  register: matrix_dimension_systemd_service_result
+
+- name: Ensure systemd reloaded after matrix-dimension.service installation
+  service:
+    daemon_reload: yes
+  when: "matrix_dimension_systemd_service_result.changed|bool"
diff --git a/roles/matrix-dimension/tasks/setup_uninstall.yml b/roles/matrix-dimension/tasks/setup_uninstall.yml
new file mode 100644
index 00000000..9bc4ac8b
--- /dev/null
+++ b/roles/matrix-dimension/tasks/setup_uninstall.yml
@@ -0,0 +1,35 @@
+---
+
+- name: Check existence of matrix-dimension service
+  stat:
+    path: "{{ matrix_systemd_path }}/matrix-dimension.service"
+  register: matrix_dimension_service_stat
+
+- name: Ensure matrix-dimension is stopped
+  service:
+    name: matrix-dimension
+    state: stopped
+    daemon_reload: yes
+  register: stopping_result
+  when: "matrix_dimension_service_stat.stat.exists|bool"
+
+- name: Ensure matrix-dimension.service doesn't exist
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-dimension.service"
+    state: absent
+  when: "matrix_dimension_service_stat.stat.exists|bool"
+
+- name: Ensure systemd reloaded after matrix-dimension.service removal
+  service:
+    daemon_reload: yes
+  when: "matrix_dimension_service_stat.stat.exists|bool"
+
+- name: Ensure Dimension base directory doesn't exist
+  file:
+    path: "{{ matrix_dimension_base_path }}"
+    state: absent
+
+- name: Ensure Dimension Docker image doesn't exist
+  docker_image:
+    name: "{{ matrix_dimension_docker_image }}"
+    state: absent

From 0790a7b2a8bc21e34d12f4436b88e9a3ce75c526 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 20:31:07 +0200
Subject: [PATCH 098/179] Add support for
 matrix_dimension_systemd_{required,wanted}_services_list

We were referencing them from `group_vars/matrix_servers` since
recently, but there were no such variables and they weren't being put to
use.
---
 roles/matrix-dimension/defaults/main.yml                 | 6 ++++++
 .../templates/systemd/matrix-dimension.service.j2        | 9 +++++++--
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/roles/matrix-dimension/defaults/main.yml b/roles/matrix-dimension/defaults/main.yml
index 3b69227a..7ead6e89 100644
--- a/roles/matrix-dimension/defaults/main.yml
+++ b/roles/matrix-dimension/defaults/main.yml
@@ -15,6 +15,12 @@ matrix_dimension_base_path: "{{ matrix_base_data_path }}/dimension"
 matrix_dimension_docker_image: "docker.io/turt2live/matrix-dimension:latest"
 matrix_dimension_docker_image_force_pull: "{{ matrix_dimension_docker_image.endswith(':latest') }}"
 
+# List of systemd services that matrix-dimension.service depends on.
+matrix_dimension_systemd_required_services_list: ['docker.service']
+
+# List of systemd services that matrix-dimension.service wants
+matrix_dimension_systemd_wanted_services_list: []
+
 # The user and group id correspond to the node user in the `turt2live/matrix-dimension` image.
 matrix_dimension_user_uid: '1000'
 matrix_dimension_user_gid: '1000'
diff --git a/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 b/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2
index 94c38491..1c900a16 100644
--- a/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2
+++ b/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2
@@ -1,8 +1,13 @@
 #jinja2: lstrip_blocks: "True"
 [Unit]
 Description=Matrix Dimension
-After=docker.service
-Requires=docker.service
+{% for service in matrix_dimension_systemd_required_services_list %}
+Requires={{ service }}
+After={{ service }}
+{% endfor %}
+{% for service in matrix_dimension_systemd_wanted_services_list %}
+Wants={{ service }}
+{% endfor %}
 DefaultDependencies=no
 
 [Service]

From 2a502db2394af179bcb6def67ac89c7fd83d2d95 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 21:01:47 +0200
Subject: [PATCH 099/179] Add (SQLite + Postgres) support and automatic
 migration to matrix-dimension

---
 group_vars/matrix_servers                     |  9 +++--
 roles/matrix-dimension/defaults/main.yml      | 22 +++++++++++++
 .../matrix-dimension/tasks/setup_install.yml  | 33 +++++++++++++++++++
 .../matrix-dimension/templates/config.yaml.j2 |  6 +++-
 .../systemd/matrix-dimension.service.j2       |  4 ++-
 5 files changed, 67 insertions(+), 7 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index ad62a86e..d7163ab7 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -716,7 +716,7 @@ matrix_dimension_systemd_required_services_list: |
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_dimension_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_dimension_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.dimension.db') | to_uuid }}"
+matrix_dimension_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -1092,13 +1092,12 @@ matrix_postgres_additional_databases: |
     }] if (matrix_mx_puppet_steam_enabled
            and matrix_mx_puppet_steam_database_engine == 'postgres'
            and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else [])
-     + ([{
+     +
+    ([{
       'name': matrix_dimension_database_db_name,
       'username': matrix_dimension_database_username,
       'password': matrix_dimension_database_password,
-    }] if (matrix_dimension_enabled
-           and matrix_dimension_database_engine == 'postgres'
-           and matrix_dimension_database_hostname == 'matrix-postgres') else [])
+    }] if (matrix_dimension_enabled and matrix_dimension_database_engine == 'postgres' and matrix_dimension_database_hostname == 'matrix-postgres') else [])
    }}
 
 ######################################################################
diff --git a/roles/matrix-dimension/defaults/main.yml b/roles/matrix-dimension/defaults/main.yml
index 7ead6e89..a6a3bf5d 100644
--- a/roles/matrix-dimension/defaults/main.yml
+++ b/roles/matrix-dimension/defaults/main.yml
@@ -40,6 +40,28 @@ matrix_dimension_integrations_jitsi_widget_url: "https://{{ matrix_server_fqn_di
 
 matrix_dimension_homeserver_federationUrl: "http://matrix-synapse:8048"
 
+
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_dimension_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_dimension_postgres_*` variables
+matrix_dimension_database_engine: 'sqlite'
+
+matrix_dimension_sqlite_database_path_local: "{{ matrix_dimension_base_path }}/dimension.db"
+matrix_dimension_sqlite_database_path_in_container: "dimension.db"
+
+matrix_dimension_database_username: 'matrix_dimension'
+matrix_dimension_database_password: 'some-password'
+matrix_dimension_database_hostname: 'matrix-postgres'
+matrix_dimension_database_port: 5432
+matrix_dimension_database_db_name: 'matrix_dimension'
+
+matrix_dimension_database_connection_string: 'postgres://{{ matrix_dimension_database_username }}:{{ matrix_dimension_database_password }}@{{ matrix_dimension_database_hostname }}:{{ matrix_dimension_database_port }}/{{ matrix_dimension_database_db_name }}'
+
+
 # Default Dimension configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
 #
diff --git a/roles/matrix-dimension/tasks/setup_install.yml b/roles/matrix-dimension/tasks/setup_install.yml
index e118dd62..9a264449 100644
--- a/roles/matrix-dimension/tasks/setup_install.yml
+++ b/roles/matrix-dimension/tasks/setup_install.yml
@@ -1,5 +1,32 @@
 ---
 
+- set_fact:
+    matrix_dimension_requires_restart: false
+
+- block:
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_dimension_sqlite_database_path_local }}"
+      register: matrix_dimension_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_dimension_sqlite_database_path_local }}"
+              dst: "{{ matrix_dimension_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_dimension_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-dimension.service']
+              pgloader_options: ['--with "quote identifiers"']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_dimension_requires_restart: true
+      when: "matrix_dimension_sqlite_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_dimension_database_engine == 'postgres'"
+
 - name: Ensure Dimension base path exists
   file:
     path: "{{ matrix_dimension_base_path }}"
@@ -34,3 +61,9 @@
   service:
     daemon_reload: yes
   when: "matrix_dimension_systemd_service_result.changed|bool"
+
+- name: Ensure matrix-dimension.service restarted, if necessary
+  service:
+    name: "matrix-dimension.service"
+    state: restarted
+  when: "matrix_dimension_requires_restart|bool"
diff --git a/roles/matrix-dimension/templates/config.yaml.j2 b/roles/matrix-dimension/templates/config.yaml.j2
index b7ca1ee3..200871e7 100644
--- a/roles/matrix-dimension/templates/config.yaml.j2
+++ b/roles/matrix-dimension/templates/config.yaml.j2
@@ -44,7 +44,11 @@ widgetBlacklist:
 
 # Where the database for Dimension is
 database:
-  uri: "postgres://matrix_dimension:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_dimension') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_dimension"
+{% if matrix_dimension_database_engine == 'sqlite' %}
+  file: {{ matrix_dimension_sqlite_database_path_in_container|to_json }}
+{% elif matrix_dimension_database_engine == 'postgres' %}
+  uri: {{ matrix_dimension_database_connection_string|to_json }}
+{% endif %}
 
 # Display settings that apply to self-hosted go-neb instances
 goneb:
diff --git a/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 b/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2
index 1c900a16..ff10224a 100644
--- a/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2
+++ b/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2
@@ -16,7 +16,9 @@ ExecStartPre=-{{ matrix_host_command_docker }} kill matrix-dimension
 ExecStartPre=-{{ matrix_host_command_docker }} rm matrix-dimension
 
 # Fixup database ownership if it got changed somehow (during a server migration, etc.)
-ExecStartPre=-{{ matrix_host_command_chown }} {{ matrix_dimension_user_uid }}:{{ matrix_dimension_user_gid }} {{ matrix_dimension_base_path }}/dimension.db
+{% if matrix_dimension_database_engine == 'sqlite' %}
+ExecStartPre=-{{ matrix_host_command_chown }} {{ matrix_dimension_user_uid }}:{{ matrix_dimension_user_gid }} {{ matrix_dimension_sqlite_database_path_local }}
+{% endif %}
 
 ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-dimension \
 			--log-driver=none \

From dd797ba6a76e18f8b2bfa676b6f3e78f18f7cf47 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 22:28:20 +0200
Subject: [PATCH 100/179] Fix Postgres database importing/upgrading conflicts

We were running into conflicts, because having initialized
the roles (users) and databases, trying to import leads to
errors (role XXX already exists, etc.).

We were previously ignoring the Synapse database (`homeserver`)
when upgrading/importing, because that one gets created by default
whenever the container starts.

For our additional databases, it's a similar situation now.
It's not created by default as soon as Postgres starts with an empty
database, but rather we create it as part of running the playbook.

So we either need to skip those role/database creation statements
while upgrading/importing, or to avoid creating the additional database
and rely on the import for that. I've gone for the former, because
it's already similar to what we were doing and it's simpler
(it lets `setup_postgres.yml` be the same in all scenarios).
---
 group_vars/matrix_servers                        | 14 ++++++++++++++
 roles/matrix-postgres/defaults/main.yml          | 16 ++++++++++++++++
 roles/matrix-postgres/tasks/import_postgres.yml  |  4 ++--
 roles/matrix-postgres/tasks/upgrade_postgres.yml |  4 ++--
 4 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index d7163ab7..932b288a 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -1100,6 +1100,20 @@ matrix_postgres_additional_databases: |
     }] if (matrix_dimension_enabled and matrix_dimension_database_engine == 'postgres' and matrix_dimension_database_hostname == 'matrix-postgres') else [])
    }}
 
+matrix_postgres_import_roles_to_ignore: |
+  {{
+    [matrix_postgres_connection_username]
+    +
+    matrix_postgres_additional_databases|map(attribute='username')
+  }}
+
+matrix_postgres_import_databases_to_ignore: |
+  {{
+    [matrix_postgres_db_name]
+    +
+    matrix_postgres_additional_databases|map(attribute='name')
+  }}
+
 ######################################################################
 #
 # /matrix-postgres
diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml
index 5755742f..0c516281 100644
--- a/roles/matrix-postgres/defaults/main.yml
+++ b/roles/matrix-postgres/defaults/main.yml
@@ -43,6 +43,22 @@ matrix_postgres_container_postgres_bind_port: ""
 #     password: some_password
 matrix_postgres_additional_databases: []
 
+# A list of roles/users to avoid creating when importing (or upgrading) the database.
+# If a dump file contains the roles and they've also been created beforehand (see `matrix_postgres_additional_databases`),
+# importing would fail.
+# We either need to not create them or to ignore the `CREATE ROLE` statements in the dump.
+matrix_postgres_import_roles_to_ignore: [matrix_postgres_connection_username]
+
+matrix_postgres_import_roles_ignore_regex: "^CREATE ROLE ({{ matrix_postgres_import_roles_to_ignore|join('|') }});"
+
+# A list of databases to avoid creating when importing (or upgrading) the database.
+# If a dump file contains the databases and they've also been created beforehand (see `matrix_postgres_additional_databases`),
+# importing would fail.
+# We either need to not create them or to ignore the `CREATE DATABASE` statements in the dump.
+matrix_postgres_import_databases_to_ignore: [matrix_postgres_db_name]
+
+matrix_postgres_import_databases_ignore_regex: "^CREATE DATABASE ({{ matrix_postgres_import_databases_to_ignore|join('|') }})\\s"
+
 # The number of seconds to wait after starting `matrix-postgres.service`
 # and before trying to run queries for creating additional databases/users against it.
 #
diff --git a/roles/matrix-postgres/tasks/import_postgres.yml b/roles/matrix-postgres/tasks/import_postgres.yml
index 33d98691..c26affbb 100644
--- a/roles/matrix-postgres/tasks/import_postgres.yml
+++ b/roles/matrix-postgres/tasks/import_postgres.yml
@@ -74,8 +74,8 @@
       {{ matrix_postgres_docker_image_latest }}
       -c "cat /{{ server_path_postgres_dump|basename }} |
       {{ 'gunzip |' if server_path_postgres_dump.endswith('.gz') else '' }}
-      grep -vE '^CREATE ROLE {{ matrix_postgres_connection_username }}' |
-      grep -vE '^CREATE DATABASE {{ matrix_postgres_db_name }}' |
+      grep -vE '{{ matrix_postgres_import_roles_ignore_regex }}' |
+      grep -vE '{{ matrix_postgres_import_databases_ignore_regex }}' |
       psql -v ON_ERROR_STOP=1 -h matrix-postgres"
 
 # This is a hack.
diff --git a/roles/matrix-postgres/tasks/upgrade_postgres.yml b/roles/matrix-postgres/tasks/upgrade_postgres.yml
index 72f327b3..564265d8 100644
--- a/roles/matrix-postgres/tasks/upgrade_postgres.yml
+++ b/roles/matrix-postgres/tasks/upgrade_postgres.yml
@@ -135,8 +135,8 @@
       {{ matrix_postgres_docker_image_latest }}
       -c "cat /in/{{ postgres_dump_name }} |
       {{ 'gunzip |' if postgres_dump_name.endswith('.gz') else '' }}
-      grep -vE '^CREATE ROLE {{ matrix_postgres_connection_username }}' |
-      grep -vE '^CREATE DATABASE {{ matrix_postgres_db_name }}' |
+      grep -vE '{{ matrix_postgres_import_roles_ignore_regex }}' |
+      grep -vE '{{ matrix_postgres_import_databases_ignore_regex }}' |
       psql -v ON_ERROR_STOP=1 -h matrix-postgres"
 
 # This is a hack.

From e2ba46bf013785079eeca380a9926598b9fd1fe4 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 22:40:37 +0200
Subject: [PATCH 101/179] Fix Jinja2 syntax error (else if -> elif)

---
 roles/matrix-bridge-appservice-irc/defaults/main.yml      | 2 +-
 roles/matrix-bridge-appservice-slack/defaults/main.yml    | 2 +-
 roles/matrix-bridge-mx-puppet-discord/defaults/main.yml   | 2 +-
 roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml | 2 +-
 roles/matrix-bridge-mx-puppet-skype/defaults/main.yml     | 2 +-
 roles/matrix-bridge-mx-puppet-slack/defaults/main.yml     | 2 +-
 roles/matrix-bridge-mx-puppet-steam/defaults/main.yml     | 2 +-
 roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml   | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/roles/matrix-bridge-appservice-irc/defaults/main.yml b/roles/matrix-bridge-appservice-irc/defaults/main.yml
index bf23b0ab..5110fd85 100644
--- a/roles/matrix-bridge-appservice-irc/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-irc/defaults/main.yml
@@ -26,7 +26,7 @@ matrix_appservice_irc_database_name: matrix_appservice_irc
 matrix_appservice_irc_database_connString: >-2
   {%- if matrix_appservice_irc_database_engine == 'postgres' -%}
   postgresql://{{ matrix_appservice_irc_database_username }}:{{ matrix_appservice_irc_database_password }}@{{ matrix_appservice_irc_database_hostname }}:{{ matrix_appservice_irc_database_port }}/{{ matrix_appservice_irc_database_name }}?sslmode=disable
-  {%- else if matrix_appservice_irc_database_engine == 'nedb' -%}
+  {%- elif matrix_appservice_irc_database_engine == 'nedb' -%}
   {{ matrix_appservice_irc_database_engine }}://{{ matrix_appservice_irc_database_file }}
   {%- endif -%}
 
diff --git a/roles/matrix-bridge-appservice-slack/defaults/main.yml b/roles/matrix-bridge-appservice-slack/defaults/main.yml
index e2127c29..365493ff 100644
--- a/roles/matrix-bridge-appservice-slack/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-slack/defaults/main.yml
@@ -55,7 +55,7 @@ matrix_appservice_slack_database_file: /data
 matrix_appservice_slack_database_connString: >-2
   {%- if matrix_appservice_slack_database_engine == 'postgres' -%}
   postgresql://{{ matrix_appservice_slack_database_username }}:{{ matrix_appservice_slack_database_password }}@{{ matrix_appservice_slack_database_hostname }}:{{ matrix_appservice_slack_database_port }}/{{ matrix_appservice_slack_database_name }}?sslmode=disable
-  {%- else if matrix_appservice_slack_database_engine == 'nedb' -%}
+  {%- elif matrix_appservice_slack_database_engine == 'nedb' -%}
   {{ matrix_appservice_slack_database_engine }}://{{ matrix_appservice_slack_database_file }}
   {%- endif -%}
 
diff --git a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
index 5a65f33e..a0ad1c0a 100644
--- a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
@@ -66,7 +66,7 @@ matrix_mx_puppet_discord_database_file: /data/database.db
 matrix_mx_puppet_discord_database_connString: >-2
   {%- if matrix_mx_puppet_discord_database_engine == 'postgres' -%}
   postgresql://{{ matrix_mx_puppet_discord_database_username }}:{{ matrix_mx_puppet_discord_database_password }}@{{ matrix_mx_puppet_discord_database_hostname }}:{{ matrix_mx_puppet_discord_database_port }}/{{ matrix_mx_puppet_discord_database_name }}?sslmode=disable
-  {%- else if matrix_mx_puppet_discord_database_engine == 'sqlite' -%}
+  {%- elif matrix_mx_puppet_discord_database_engine == 'sqlite' -%}
   {{ matrix_mx_puppet_discord_database_engine }}://{{ matrix_mx_puppet_discord_database_file }}
   {%- endif -%}
 
diff --git a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
index c30f9b8b..4c40bda4 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
@@ -56,7 +56,7 @@ matrix_mx_puppet_instagram_database_file: /data/database.db
 matrix_mx_puppet_instagram_database_connString: >-2
   {%- if matrix_mx_puppet_instagram_database_engine == 'postgres' -%}
   postgresql://{{ matrix_mx_puppet_instagram_database_username }}:{{ matrix_mx_puppet_instagram_database_password }}@{{ matrix_mx_puppet_instagram_database_hostname }}:{{ matrix_mx_puppet_instagram_database_port }}/{{ matrix_mx_puppet_instagram_database_name }}?sslmode=disable
-  {%- else if matrix_mx_puppet_instagram_database_engine == 'sqlite' -%}
+  {%- elif matrix_mx_puppet_instagram_database_engine == 'sqlite' -%}
   {{ matrix_mx_puppet_instagram_database_engine }}://{{ matrix_mx_puppet_instagram_database_file }}
   {%- endif -%}
 
diff --git a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
index 3b7448c6..db0f4ede 100644
--- a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
@@ -64,7 +64,7 @@ matrix_mx_puppet_skype_database_file: /data/database.db
 matrix_mx_puppet_skype_database_connString: >-2
   {%- if matrix_mx_puppet_skype_database_engine == 'postgres' -%}
   postgresql://{{ matrix_mx_puppet_skype_database_username }}:{{ matrix_mx_puppet_skype_database_password }}@{{ matrix_mx_puppet_skype_database_hostname }}:{{ matrix_mx_puppet_skype_database_port }}/{{ matrix_mx_puppet_skype_database_name }}?sslmode=disable
-  {%- else if matrix_mx_puppet_skype_database_engine == 'sqlite' -%}
+  {%- elif matrix_mx_puppet_skype_database_engine == 'sqlite' -%}
   {{ matrix_mx_puppet_skype_database_engine }}://{{ matrix_mx_puppet_skype_database_file }}
   {%- endif -%}
 
diff --git a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
index 5f91beb8..2ed72611 100644
--- a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
@@ -68,7 +68,7 @@ matrix_mx_puppet_slack_database_file: /data/database.db
 matrix_mx_puppet_slack_database_connString: >-2
   {%- if matrix_mx_puppet_slack_database_engine == 'postgres' -%}
   postgresql://{{ matrix_mx_puppet_slack_database_username }}:{{ matrix_mx_puppet_slack_database_password }}@{{ matrix_mx_puppet_slack_database_hostname }}:{{ matrix_mx_puppet_slack_database_port }}/{{ matrix_mx_puppet_slack_database_name }}?sslmode=disable
-  {%- else if matrix_mx_puppet_slack_database_engine == 'sqlite' -%}
+  {%- elif matrix_mx_puppet_slack_database_engine == 'sqlite' -%}
   {{ matrix_mx_puppet_slack_database_engine }}://{{ matrix_mx_puppet_slack_database_file }}
   {%- endif -%}
 
diff --git a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
index be343db4..77f7804b 100644
--- a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
@@ -65,7 +65,7 @@ matrix_mx_puppet_steam_database_file: /data/database.db
 matrix_mx_puppet_steam_database_connString: >-2
   {%- if matrix_mx_puppet_steam_database_engine == 'postgres' -%}
   postgresql://{{ matrix_mx_puppet_steam_database_username }}:{{ matrix_mx_puppet_steam_database_password }}@{{ matrix_mx_puppet_steam_database_hostname }}:{{ matrix_mx_puppet_steam_database_port }}/{{ matrix_mx_puppet_steam_database_name }}?sslmode=disable
-  {%- else if matrix_mx_puppet_steam_database_engine == 'sqlite' -%}
+  {%- elif matrix_mx_puppet_steam_database_engine == 'sqlite' -%}
   {{ matrix_mx_puppet_steam_database_engine }}://{{ matrix_mx_puppet_steam_database_file }}
   {%- endif -%}
 
diff --git a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
index f536029c..91aa264a 100644
--- a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
@@ -71,7 +71,7 @@ matrix_mx_puppet_twitter_database_file: /data/database.db
 matrix_mx_puppet_twitter_database_connString: >-2
   {%- if matrix_mx_puppet_twitter_database_engine == 'postgres' -%}
   postgresql://{{ matrix_mx_puppet_twitter_database_username }}:{{ matrix_mx_puppet_twitter_database_password }}@{{ matrix_mx_puppet_twitter_database_hostname }}:{{ matrix_mx_puppet_twitter_database_port }}/{{ matrix_mx_puppet_twitter_database_name }}?sslmode=disable
-  {%- else if matrix_mx_puppet_twitter_database_engine == 'sqlite' -%}
+  {%- elif matrix_mx_puppet_twitter_database_engine == 'sqlite' -%}
   {{ matrix_mx_puppet_twitter_database_engine }}://{{ matrix_mx_puppet_twitter_database_file }}
   {%- endif -%}
 

From 6a3f6fbcb56b8aa2da098d1d4cbd1820a347ee34 Mon Sep 17 00:00:00 2001
From: Tomas Strand <tomas@fik1.net>
Date: Tue, 15 Dec 2020 14:15:40 +0200
Subject: [PATCH 102/179] Missing ` in traefik domains instructions

---
 docs/configuring-playbook-own-webserver.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuring-playbook-own-webserver.md b/docs/configuring-playbook-own-webserver.md
index 8c671df4..c930da40 100644
--- a/docs/configuring-playbook-own-webserver.md
+++ b/docs/configuring-playbook-own-webserver.md
@@ -144,7 +144,7 @@ matrix_nginx_proxy_container_extra_arguments:
   - '--label "traefik.enable=true"'
 
   # The Nginx proxy container will receive traffic from these subdomains
-  - '--label "traefik.http.routers.matrix-nginx-proxy.rule=Host(`{{ matrix_server_fqn_matrix }}`,`{{ matrix_server_fqn_element }}`,`{{ matrix_server_fqn_dimension }},`{{ matrix_server_fqn_jitsi }}`)"'
+  - '--label "traefik.http.routers.matrix-nginx-proxy.rule=Host(`{{ matrix_server_fqn_matrix }}`,`{{ matrix_server_fqn_element }}`,`{{ matrix_server_fqn_dimension }}`,`{{ matrix_server_fqn_jitsi }}`)"'
 
   # (The 'web-secure' entrypoint must bind to port 443 in Traefik config)
   - '--label "traefik.http.routers.matrix-nginx-proxy.entrypoints=web-secure"'

From 69f71f48a660a104bd394309edae83026e75636c Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 15 Dec 2020 22:00:24 +0200
Subject: [PATCH 103/179] Upgrade matrix-registration (v0.7.1 -> 0.7.2) and use
 official image

This switches us to a container image maintained by the
matrix-registration developer.

0.7.2 also supports a `base_url` configuration option we can use to
make it easier to reverse-proxy at a different base URL.

We still keep some workarounds, because of this issue:
https://github.com/ZerataX/matrix-registration/issues/47
---
 roles/matrix-registration/defaults/main.yml        | 6 ++++--
 roles/matrix-registration/tasks/init.yml           | 9 +++------
 roles/matrix-registration/templates/config.yaml.j2 | 1 +
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/roles/matrix-registration/defaults/main.yml b/roles/matrix-registration/defaults/main.yml
index 95147665..d89bf070 100644
--- a/roles/matrix-registration/defaults/main.yml
+++ b/roles/matrix-registration/defaults/main.yml
@@ -12,9 +12,9 @@ matrix_registration_config_path: "{{ matrix_registration_base_path }}/config"
 matrix_registration_data_path: "{{ matrix_registration_base_path }}/data"
 matrix_registration_docker_src_files_path: "{{ matrix_registration_base_path }}/docker-src"
 
-matrix_registration_version: "v0.7.1"
+matrix_registration_version: "v0.7.2"
 
-matrix_registration_docker_image: "{{ matrix_registration_docker_image_name_prefix }}devture/zeratax-matrix-registration:{{ matrix_registration_version }}"
+matrix_registration_docker_image: "{{ matrix_registration_docker_image_name_prefix }}zeratax/matrix-registration:{{ matrix_registration_version }}"
 matrix_registration_docker_image_name_prefix: "{{ 'localhost/' if matrix_registration_container_image_self_build else 'docker.io/' }}"
 matrix_registration_docker_image_force_pull: "{{ matrix_registration_docker_image.endswith(':latest') }}"
 
@@ -36,6 +36,8 @@ matrix_registration_container_http_host_bind_port: ''
 # (only applies when matrix-nginx-proxy is used).
 matrix_registration_public_endpoint: /matrix-registration
 
+matrix_registration_base_url: "{{ matrix_registration_public_endpoint }}"
+
 matrix_registration_api_register_endpoint: "{{ matrix_homeserver_url }}{{ matrix_registration_public_endpoint }}/register"
 matrix_registration_api_token_endpoint: "{{ matrix_homeserver_url }}{{ matrix_registration_public_endpoint }}/token"
 
diff --git a/roles/matrix-registration/tasks/init.yml b/roles/matrix-registration/tasks/init.yml
index 8a7cdc75..bdb3928e 100644
--- a/roles/matrix-registration/tasks/init.yml
+++ b/roles/matrix-registration/tasks/init.yml
@@ -30,15 +30,12 @@
         {% endif %}
 
           {#
-            Workaround matrix-registration serving static files at /static
-            (see https://github.com/ZerataX/matrix-registration/issues/29)
-
-            Also fixing the form, which goes to /register.
+            Workaround matrix-registration serving the background image at /static
+            (see https://github.com/ZerataX/matrix-registration/issues/47)
           #}
           sub_filter_once off;
-          sub_filter_types text/html text/css;
+          sub_filter_types text/css;
           sub_filter "/static/" "{{ matrix_registration_public_endpoint }}/static/";
-          sub_filter "/register" "{{ matrix_registration_public_endpoint }}/register";
         }
 
   - name: Register matrix-registration proxying configuration with matrix-nginx-proxy
diff --git a/roles/matrix-registration/templates/config.yaml.j2 b/roles/matrix-registration/templates/config.yaml.j2
index f3b1c57b..1b2464e5 100644
--- a/roles/matrix-registration/templates/config.yaml.j2
+++ b/roles/matrix-registration/templates/config.yaml.j2
@@ -28,3 +28,4 @@ logging:
 # password requirements
 password:
   min_length: 8
+base_url: {{ matrix_registration_base_url|to_json }}

From 1bd5c240e582bfc4d4d1fcaaea5be326f5186e92 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 15 Dec 2020 23:18:29 +0200
Subject: [PATCH 104/179] Add support for executing additional DB migration
 statements

In cases where pgloader is not enough and we need to do some additional
migration work after it, we can now use
`additional_psql_statements_list` and
`additional_psql_statements_db_name`.

This is to be used when migrating `matrix-registration`'s data at the
very least.
---
 .../tasks/util/migrate_db_to_postgres.yml     | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
index 5d917538..d01611ef 100644
--- a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
+++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
@@ -99,6 +99,29 @@
   command:
     cmd: "mv {{ matrix_postgres_db_migration_request.src }} {{ matrix_postgres_db_migration_request.src }}.backup"
 
+- block:
+    # We can't use `{{ role_path }}` here, neither with `import_tasks`, nor with `include_tasks`,
+    # because it refers to the role that included this util, and not to the role this file belongs to.
+    - import_tasks: "roles/matrix-postgres/tasks/util/detect_existing_postgres_version.yml"
+
+    - set_fact:
+        matrix_postgres_docker_image_to_use: "{{ matrix_postgres_docker_image_latest if matrix_postgres_detected_version_corresponding_docker_image == '' else matrix_postgres_detected_version_corresponding_docker_image }}"
+
+    - name: Execute additional Postgres SQL migration statements
+      command:
+        cmd: >-
+          {{ matrix_host_command_docker }} run
+          --rm
+          --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
+          --cap-drop=ALL
+          --env-file={{ matrix_postgres_base_path }}/env-postgres-psql
+          --network={{ matrix_docker_network }}
+          {{ matrix_postgres_docker_image_to_use }}
+          psql --host=matrix-postgres --dbname={{ matrix_postgres_db_migration_request.additional_psql_statements_db_name }} --command='{{ item }}'
+      with_items: "{{ matrix_postgres_db_migration_request.additional_psql_statements_list }}"
+
+  when: "matrix_postgres_db_migration_request.additional_psql_statements_list|default([])|length > 0"
+
 - name: Inject result
   set_fact:
     matrix_playbook_runtime_results: |

From a197968b7f95c6f7e6d5f3882cb25e1dc1892124 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 15 Dec 2020 23:19:56 +0200
Subject: [PATCH 105/179] Make matrix-registration use Postgres by default

Now that 0.7.2 is out, the Docker image supports Postgres
and we can do the (SQLite -> Postgres) migration.

I've also found out that we needed to fix up the `tokens.ex_date` column
data type a bit to prevent matrix-registration from raising exceptions
when comparing `datetime.now()` with `ex_date` coming from the database.

Example:

> File "/usr/local/lib/python3.8/site-packages/matrix_registration/tokens.py", line 58, in valid
> expired = self.ex_date < datetime.now()
> TypeError: can't compare offset-naive and offset-aware datetimes
---
 group_vars/matrix_servers                           | 5 ++---
 roles/matrix-registration/defaults/main.yml         | 1 -
 roles/matrix-registration/tasks/setup_install.yml   | 5 +++++
 roles/matrix-registration/tasks/validate_config.yml | 7 -------
 4 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 932b288a..fda40efd 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -1317,9 +1317,8 @@ matrix_registration_systemd_required_services_list: |
     (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
-# We'd like to use 'postgres' if matrix_postgres_enabled, but the container image doesn't seem to support that.
-# Learn more here: https://github.com/ZerataX/matrix-registration/issues/44
-matrix_registration_database_engine: 'sqlite'
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_registration_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
 matrix_registration_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx.registr.db') | to_uuid }}"
 
 ######################################################################
diff --git a/roles/matrix-registration/defaults/main.yml b/roles/matrix-registration/defaults/main.yml
index b39f02c4..d85faf89 100644
--- a/roles/matrix-registration/defaults/main.yml
+++ b/roles/matrix-registration/defaults/main.yml
@@ -88,7 +88,6 @@ matrix_registration_admin_secret: ""
 
 matrix_registration_riot_instance: "https://riot.im/app/"
 
-
 # Default matrix-registration configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
 #
diff --git a/roles/matrix-registration/tasks/setup_install.yml b/roles/matrix-registration/tasks/setup_install.yml
index 489bab8b..708cb1df 100644
--- a/roles/matrix-registration/tasks/setup_install.yml
+++ b/roles/matrix-registration/tasks/setup_install.yml
@@ -18,6 +18,11 @@
               engine_variable_name: 'matrix_registration_database_engine'
               engine_old: 'sqlite'
               systemd_services_to_stop: ['matrix-registration.service']
+              # pgloader makes `ex_date` of type `TIMESTAMP WITH TIMEZONE`,
+              # which makes matrix-registration choke on it later on when comparing dates.
+              additional_psql_statements_list:
+                - ALTER TABLE tokens ALTER COLUMN ex_date TYPE TIMESTAMP WITHOUT TIME ZONE;
+              additional_psql_statements_db_name: "{{ matrix_registration_database_db_name }}"
 
         - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
 
diff --git a/roles/matrix-registration/tasks/validate_config.yml b/roles/matrix-registration/tasks/validate_config.yml
index 80293bcb..90466b46 100644
--- a/roles/matrix-registration/tasks/validate_config.yml
+++ b/roles/matrix-registration/tasks/validate_config.yml
@@ -18,10 +18,3 @@
   when: "item.old in vars"
   with_items:
     - {'old': 'matrix_registration_docker_repo', 'new': 'matrix_registration_container_image_self_build_repo'}
-
-- name: Fail if Postgres usage attempted
-  fail:
-    msg: >
-      matrix-registration doesn't support using Postgres just yet.
-      Learn more here: https://github.com/ZerataX/matrix-registration/issues/44
-  when: "matrix_registration_database_engine == 'postgres'"

From a4b8baee494e670c15999be1c2c14b314e6d6f04 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Wed, 16 Dec 2020 03:32:31 +0200
Subject: [PATCH 106/179] Fix inability to send (Matrix -> Discord) messages
 via appservice-discord

Revert "Correct inabillity for appservice-discord to connect"
This reverts commit 673e19f8307bdfc76dc4d1e63dcab40c84ebb37e.

While certain things do work even with such a local URL, sending
messages leads to an error like this:

> [DiscordBot] verbose: DiscordAPIError: Invalid Form Body
> avatar_url: Not a well formed URL.

Fixes https://github.com/Half-Shot/matrix-appservice-discord/issues/649

The sample configuration file for appservice-discord
https://github.com/Half-Shot/matrix-appservice-discord/blob/c29cfc72f55f1f81e43e2d71f5c080d752d1a884/config/config.sample.yaml#L8
explicitly says that we need a public URL.
---
 roles/matrix-bridge-appservice-discord/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml
index 357b93d6..9ca06b05 100644
--- a/roles/matrix-bridge-appservice-discord/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml
@@ -37,7 +37,7 @@ matrix_appservice_discord_appservice_url: 'http://matrix-appservice-discord:9005
 
 matrix_appservice_discord_bridge_domain: "{{ matrix_domain }}"
 # As of right now, the homeserver URL must be a public URL. See below.
-matrix_appservice_discord_bridge_homeserverUrl: "http://matrix-synapse:8008"
+matrix_appservice_discord_bridge_homeserverUrl: "{{ matrix_homeserver_url }}"
 matrix_appservice_discord_bridge_disablePresence: false
 matrix_appservice_discord_bridge_enableSelfServiceBridging: false
 

From db69a51653271d8bf563cc2ef952064fa05bb825 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Wed, 16 Dec 2020 16:25:38 +0200
Subject: [PATCH 107/179] Do not recommend usePrivilegedIntents for
 appservice-discord

We're getting reports of it being broken and I can confirm that the
bridge works well without it, so I don't see why we'd be asking
people to enable it.

Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/751
---
 ...configuring-playbook-bridge-appservice-discord.md | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/docs/configuring-playbook-bridge-appservice-discord.md b/docs/configuring-playbook-bridge-appservice-discord.md
index 11be678e..f3efc555 100644
--- a/docs/configuring-playbook-bridge-appservice-discord.md
+++ b/docs/configuring-playbook-bridge-appservice-discord.md
@@ -14,20 +14,18 @@ Instructions loosely based on [this](https://github.com/Half-Shot/matrix-appserv
 1. Create a Discord Application [here](https://discordapp.com/developers/applications).
 2. Retrieve Client ID.
 3. Create a bot from the Bot tab and retrieve the Bot token.
-4. From the Bot tab, enable all checkboxes related to Privileged Gateway Intents (you can skip this step if you're not using `matrix_appservice_discord_auth_usePrivilegedIntents: true` below)
-5. Enable the bridge with the following configuration in your `vars.yml` file:
+4. Enable the bridge with the following configuration in your `vars.yml` file:
 
 ```yaml
 matrix_appservice_discord_enabled: true
 matrix_appservice_discord_client_id: "YOUR DISCORD APP CLIENT ID"
 matrix_appservice_discord_bot_token: "YOUR DISCORD APP BOT TOKEN"
-matrix_appservice_discord_auth_usePrivilegedIntents: true
 ```
 
-6. If you've already installed Matrix services using the playbook before, you'll need to re-run it (`--tags=setup-all,start`). If not, proceed with [configuring other playbook services](configuring-playbook.md) and then with [Installing](installing.md). Get back to this guide once ready.
-7. Retrieve Discord invite link from the `{{ matrix_appservice_discord_config_path }}/invite_link` file on the server (this defaults to `/matrix/appservice-discord/config/invite_link`). You need to peek at the file on the server via SSH, etc., because it's not available via HTTP(S).
-8. Invite the Bot to Discord servers you wish to bridge. Administrator permission is recommended.
-9. Room addresses follow this syntax: `#_discord_guildid_channelid`. You can easily find the guild and channel ids by logging into Discord in a browser and opening the desired channel. The URL will have this format: `discordapp.com/channels/guild_id/channel_id`. Once you have figured out the appropriate room addrss, you can join by doing `/join #_discord_guildid_channelid` in your Matrix client.
+5. If you've already installed Matrix services using the playbook before, you'll need to re-run it (`--tags=setup-all,start`). If not, proceed with [configuring other playbook services](configuring-playbook.md) and then with [Installing](installing.md). Get back to this guide once ready.
+6. Retrieve Discord invite link from the `{{ matrix_appservice_discord_config_path }}/invite_link` file on the server (this defaults to `/matrix/appservice-discord/config/invite_link`). You need to peek at the file on the server via SSH, etc., because it's not available via HTTP(S).
+7. Invite the Bot to Discord servers you wish to bridge. Administrator permission is recommended.
+8. Room addresses follow this syntax: `#_discord_guildid_channelid`. You can easily find the guild and channel ids by logging into Discord in a browser and opening the desired channel. The URL will have this format: `discordapp.com/channels/guild_id/channel_id`. Once you have figured out the appropriate room addrss, you can join by doing `/join #_discord_guildid_channelid` in your Matrix client.
 
 Other configuration options are available via the `matrix_appservice_discord_configuration_extension_yaml` variable.
 

From e2e6cfaa8b66a995aa4b4ef6b6409d071e648fe9 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Wed, 16 Dec 2020 18:14:52 +0200
Subject: [PATCH 108/179] Document Ansible's Python interpreter discovery some
 more

Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/570

Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/641
---
 docs/prerequisites.md | 2 +-
 examples/hosts        | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/prerequisites.md b/docs/prerequisites.md
index daf6ff80..488f6c6f 100644
--- a/docs/prerequisites.md
+++ b/docs/prerequisites.md
@@ -10,7 +10,7 @@ This playbook doesn't support running on ARM (see [this issue](https://github.co
 
 - `root` access to your server (or a user capable of elevating to `root` via `sudo`).
 
-- [Python](https://www.python.org/) being installed on the server. Most distributions install Python by default, but some don't (e.g. Ubuntu 18.04) and require manual installation (something like `apt-get install python`).
+- [Python](https://www.python.org/) being installed on the server. Most distributions install Python by default, but some don't (e.g. Ubuntu 18.04) and require manual installation (something like `apt-get install python3`). On some distros, Ansible may incorrectly [detect the Python version](https://docs.ansible.com/ansible/latest/reference_appendices/interpreter_discovery.html) (2 vs 3) and you may need to explicitly specify the interpreter path in `inventory/hosts` during installation (e.g. `ansible_python_interpreter=/usr/bin/python3`)
 
 - A `cron`-like tool installed on the server such as `cron` or `anacron` to automatically schedule the Let's Encrypt SSL certificates's renewal. *This can be ignored if you use your own SSL certificates.*
 
diff --git a/examples/hosts b/examples/hosts
index 3b14f09e..daf2cfc5 100644
--- a/examples/hosts
+++ b/examples/hosts
@@ -10,6 +10,9 @@
 #
 # If you're running this Ansible playbook on the same server as the one you're installing to,
 # consider adding an additional `ansible_connection=local` argument below.
+#
+# Ansible may fail to discover which Python interpreter to use on the host for some distros (like Ubuntu 20.04).
+# You may sometimes need to explicitly add `ansible_python_interpreter=/usr/bin/python3` to lines below.
 
 [matrix_servers]
 matrix.<your-domain> ansible_host=<your-server's external IP address> ansible_ssh_user=root

From ed159cc7427c9e856b037864bb9e923f5b7fb2ca Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Thu, 17 Dec 2020 11:33:18 +0200
Subject: [PATCH 109/179] Move matrix_architecture to matrix-base

We were only defining this in `group_vars/matrix_servers`, which is
inconsistent with how we normally do things.
---
 group_vars/matrix_servers           | 14 --------------
 roles/matrix-base/defaults/main.yml |  6 ++++++
 2 files changed, 6 insertions(+), 14 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 78a03851..f5eb1cb8 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -24,20 +24,6 @@ matrix_identity_server_url: "{{ ('https://' + matrix_server_fqn_matrix) if matri
 #
 ######################################################################
 
-######################################################################
-#
-# matrix-architecture
-#
-######################################################################
-
-matrix_architecture: "amd64"
-
-######################################################################
-#
-# /matrix-architecture
-#
-######################################################################
-
 
 ######################################################################
 #
diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml
index e0522ba8..aabbbed8 100644
--- a/roles/matrix-base/defaults/main.yml
+++ b/roles/matrix-base/defaults/main.yml
@@ -23,6 +23,12 @@ matrix_server_fqn_jitsi: "jitsi.{{ matrix_domain }}"
 
 matrix_federation_public_port: 8448
 
+# The architecture that your server runs.
+# Recognized values by us are 'amd64', 'arm32' and 'arm64'.
+# Not all architectures support all services, so your experience (on non-amd64) may vary.
+# See docs/alternative-architectures.md
+matrix_architecture: amd64
+
 matrix_user_username: "matrix"
 matrix_user_groupname: "matrix"
 

From 55f252a6ed276d000e193158a4390b1315354cb2 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Thu, 17 Dec 2020 11:37:30 +0200
Subject: [PATCH 110/179] Do not hardcode amd64 in setup_debian.yml

Until now, we've only supported non-amd64 on Raspbian.

Seems like there are now people running Debian/Ubuntu on ARM,
so we were forcing them into amd64 Docker packages.

I've gotten a report that this change fixes support
for Ubuntu Server 20.04 on RPi 4B.
---
 roles/matrix-base/defaults/main.yml                  | 5 +++++
 roles/matrix-base/tasks/server_base/setup_debian.yml | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml
index aabbbed8..2cc4b42f 100644
--- a/roles/matrix-base/defaults/main.yml
+++ b/roles/matrix-base/defaults/main.yml
@@ -29,6 +29,11 @@ matrix_federation_public_port: 8448
 # See docs/alternative-architectures.md
 matrix_architecture: amd64
 
+# The architecture for Debian packages.
+# See: https://wiki.debian.org/SupportedArchitectures
+# We just remap from our `matrix_architecture` values to what Debian and possibly other distros call things.
+matrix_debian_arch: "{{ 'armhf' if matrix_architecture == 'arm32' else matrix_architecture }}"
+
 matrix_user_username: "matrix"
 matrix_user_groupname: "matrix"
 
diff --git a/roles/matrix-base/tasks/server_base/setup_debian.yml b/roles/matrix-base/tasks/server_base/setup_debian.yml
index 6d8d18fa..e30d3b93 100644
--- a/roles/matrix-base/tasks/server_base/setup_debian.yml
+++ b/roles/matrix-base/tasks/server_base/setup_debian.yml
@@ -20,7 +20,7 @@
 
 - name: Ensure Docker repository is enabled
   apt_repository:
-    repo: "deb [arch=amd64] https://download.docker.com/linux/{{ ansible_distribution|lower }} {{ ansible_distribution_release }} stable"
+    repo: "deb [arch={{ matrix_debian_arch }}] https://download.docker.com/linux/{{ ansible_distribution|lower }} {{ ansible_distribution_release }} stable"
     state: present
     update_cache: yes
   when: matrix_docker_installation_enabled|bool and matrix_docker_package_name == 'docker-ce'

From f545de53f742e4443bf26608e67da786b8d6dc16 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Thu, 17 Dec 2020 11:39:18 +0200
Subject: [PATCH 111/179] Do not hardcode "ubuntu" for the Docker APT key URL

Well, `ubuntu` or `debian`, the same key is served right now,
so it doesn't really matter.

This seems cleaner and less prone to breakage though.
---
 roles/matrix-base/tasks/server_base/setup_debian.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-base/tasks/server_base/setup_debian.yml b/roles/matrix-base/tasks/server_base/setup_debian.yml
index e30d3b93..37706d1f 100644
--- a/roles/matrix-base/tasks/server_base/setup_debian.yml
+++ b/roles/matrix-base/tasks/server_base/setup_debian.yml
@@ -11,7 +11,7 @@
 
 - name: Ensure Docker's APT key is trusted
   apt_key:
-    url: https://download.docker.com/linux/ubuntu/gpg
+    url: "https://download.docker.com/linux/{{ ansible_distribution|lower }}/gpg"
     id: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88
     state: present
   register: add_repository_key

From a09ed58892965241f0ee73b11139b5cfb5995cde Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Thu, 17 Dec 2020 11:45:32 +0200
Subject: [PATCH 112/179] Ensure gnupg installed on Raspbian

It's likely installed by default, but it doesn't hurt to specify it.
It also makes us more the same with `setup_debian.yml`.
---
 roles/matrix-base/tasks/server_base/setup_raspbian.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/roles/matrix-base/tasks/server_base/setup_raspbian.yml b/roles/matrix-base/tasks/server_base/setup_raspbian.yml
index a9a5d20a..a84228a5 100644
--- a/roles/matrix-base/tasks/server_base/setup_raspbian.yml
+++ b/roles/matrix-base/tasks/server_base/setup_raspbian.yml
@@ -5,6 +5,7 @@
     name:
       - apt-transport-https
       - ca-certificates
+      - gnupg
     state: present
     update_cache: yes
 

From 349fbb64343bc3a877db02948ef6e715fa8c0128 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Thu, 17 Dec 2020 11:47:34 +0200
Subject: [PATCH 113/179] Do not hardcode armhf for Raspbian

Raspbian doesn't seem to support arm64, so this is somewhat pointless
right now.

However, they might in the future. Doing this should also unify us
some more with `setup_debian.yml` with the ultimate goal of
eliminating `setup_raspbian.yml`.
---
 roles/matrix-base/tasks/server_base/setup_raspbian.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-base/tasks/server_base/setup_raspbian.yml b/roles/matrix-base/tasks/server_base/setup_raspbian.yml
index a84228a5..daf6c165 100644
--- a/roles/matrix-base/tasks/server_base/setup_raspbian.yml
+++ b/roles/matrix-base/tasks/server_base/setup_raspbian.yml
@@ -20,7 +20,7 @@
 
 - name: Ensure Docker repository is enabled
   apt_repository:
-    repo: "deb [arch=armhf] https://download.docker.com/linux/raspbian {{ ansible_distribution_release }} stable"
+    repo: "deb [arch={{ matrix_debian_arch }}] https://download.docker.com/linux/raspbian {{ ansible_distribution_release }} stable"
     state: present
     update_cache: yes
   when: matrix_docker_installation_enabled|bool and matrix_docker_package_name == 'docker-ce'

From 8748f3d443d7b7388f41d06f2efab17f291735eb Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Thu, 17 Dec 2020 11:49:56 +0200
Subject: [PATCH 114/179] Move python{,3}-docker installation to another task

This also adds support for installing python3-docker (not python-docker)
in systems that run Python 3.
---
 roles/matrix-base/tasks/server_base/setup_raspbian.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-base/tasks/server_base/setup_raspbian.yml b/roles/matrix-base/tasks/server_base/setup_raspbian.yml
index daf6c165..421905a0 100644
--- a/roles/matrix-base/tasks/server_base/setup_raspbian.yml
+++ b/roles/matrix-base/tasks/server_base/setup_raspbian.yml
@@ -28,7 +28,6 @@
 - name: Ensure APT packages are installed
   apt:
     name:
-      - python-docker
       - "{{ matrix_ntpd_package }}"
       - fuse
     state: latest
@@ -38,5 +37,6 @@
   apt:
     name:
       - "{{ matrix_docker_package_name }}"
+      - "python{{'3' if ansible_python.version.major == 3 else ''}}-docker"
     state: latest
   when: matrix_docker_installation_enabled|bool

From d0ee86e0a54f5ae93c872550d8b66f5b660d0b1d Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 21 Dec 2020 15:44:14 +0200
Subject: [PATCH 115/179] Fix matrix_corporal_docker_image_name_prefix
 referencing matrix_synapse_ stuff

---
 roles/matrix-corporal/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-corporal/defaults/main.yml b/roles/matrix-corporal/defaults/main.yml
index 1cab3119..cccaadd0 100644
--- a/roles/matrix-corporal/defaults/main.yml
+++ b/roles/matrix-corporal/defaults/main.yml
@@ -23,7 +23,7 @@ matrix_corporal_container_extra_arguments: []
 matrix_corporal_systemd_required_services_list: ['docker.service']
 
 matrix_corporal_docker_image: "{{ matrix_corporal_docker_image_name_prefix }}devture/matrix-corporal:{{ matrix_corporal_docker_image_tag }}"
-matrix_corporal_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_container_image_self_build else 'docker.io/' }}"
+matrix_corporal_docker_image_name_prefix: "{{ 'localhost/' if matrix_corporal_container_image_self_build else 'docker.io/' }}"
 matrix_corporal_docker_image_tag: "1.11.0"
 matrix_corporal_docker_image_force_pull: "{{ matrix_corporal_docker_image.endswith(':latest') }}"
 

From dcd47166366ad5272525513f3aed409b752057a8 Mon Sep 17 00:00:00 2001
From: louis <louis@systemli.org>
Date: Sun, 20 Dec 2020 17:30:28 +0100
Subject: [PATCH 116/179] add option to disable nginx access log

---
 docs/configuring-playbook-nginx.md                     | 9 ++++++++-
 roles/matrix-nginx-proxy/defaults/main.yml             | 2 ++
 roles/matrix-nginx-proxy/templates/nginx/nginx.conf.j2 | 4 ++++
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/docs/configuring-playbook-nginx.md b/docs/configuring-playbook-nginx.md
index 945864e9..5693c569 100644
--- a/docs/configuring-playbook-nginx.md
+++ b/docs/configuring-playbook-nginx.md
@@ -24,7 +24,6 @@ matrix_nginx_proxy_proxy_matrix_nginx_status_allowed_addresses:
 - 1.1.1.1
 ```
 
-
 ## Synapse + OpenID Connect for Single-Sign-On
 
 If you want to use OpenID Connect as an SSO provider (as per the [Synapse OpenID docs](https://github.com/matrix-org/synapse/blob/develop/docs/openid.md)), you need to use the following configuration (in your `vars.yml` file) to instruct nginx to forward `/_synapse/oidc` to Synapse:
@@ -32,3 +31,11 @@ If you want to use OpenID Connect as an SSO provider (as per the [Synapse OpenID
 ```yaml
 matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_oidc_api_enabled: true
 ```
+
+## Disable Nginx access logs
+
+This will disable the access logging for nginx.
+
+```yaml
+matrix_nginx_proxy_access_log_enabled: false
+```
diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml
index 7331c589..b18bedde 100644
--- a/roles/matrix-nginx-proxy/defaults/main.yml
+++ b/roles/matrix-nginx-proxy/defaults/main.yml
@@ -92,6 +92,8 @@ matrix_nginx_proxy_base_domain_homepage_template: |-
     </body>
   </html>
 
+# Option to disable the access log
+matrix_nginx_proxy_access_log_enabled: true
 
 # Controls whether proxying the riot domain should be done.
 matrix_nginx_proxy_proxy_riot_compat_redirect_enabled: false
diff --git a/roles/matrix-nginx-proxy/templates/nginx/nginx.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/nginx.conf.j2
index 51aa8a00..975c8b4f 100644
--- a/roles/matrix-nginx-proxy/templates/nginx/nginx.conf.j2
+++ b/roles/matrix-nginx-proxy/templates/nginx/nginx.conf.j2
@@ -33,7 +33,11 @@ http {
 					'$status $body_bytes_sent "$http_referer" '
 					'"$http_user_agent" "$http_x_forwarded_for"';
 
+	{% if matrix_nginx_proxy_access_log_enabled %}
 	access_log /var/log/nginx/access.log main;
+	{% else %}
+	access_log off;
+	{% endif %}
 
 	sendfile on;
 	#tcp_nopush on;

From 6488e11d692d0a44fc5c53e81d986ada35c92be4 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 10:52:36 +0200
Subject: [PATCH 117/179] Relocate some tasks

---
 .../tasks/setup_install.yml                   | 34 +++++++++----------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
index c3f2b01f..9022f309 100644
--- a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
@@ -8,6 +8,23 @@
       The matrix-bridge-mx-puppet-discord role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+- name: Check if an old database file already exists
+  stat:
+    path: "{{ matrix_mx_puppet_discord_base_path }}/database.db"
+  register: matrix_mx_puppet_discord_stat_database
+
+- block:
+    - name: (Data relocation) Ensure matrix-mx-puppet-discord.service is stopped
+      service:
+        name: matrix-mx-puppet-discord
+        state: stopped
+        daemon_reload: yes
+      failed_when: False
+
+    - name: (Data relocation) Move mx-puppet-discord database file to ./data directory
+      command: "mv {{ matrix_mx_puppet_discord_base_path }}/database.db {{ matrix_mx_puppet_discord_data_path }}/database.db"
+  when: "matrix_mx_puppet_discord_stat_database.stat.exists"
+
 - name: Ensure MX Puppet Discord image is pulled
   docker_image:
     name: "{{ matrix_mx_puppet_discord_docker_image }}"
@@ -49,23 +66,6 @@
       pull: yes
   when: "matrix_mx_puppet_discord_enabled|bool and matrix_mx_puppet_discord_container_image_self_build|bool"
 
-- name: Check if an old database file already exists
-  stat:
-    path: "{{ matrix_mx_puppet_discord_base_path }}/database.db"
-  register: matrix_mx_puppet_discord_stat_database
-
-- name: (Data relocation) Ensure matrix-mx-puppet-discord.service is stopped
-  service:
-    name: matrix-mx-puppet-discord
-    state: stopped
-    daemon_reload: yes
-  failed_when: false
-  when: "matrix_mx_puppet_discord_stat_database.stat.exists"
-
-- name: (Data relocation) Move mx-puppet-discord database file to ./data directory
-  command: "mv {{ matrix_mx_puppet_discord_base_path }}/database.db {{ matrix_mx_puppet_discord_data_path }}/database.db"
-  when: "matrix_mx_puppet_discord_stat_database.stat.exists"
-
 - name: Ensure mx-puppet-discord config.yaml installed
   copy:
     content: "{{ matrix_mx_puppet_discord_configuration|to_nice_yaml }}"

From 9b4bf7358784f4f7df163cbf922e643dca5f0977 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 11:08:07 +0200
Subject: [PATCH 118/179] Fix undefined variable reference

---
 roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
index c24c7e54..27f0c567 100644
--- a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
@@ -100,7 +100,7 @@ namePatterns:
   group: :name
 
 database:
-{% if matrix_mx_puppet_slack_discord_database_engine == 'sqlite' %}
+{% if matrix_mx_puppet_discord_database_engine == 'sqlite' %}
   # Use SQLite3 as a database backend
   # The name of the database file
   filename: /data/database.db

From 149872e00c59960494877bac032b9a4d0a5d40ea Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 11:10:10 +0200
Subject: [PATCH 119/179] Add support for automatic (Postgres -> SQLite)
 migration to mx-puppet-discord

---
 .../defaults/main.yml                         | 15 +++++------
 .../tasks/setup_install.yml                   | 26 +++++++++++++++++++
 .../tasks/validate_config.yml                 | 20 --------------
 .../templates/config.yaml.j2                  |  4 +--
 4 files changed, 35 insertions(+), 30 deletions(-)

diff --git a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
index a0ad1c0a..ad86a241 100644
--- a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
@@ -57,18 +57,17 @@ matrix_mx_puppet_discord_login_shared_secret: ''
 
 # Database configuration
 matrix_mx_puppet_discord_database_engine: 'sqlite'
+
+matrix_mx_puppet_discord_sqlite_database_path_local: "{{ matrix_mx_puppet_discord_data_path }}/database.db"
+matrix_mx_puppet_discord_sqlite_database_path_in_container: "/data/database.db"
+
 matrix_mx_puppet_discord_database_username: matrix_mx_puppet_discord
 matrix_mx_puppet_discord_database_password: ~
 matrix_mx_puppet_discord_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_discord_database_port: 5432
-matrix_mx_puppet_discord_database_name: matrix_mx_puppet_discord
-matrix_mx_puppet_discord_database_file: /data/database.db
-matrix_mx_puppet_discord_database_connString: >-2
-  {%- if matrix_mx_puppet_discord_database_engine == 'postgres' -%}
-  postgresql://{{ matrix_mx_puppet_discord_database_username }}:{{ matrix_mx_puppet_discord_database_password }}@{{ matrix_mx_puppet_discord_database_hostname }}:{{ matrix_mx_puppet_discord_database_port }}/{{ matrix_mx_puppet_discord_database_name }}?sslmode=disable
-  {%- elif matrix_mx_puppet_discord_database_engine == 'sqlite' -%}
-  {{ matrix_mx_puppet_discord_database_engine }}://{{ matrix_mx_puppet_discord_database_file }}
-  {%- endif -%}
+matrix_mx_puppet_discord_database_db_name: matrix_mx_puppet_discord
+
+matrix_mx_puppet_discord_database_connection_string: 'postgresql://{{ matrix_mx_puppet_discord_database_username }}:{{ matrix_mx_puppet_discord_database_password }}@{{ matrix_mx_puppet_discord_database_hostname }}:{{ matrix_mx_puppet_discord_database_port }}/{{ matrix_mx_puppet_discord_database_db_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
index 9022f309..08a199c4 100644
--- a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
@@ -25,6 +25,32 @@
       command: "mv {{ matrix_mx_puppet_discord_base_path }}/database.db {{ matrix_mx_puppet_discord_data_path }}/database.db"
   when: "matrix_mx_puppet_discord_stat_database.stat.exists"
 
+- set_fact:
+    matrix_mx_puppet_discord_requires_restart: false
+
+- block:
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_mx_puppet_discord_sqlite_database_path_local }}"
+      register: matrix_mx_puppet_discord_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_mx_puppet_discord_sqlite_database_path_local }}"
+              dst: "{{ matrix_mx_puppet_discord_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_mx_puppet_discord_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-mx-puppet-discord.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_mx_puppet_discord_requires_restart: true
+      when: "matrix_mx_puppet_discord_sqlite_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_mx_puppet_discord_database_engine == 'postgres'"
+
 - name: Ensure MX Puppet Discord image is pulled
   docker_image:
     name: "{{ matrix_mx_puppet_discord_docker_image }}"
diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml
index 8db47ed1..c253eda2 100644
--- a/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml
@@ -8,23 +8,3 @@
   with_items:
     - "matrix_mx_puppet_discord_appservice_token"
     - "matrix_mx_puppet_discord_homeserver_token"
-
-- block:
-  - name: Check if a SQLite database already exists
-    stat:
-      path: "{{ matrix_mx_puppet_discord_data_path }}/database.db"
-    register: matrix_mx_puppet_discord_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-2
-        matrix_mx_puppet_discord_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_discord_data_path }}/database.db.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_discord_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_discord_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_discord_database_connString'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_mx_puppet_discord_sqlite_database_path_local_stat_result.stat.exists"
-  when: "matrix_mx_puppet_discord_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
index 27f0c567..1f4548d8 100644
--- a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
@@ -103,14 +103,14 @@ database:
 {% if matrix_mx_puppet_discord_database_engine == 'sqlite' %}
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: /data/database.db
+  filename: {{ matrix_mx_puppet_discord_sqlite_database_path_in_container|to_json }}
 {% else %}
   # Use Postgres as a database backend
   # If set, will be used instead of SQLite3
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: {{ matrix_mx_puppet_discord_database_connString|to_json }}
+  connString: {{ matrix_mx_puppet_discord_database_connection_string|to_json }}
 {% endif %}
 
 logging:

From e64758c11951b7d2720431b139be0edc454085bc Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 13:24:53 +0200
Subject: [PATCH 120/179] Add missing restart task

Should have been part of 149872e00c5
---
 .../matrix-bridge-mx-puppet-discord/tasks/setup_install.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
index 08a199c4..9801bf82 100644
--- a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
@@ -119,3 +119,9 @@
   service:
     daemon_reload: yes
   when: "matrix_mx_puppet_discord_systemd_service_result.changed"
+
+- name: Ensure matrix-mx-puppet-discord.service restarted, if necessary
+  service:
+    name: "matrix-mx-puppet-discord.service"
+    state: restarted
+  when: "matrix_mx_puppet_discord_requires_restart|bool"

From 44c9f4daca05fca27aa1fe8399349a34a9837015 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 13:30:52 +0200
Subject: [PATCH 121/179] Add support for automatic (Postgres -> SQLite)
 migration to mx-puppet-instagram

---
 .../defaults/main.yml                         | 15 ++++-----
 .../tasks/setup_install.yml                   | 33 ++++++++++++++++++-
 .../tasks/validate_config.yml                 | 20 -----------
 .../templates/config.yaml.j2                  |  4 +--
 4 files changed, 41 insertions(+), 31 deletions(-)

diff --git a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
index 4c40bda4..61bf3dcf 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
@@ -47,18 +47,17 @@ matrix_mx_puppet_instagram_homeserver_token: ''
 matrix_mx_puppet_instagram_login_shared_secret: ''
 
 matrix_mx_puppet_instagram_database_engine: sqlite
+
+matrix_mx_puppet_instagram_sqlite_database_path_local: "{{ matrix_mx_puppet_instagram_data_path }}/database.db"
+matrix_mx_puppet_instagram_sqlite_database_path_in_container: "/data/database.db"
+
 matrix_mx_puppet_instagram_database_username: matrix_mx_puppet_instagram
 matrix_mx_puppet_instagram_database_password: ~
 matrix_mx_puppet_instagram_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_instagram_database_port: 5432
-matrix_mx_puppet_instagram_database_name: matrix_mx_puppet_instagram
-matrix_mx_puppet_instagram_database_file: /data/database.db
-matrix_mx_puppet_instagram_database_connString: >-2
-  {%- if matrix_mx_puppet_instagram_database_engine == 'postgres' -%}
-  postgresql://{{ matrix_mx_puppet_instagram_database_username }}:{{ matrix_mx_puppet_instagram_database_password }}@{{ matrix_mx_puppet_instagram_database_hostname }}:{{ matrix_mx_puppet_instagram_database_port }}/{{ matrix_mx_puppet_instagram_database_name }}?sslmode=disable
-  {%- elif matrix_mx_puppet_instagram_database_engine == 'sqlite' -%}
-  {{ matrix_mx_puppet_instagram_database_engine }}://{{ matrix_mx_puppet_instagram_database_file }}
-  {%- endif -%}
+matrix_mx_puppet_instagram_database_db_name: matrix_mx_puppet_instagram
+
+matrix_mx_puppet_instagram_database_connection_string: 'postgresql://{{ matrix_mx_puppet_instagram_database_username }}:{{ matrix_mx_puppet_instagram_database_password }}@{{ matrix_mx_puppet_instagram_database_hostname }}:{{ matrix_mx_puppet_instagram_database_port }}/{{ matrix_mx_puppet_instagram_database_db_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml
index 78bd78c1..76bbd629 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml
@@ -8,6 +8,33 @@
       The matrix-bridge-mx-puppet-instagram role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+
+- set_fact:
+    matrix_mx_puppet_instagram_requires_restart: false
+
+- block:
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_mx_puppet_instagram_sqlite_database_path_local }}"
+      register: matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_mx_puppet_instagram_sqlite_database_path_local }}"
+              dst: "{{ matrix_mx_puppet_instagram_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_mx_puppet_instagram_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-mx-puppet-instagram.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_mx_puppet_instagram_requires_restart: true
+      when: "matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_mx_puppet_instagram_database_engine == 'postgres'"
+
 - name: Ensure mx-puppet-instagram image is pulled
   docker_image:
     name: "{{ matrix_mx_puppet_instagram_docker_image }}"
@@ -77,4 +104,8 @@
     daemon_reload: yes
   when: "matrix_mx_puppet_instagram_systemd_service_result.changed"
 
-
+- name: Ensure matrix-mx-puppet-instagram.service restarted, if necessary
+  service:
+    name: "matrix-mx-puppet-instagram.service"
+    state: restarted
+  when: "matrix_mx_puppet_instagram_requires_restart|bool"
diff --git a/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml
index c2115cbb..b6d9d994 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml
@@ -8,23 +8,3 @@
   with_items:
     - "matrix_mx_puppet_instagram_appservice_token"
     - "matrix_mx_puppet_instagram_homeserver_token"
-
-- block:
-  - name: Check if an SQLite database already exists
-    stat:
-      path: "{{ matrix_mx_puppet_instagram_data_path }}/database.db"
-    register: matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-
-        matrix_mx_puppet_instagram_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_instagram_data_path }}/database.db.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_instagram_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_instagram_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_instagram_database_connString'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result.stat.exists"
-  when: "matrix_mx_puppet_instagram_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2
index d89c5bdb..b830da2b 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2
@@ -50,11 +50,11 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: {{ matrix_mx_puppet_instagram_database_connString | to_json }}
+  connString: {{ matrix_mx_puppet_instagram_database_connection_string|to_json }}
 {% else %}
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: {{ matrix_mx_puppet_instagram_database_file }}
+  filename: {{ matrix_mx_puppet_instagram_sqlite_database_path_in_container|to_json }}
 {% endif %}
 
 logging:

From d135cd9cd3117ca659459db626d8ed942d474325 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 13:44:33 +0200
Subject: [PATCH 122/179] Ensure mx-puppet-discord directories are created
 before attempting migration

Our old (base-path -> data-path) SQLite migration can't work otherwise.

It's probably not necessary to keep it anymore, but since we still do,
at least we should take care to ensure it works.
---
 .../tasks/setup_install.yml                   | 28 +++++++++----------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
index 9801bf82..3e3b2f94 100644
--- a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
@@ -8,6 +8,20 @@
       The matrix-bridge-mx-puppet-discord role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+- name: Ensure MX Puppet Discord paths exist
+  file:
+    path: "{{ item.path }}"
+    state: directory
+    mode: 0750
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+  with_items:
+    - { path: "{{ matrix_mx_puppet_discord_base_path }}", when: true }
+    - { path: "{{ matrix_mx_puppet_discord_config_path }}", when: true }
+    - { path: "{{ matrix_mx_puppet_discord_data_path }}", when: true }
+    - { path: "{{ matrix_mx_puppet_discord_docker_src_files_path }}", when: "{{ matrix_mx_puppet_discord_container_image_self_build }}" }
+  when: matrix_mx_puppet_discord_enabled|bool and item.when|bool
+
 - name: Check if an old database file already exists
   stat:
     path: "{{ matrix_mx_puppet_discord_base_path }}/database.db"
@@ -59,20 +73,6 @@
     force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_discord_docker_image_force_pull }}"
   when: matrix_mx_puppet_discord_enabled|bool and not matrix_mx_puppet_discord_container_image_self_build
 
-- name: Ensure MX Puppet Discord paths exist
-  file:
-    path: "{{ item.path }}"
-    state: directory
-    mode: 0750
-    owner: "{{ matrix_user_username }}"
-    group: "{{ matrix_user_groupname }}"
-  with_items:
-    - { path: "{{ matrix_mx_puppet_discord_base_path }}", when: true }
-    - { path: "{{ matrix_mx_puppet_discord_config_path }}", when: true }
-    - { path: "{{ matrix_mx_puppet_discord_data_path }}", when: true }
-    - { path: "{{ matrix_mx_puppet_discord_docker_src_files_path }}", when: "{{ matrix_mx_puppet_discord_container_image_self_build }}" }
-  when: matrix_mx_puppet_discord_enabled|bool and item.when|bool
-
 - name: Ensure MX Puppet Discord repository is present on self build
   git:
     repo: "{{ matrix_mx_puppet_discord_container_image_self_build_repo }}"

From 10e0fa17adffde2b79048d597828f8beed00d05c Mon Sep 17 00:00:00 2001
From: Dan Arnfield <dan@arnfield.com>
Date: Tue, 22 Dec 2020 08:23:37 -0600
Subject: [PATCH 123/179] Update nginx (1.19.5 -> 1.19.6)

---
 roles/matrix-nginx-proxy/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml
index b18bedde..61653db4 100644
--- a/roles/matrix-nginx-proxy/defaults/main.yml
+++ b/roles/matrix-nginx-proxy/defaults/main.yml
@@ -3,7 +3,7 @@ matrix_nginx_proxy_enabled: true
 # We use an official nginx image, which we fix-up to run unprivileged.
 # An alternative would be an `nginxinc/nginx-unprivileged` image, but
 # that is frequently out of date.
-matrix_nginx_proxy_docker_image: "docker.io/nginx:1.19.5-alpine"
+matrix_nginx_proxy_docker_image: "docker.io/nginx:1.19.6-alpine"
 matrix_nginx_proxy_docker_image_force_pull: "{{ matrix_nginx_proxy_docker_image.endswith(':latest') }}"
 
 matrix_nginx_proxy_base_path: "{{ matrix_base_data_path }}/nginx-proxy"

From c3b63c6c97e99c81af3fe5fda48a72979dc8b5af Mon Sep 17 00:00:00 2001
From: Dan Arnfield <dan@arnfield.com>
Date: Tue, 22 Dec 2020 08:29:37 -0600
Subject: [PATCH 124/179] Update element-web (1.7.15 -> 1.7.16)

---
 roles/matrix-client-element/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml
index 8b032ac2..d0297193 100644
--- a/roles/matrix-client-element/defaults/main.yml
+++ b/roles/matrix-client-element/defaults/main.yml
@@ -3,7 +3,7 @@ matrix_client_element_enabled: true
 matrix_client_element_container_image_self_build: false
 matrix_client_element_container_image_self_build_repo: "https://github.com/vector-im/riot-web.git"
 
-matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:v1.7.15"
+matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:v1.7.16"
 matrix_client_element_docker_image_name_prefix: "{{ 'localhost/' if matrix_client_element_container_image_self_build else 'docker.io/' }}"
 matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}"
 

From e49eb078a25bab1e320ac6d5c56f58461f8b19ad Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 16:29:47 +0200
Subject: [PATCH 125/179] Add support for automatic (Postgres -> SQLite)
 migration to mx-puppet-skype

---
 .../defaults/main.yml                         | 15 ++--
 .../tasks/setup_install.yml                   | 82 +++++++++++++------
 .../tasks/validate_config.yml                 | 20 -----
 .../templates/config.yaml.j2                  |  4 +-
 4 files changed, 66 insertions(+), 55 deletions(-)

diff --git a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
index db0f4ede..f3e17a59 100644
--- a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
@@ -55,18 +55,17 @@ matrix_mx_puppet_skype_login_shared_secret: ''
 
 # Database configuration, role default is `sqlite` but playbook default is `postgres`
 matrix_mx_puppet_skype_database_engine: sqlite
+
+matrix_mx_puppet_skype_sqlite_database_path_local: "{{ matrix_mx_puppet_skype_data_path }}/database.db"
+matrix_mx_puppet_skype_sqlite_database_path_in_container: "/data/database.db"
+
 matrix_mx_puppet_skype_database_username: matrix_mx_puppet_skype
 matrix_mx_puppet_skype_database_password: ~
 matrix_mx_puppet_skype_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_skype_database_port: 5432
-matrix_mx_puppet_skype_database_name: matrix_mx_puppet_skype
-matrix_mx_puppet_skype_database_file: /data/database.db
-matrix_mx_puppet_skype_database_connString: >-2
-  {%- if matrix_mx_puppet_skype_database_engine == 'postgres' -%}
-  postgresql://{{ matrix_mx_puppet_skype_database_username }}:{{ matrix_mx_puppet_skype_database_password }}@{{ matrix_mx_puppet_skype_database_hostname }}:{{ matrix_mx_puppet_skype_database_port }}/{{ matrix_mx_puppet_skype_database_name }}?sslmode=disable
-  {%- elif matrix_mx_puppet_skype_database_engine == 'sqlite' -%}
-  {{ matrix_mx_puppet_skype_database_engine }}://{{ matrix_mx_puppet_skype_database_file }}
-  {%- endif -%}
+matrix_mx_puppet_skype_database_db_name: matrix_mx_puppet_skype
+
+matrix_mx_puppet_skype_database_connection_string: 'postgresql://{{ matrix_mx_puppet_skype_database_username }}:{{ matrix_mx_puppet_skype_database_password }}@{{ matrix_mx_puppet_skype_database_hostname }}:{{ matrix_mx_puppet_skype_database_port }}/{{ matrix_mx_puppet_skype_database_db_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml
index fb5185ed..9289a793 100644
--- a/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml
@@ -8,14 +8,6 @@
       The matrix-bridge-mx-puppet-skype role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
-- name: Ensure MX Puppet Skype image is pulled
-  docker_image:
-    name: "{{ matrix_mx_puppet_skype_docker_image }}"
-    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
-    force_source: "{{ matrix_mx_puppet_skype_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
-    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_skype_docker_image_force_pull }}"
-  when: matrix_mx_puppet_skype_enabled|bool and not matrix_mx_puppet_skype_container_image_self_build
-
 - name: Ensure MX Puppet Skype paths exist
   file:
     path: "{{ item.path }}"
@@ -30,6 +22,57 @@
     - { path: "{{ matrix_mx_puppet_skype_docker_src_files_path }}", when: "{{ matrix_mx_puppet_skype_container_image_self_build }}" }
   when: matrix_mx_puppet_skype_enabled|bool and item.when|bool
 
+- name: Check if an old database file already exists
+  stat:
+    path: "{{ matrix_mx_puppet_skype_base_path }}/database.db"
+  register: matrix_mx_puppet_skype_stat_database
+
+- name: (Data relocation) Ensure matrix-mx-puppet-skype.service is stopped
+  service:
+    name: matrix-mx-puppet-skype
+    state: stopped
+    daemon_reload: yes
+  failed_when: false
+  when: "matrix_mx_puppet_skype_stat_database.stat.exists"
+
+- name: (Data relocation) Move mx-puppet-skype database file to ./data directory
+  command: "mv {{ matrix_mx_puppet_skype_base_path }}/database.db {{ matrix_mx_puppet_skype_data_path }}/database.db"
+  when: "matrix_mx_puppet_skype_stat_database.stat.exists"
+
+- set_fact:
+    matrix_mx_puppet_skype_requires_restart: false
+
+- block:
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_mx_puppet_skype_sqlite_database_path_local }}"
+      register: matrix_mx_puppet_skype_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_mx_puppet_skype_sqlite_database_path_local }}"
+              dst: "{{ matrix_mx_puppet_skype_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_mx_puppet_skype_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-mx-puppet-skype.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_mx_puppet_skype_requires_restart: true
+      when: "matrix_mx_puppet_skype_sqlite_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_mx_puppet_skype_database_engine == 'postgres'"
+
+- name: Ensure MX Puppet Skype image is pulled
+  docker_image:
+    name: "{{ matrix_mx_puppet_skype_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_mx_puppet_skype_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_skype_docker_image_force_pull }}"
+  when: matrix_mx_puppet_skype_enabled|bool and not matrix_mx_puppet_skype_container_image_self_build
+
 - name: Ensure MX Puppet Skype repository is present on self build
   git:
     repo: "{{ matrix_mx_puppet_skype_container_image_self_build_repo }}"
@@ -49,23 +92,6 @@
       pull: yes
   when: "matrix_mx_puppet_skype_enabled|bool and matrix_mx_puppet_skype_container_image_self_build|bool"
 
-- name: Check if an old database file already exists
-  stat:
-    path: "{{ matrix_mx_puppet_skype_base_path }}/database.db"
-  register: matrix_mx_puppet_skype_stat_database
-
-- name: (Data relocation) Ensure matrix-mx-puppet-skype.service is stopped
-  service:
-    name: matrix-mx-puppet-skype
-    state: stopped
-    daemon_reload: yes
-  failed_when: false
-  when: "matrix_mx_puppet_skype_stat_database.stat.exists"
-
-- name: (Data relocation) Move mx-puppet-skype database file to ./data directory
-  command: "mv {{ matrix_mx_puppet_skype_base_path }}/database.db {{ matrix_mx_puppet_skype_data_path }}/database.db"
-  when: "matrix_mx_puppet_skype_stat_database.stat.exists"
-
 - name: Ensure mx-puppet-skype config.yaml installed
   copy:
     content: "{{ matrix_mx_puppet_skype_configuration|to_nice_yaml }}"
@@ -93,3 +119,9 @@
   service:
     daemon_reload: yes
   when: "matrix_mx_puppet_skype_systemd_service_result.changed"
+
+- name: Ensure matrix-mx-puppet-skype.service restarted, if necessary
+  service:
+    name: "matrix-mx-puppet-skype.service"
+    state: restarted
+  when: "matrix_mx_puppet_skype_requires_restart|bool"
diff --git a/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml
index c7100e51..7ed433b1 100644
--- a/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml
@@ -8,23 +8,3 @@
   with_items:
     - "matrix_mx_puppet_skype_appservice_token"
     - "matrix_mx_puppet_skype_homeserver_token"
-
-- block:
-  - name: Check if an SQLite database already exists
-    stat:
-      path: "{{ matrix_mx_puppet_skype_data_path }}/database.db"
-    register: matrix_mx_puppet_skype_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-
-        matrix_mx_puppet_skype_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_skype_data_path }}/database.db.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_skype_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_skype_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_skype_database_connString'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_mx_puppet_skype_sqlite_database_path_local_stat_result.stat.exists"
-  when: "matrix_mx_puppet_skype_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2
index a32523b1..d41d3a23 100644
--- a/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2
@@ -74,11 +74,11 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: {{ matrix_mx_puppet_skype_database_connString | to_json }}
+  connString: {{ matrix_mx_puppet_skype_database_connection_string|to_json }}
 {% else %}
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: {{ matrix_mx_puppet_skype_database_file }}
+  filename: {{ matrix_mx_puppet_skype_sqlite_database_path_in_container|to_json }}
 {% endif %}
 
 provisioning:

From 262a25f997a032826b0348f3d966bcac55b9959e Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 16:39:21 +0200
Subject: [PATCH 126/179] Add support for automatic (Postgres -> SQLite)
 migration to mx-puppet-slack

---
 .../defaults/main.yml                         | 15 ++--
 .../tasks/setup_install.yml                   | 74 +++++++++++++------
 .../tasks/validate_config.yml                 | 20 -----
 .../templates/config.yaml.j2                  |  4 +-
 4 files changed, 62 insertions(+), 51 deletions(-)

diff --git a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
index 2ed72611..9aad602a 100644
--- a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
@@ -59,18 +59,17 @@ matrix_mx_puppet_slack_login_shared_secret: ''
 
 # Database configuration, role uses 'sqlite' per default but playbook sets up postgres by default
 matrix_mx_puppet_slack_database_engine: sqlite
+
+matrix_mx_puppet_slack_sqlite_database_path_local: "{{ matrix_mx_puppet_slack_data_path }}/database.db"
+matrix_mx_puppet_slack_sqlite_database_path_in_container: "/data/database.db"
+
 matrix_mx_puppet_slack_database_username: matrix_mx_puppet_slack
 matrix_mx_puppet_slack_database_password: ~
 matrix_mx_puppet_slack_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_slack_database_port: 5432
-matrix_mx_puppet_slack_database_name: matrix_mx_puppet_slack
-matrix_mx_puppet_slack_database_file: /data/database.db
-matrix_mx_puppet_slack_database_connString: >-2
-  {%- if matrix_mx_puppet_slack_database_engine == 'postgres' -%}
-  postgresql://{{ matrix_mx_puppet_slack_database_username }}:{{ matrix_mx_puppet_slack_database_password }}@{{ matrix_mx_puppet_slack_database_hostname }}:{{ matrix_mx_puppet_slack_database_port }}/{{ matrix_mx_puppet_slack_database_name }}?sslmode=disable
-  {%- elif matrix_mx_puppet_slack_database_engine == 'sqlite' -%}
-  {{ matrix_mx_puppet_slack_database_engine }}://{{ matrix_mx_puppet_slack_database_file }}
-  {%- endif -%}
+matrix_mx_puppet_slack_database_db_name: matrix_mx_puppet_slack
+
+matrix_mx_puppet_slack_database_connection_string: 'postgresql://{{ matrix_mx_puppet_slack_database_username }}:{{ matrix_mx_puppet_slack_database_password }}@{{ matrix_mx_puppet_slack_database_hostname }}:{{ matrix_mx_puppet_slack_database_port }}/{{ matrix_mx_puppet_slack_database_db_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml
index 2e45ecf6..5d68435c 100644
--- a/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml
@@ -8,14 +8,6 @@
       The matrix-bridge-mx-puppet-slack role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
-- name: Ensure MX Puppet Slack image is pulled
-  docker_image:
-    name: "{{ matrix_mx_puppet_slack_docker_image }}"
-    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
-    force_source: "{{ matrix_mx_puppet_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
-    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_slack_docker_image_force_pull }}"
-  when: matrix_mx_puppet_slack_enabled|bool and not matrix_mx_puppet_slack_container_image_self_build
-
 - name: Ensure MX Puppet Slack paths exist
   file:
     path: "{{ item.path }}"
@@ -30,6 +22,53 @@
     - { path: "{{ matrix_mx_puppet_slack_docker_src_files_path }}", when: "{{ matrix_mx_puppet_slack_container_image_self_build }}" }
   when: matrix_mx_puppet_slack_enabled|bool and item.when|bool
 
+- name: Check if an old database file already exists
+  stat:
+    path: "{{ matrix_mx_puppet_slack_base_path }}/database.db"
+  register: matrix_mx_puppet_slack_stat_database
+
+- name: (Data relocation) Ensure matrix-mx-puppet-slack.service is stopped
+  service:
+    name: matrix-mx-puppet-slack
+    state: stopped
+    daemon_reload: yes
+  failed_when: false
+  when: "matrix_mx_puppet_slack_stat_database.stat.exists"
+
+- set_fact:
+    matrix_mx_puppet_slack_requires_restart: false
+
+- block:
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_mx_puppet_slack_sqlite_database_path_local }}"
+      register: matrix_mx_puppet_slack_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_mx_puppet_slack_sqlite_database_path_local }}"
+              dst: "{{ matrix_mx_puppet_slack_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_mx_puppet_slack_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-mx-puppet-slack.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_mx_puppet_slack_requires_restart: true
+      when: "matrix_mx_puppet_slack_sqlite_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_mx_puppet_slack_database_engine == 'postgres'"
+
+- name: Ensure MX Puppet Slack image is pulled
+  docker_image:
+    name: "{{ matrix_mx_puppet_slack_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_mx_puppet_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_slack_docker_image_force_pull }}"
+  when: matrix_mx_puppet_slack_enabled|bool and not matrix_mx_puppet_slack_container_image_self_build
+
 - name: Ensure MX Puppet Slack repository is present on self build
   git:
     repo: "{{ matrix_mx_puppet_slack_container_image_self_build_repo }}"
@@ -49,19 +88,6 @@
       pull: yes
   when: "matrix_mx_puppet_slack_enabled|bool and matrix_mx_puppet_slack_container_image_self_build"
 
-- name: Check if an old database file already exists
-  stat:
-    path: "{{ matrix_mx_puppet_slack_base_path }}/database.db"
-  register: matrix_mx_puppet_slack_stat_database
-
-- name: (Data relocation) Ensure matrix-mx-puppet-slack.service is stopped
-  service:
-    name: matrix-mx-puppet-slack
-    state: stopped
-    daemon_reload: yes
-  failed_when: false
-  when: "matrix_mx_puppet_slack_stat_database.stat.exists"
-
 - name: (Data relocation) Move mx-puppet-slack database file to ./data directory
   command: "mv {{ matrix_mx_puppet_slack_base_path }}/database.db {{ matrix_mx_puppet_slack_data_path }}/database.db"
   when: "matrix_mx_puppet_slack_stat_database.stat.exists"
@@ -93,3 +119,9 @@
   service:
     daemon_reload: yes
   when: "matrix_mx_puppet_slack_systemd_service_result.changed"
+
+- name: Ensure matrix-mx-puppet-slack.service restarted, if necessary
+  service:
+    name: "matrix-mx-puppet-slack.service"
+    state: restarted
+  when: "matrix_mx_puppet_slack_requires_restart|bool"
diff --git a/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml
index bc722ee6..3a0bca11 100644
--- a/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml
@@ -8,23 +8,3 @@
   with_items:
     - "matrix_mx_puppet_slack_appservice_token"
     - "matrix_mx_puppet_slack_homeserver_token"
-
-- block:
-  - name: Check if sqlite database already exists
-    stat:
-      path: "{{ matrix_mx_puppet_slack_data_path }}/database.db"
-    register: matrix_mx_puppet_slack_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-
-        matrix_mx_puppet_slack_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_slack_data_path }}/database.db.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_slack_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_slack_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_slack_database_connString'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_mx_puppet_slack_sqlite_database_path_local_stat_result.stat.exists"
-  when: "matrix_mx_puppet_slack_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2
index 58c5c11a..af6b5cb8 100644
--- a/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2
@@ -64,11 +64,11 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: {{ matrix_mx_puppet_slack_database_connString | to_json }}
+  connString: {{ matrix_mx_puppet_slack_database_connection_string|to_json }}
 {% else %}
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: {{ matrix_mx_puppet_slack_database_file }}
+  filename: {{ matrix_mx_puppet_slack_sqlite_database_path_in_container|to_json }}
 {% endif %}
 
 logging:

From 69cc2145d2c710d3874fe2ec12315266561e10a1 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 16:51:59 +0200
Subject: [PATCH 127/179] Add support for automatic (Postgres -> SQLite)
 migration to mx-puppet-steam

---
 group_vars/matrix_servers                     |  2 +-
 .../defaults/main.yml                         | 15 ++--
 .../tasks/setup_install.yml                   | 82 +++++++++++++------
 .../tasks/validate_config.yml                 | 22 -----
 .../templates/config.yaml.j2                  |  4 +-
 5 files changed, 67 insertions(+), 58 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index fda40efd..283d5b6e 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -583,7 +583,7 @@ matrix_mx_puppet_steam_homeserver_token: "{{ matrix_synapse_macaroon_secret_key
 matrix_mx_puppet_steam_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
-matrix_mx_puppet_stream_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_mx_puppet_steam_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
 matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_steam.db') | to_uuid }}"
 
 ######################################################################
diff --git a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
index 77f7804b..c1f6a385 100644
--- a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
@@ -56,18 +56,17 @@ matrix_mx_puppet_steam_homeserver_token: ''
 matrix_mx_puppet_steam_login_shared_secret: ''
 
 matrix_mx_puppet_steam_database_engine: sqlite
+
+matrix_mx_puppet_steam_sqlite_database_path_local: "{{ matrix_mx_puppet_steam_data_path }}/database.db"
+matrix_mx_puppet_steam_sqlite_database_path_in_container: "/data/database.db"
+
 matrix_mx_puppet_steam_database_username: matrix_mx_puppet_steam
 matrix_mx_puppet_steam_database_password: ~
 matrix_mx_puppet_steam_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_steam_database_port: 5432
-matrix_mx_puppet_steam_database_name: matrix_mx_puppet_steam
-matrix_mx_puppet_steam_database_file: /data/database.db
-matrix_mx_puppet_steam_database_connString: >-2
-  {%- if matrix_mx_puppet_steam_database_engine == 'postgres' -%}
-  postgresql://{{ matrix_mx_puppet_steam_database_username }}:{{ matrix_mx_puppet_steam_database_password }}@{{ matrix_mx_puppet_steam_database_hostname }}:{{ matrix_mx_puppet_steam_database_port }}/{{ matrix_mx_puppet_steam_database_name }}?sslmode=disable
-  {%- elif matrix_mx_puppet_steam_database_engine == 'sqlite' -%}
-  {{ matrix_mx_puppet_steam_database_engine }}://{{ matrix_mx_puppet_steam_database_file }}
-  {%- endif -%}
+matrix_mx_puppet_steam_database_db_name: matrix_mx_puppet_steam
+
+matrix_mx_puppet_steam_database_connection_string: 'postgresql://{{ matrix_mx_puppet_steam_database_username }}:{{ matrix_mx_puppet_steam_database_password }}@{{ matrix_mx_puppet_steam_database_hostname }}:{{ matrix_mx_puppet_steam_database_port }}/{{ matrix_mx_puppet_steam_database_db_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml
index 7b7f8211..71f6d889 100644
--- a/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml
@@ -8,14 +8,6 @@
       The matrix-bridge-mx-puppet-steam role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
-- name: Ensure MX Puppet Steam image is pulled
-  docker_image:
-    name: "{{ matrix_mx_puppet_steam_docker_image }}"
-    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
-    force_source: "{{ matrix_mx_puppet_steam_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
-    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_steam_docker_image_force_pull }}"
-  when: matrix_mx_puppet_steam_enabled|bool and not matrix_mx_puppet_steam_container_image_self_build
-
 - name: Ensure MX Puppet Steam paths exist
   file:
     path: "{{ item.path }}"
@@ -30,6 +22,57 @@
     - { path: "{{ matrix_mx_puppet_steam_docker_src_files_path }}", when: "{{ matrix_mx_puppet_steam_container_image_self_build }}" }
   when: matrix_mx_puppet_steam_enabled|bool and item.when|bool
 
+- name: Check if an old database file already exists
+  stat:
+    path: "{{ matrix_mx_puppet_steam_base_path }}/database.db"
+  register: matrix_mx_puppet_steam_stat_database
+
+- name: (Data relocation) Ensure matrix-mx-puppet-steam.service is stopped
+  service:
+    name: matrix-mx-puppet-steam
+    state: stopped
+    daemon_reload: yes
+  failed_when: false
+  when: "matrix_mx_puppet_steam_stat_database.stat.exists"
+
+- name: (Data relocation) Move mx-puppet-steam database file to ./data directory
+  command: "mv {{ matrix_mx_puppet_steam_base_path }}/database.db {{ matrix_mx_puppet_steam_data_path }}/database.db"
+  when: "matrix_mx_puppet_steam_stat_database.stat.exists"
+
+- set_fact:
+    matrix_mx_puppet_steam_requires_restart: false
+
+- block:
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_mx_puppet_steam_sqlite_database_path_local }}"
+      register: matrix_mx_puppet_steam_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_mx_puppet_steam_sqlite_database_path_local }}"
+              dst: "{{ matrix_mx_puppet_steam_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_mx_puppet_steam_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-mx-puppet-steam.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_mx_puppet_steam_requires_restart: true
+      when: "matrix_mx_puppet_steam_sqlite_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_mx_puppet_steam_database_engine == 'postgres'"
+
+- name: Ensure MX Puppet Steam image is pulled
+  docker_image:
+    name: "{{ matrix_mx_puppet_steam_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_mx_puppet_steam_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_steam_docker_image_force_pull }}"
+  when: matrix_mx_puppet_steam_enabled|bool and not matrix_mx_puppet_steam_container_image_self_build
+
 - name: Ensure MX Puppet Steam repository is present on self build
   git:
     repo: "{{ matrix_mx_puppet_steam_container_image_self_build_repo }}"
@@ -49,23 +92,6 @@
       pull: yes
   when: "matrix_mx_puppet_steam_enabled|bool and matrix_mx_puppet_steam_container_image_self_build"
 
-- name: Check if an old database file already exists
-  stat:
-    path: "{{ matrix_mx_puppet_steam_base_path }}/database.db"
-  register: matrix_mx_puppet_steam_stat_database
-
-- name: (Data relocation) Ensure matrix-mx-puppet-steam.service is stopped
-  service:
-    name: matrix-mx-puppet-steam
-    state: stopped
-    daemon_reload: yes
-  failed_when: false
-  when: "matrix_mx_puppet_steam_stat_database.stat.exists"
-
-- name: (Data relocation) Move mx-puppet-steam database file to ./data directory
-  command: "mv {{ matrix_mx_puppet_steam_base_path }}/database.db {{ matrix_mx_puppet_steam_data_path }}/database.db"
-  when: "matrix_mx_puppet_steam_stat_database.stat.exists"
-
 - name: Ensure mx-puppet-steam config.yaml installed
   copy:
     content: "{{ matrix_mx_puppet_steam_configuration|to_nice_yaml }}"
@@ -93,3 +119,9 @@
   service:
     daemon_reload: yes
   when: "matrix_mx_puppet_steam_systemd_service_result.changed"
+
+- name: Ensure matrix-mx-puppet-steam.service restarted, if necessary
+  service:
+    name: "matrix-mx-puppet-steam.service"
+    state: restarted
+  when: "matrix_mx_puppet_steam_requires_restart|bool"
diff --git a/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml
index 50ce15d9..a8bc6a42 100644
--- a/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml
@@ -8,25 +8,3 @@
   with_items:
     - "matrix_mx_puppet_steam_appservice_token"
     - "matrix_mx_puppet_steam_homeserver_token"
-
-- block:
-  - name: Check if an SQLite database already exists
-    stat:
-      path: "{{ matrix_mx_puppet_steam_data_path }}"
-    register: matrix_mx_puppet_steam_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-
-        matrix_mx_puppet_steam_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_steam_data_path }}/database.db.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_steam_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_steam_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_steam_database_connString'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_mx_puppet_steam_sqlite_database_path_local_stat_result.stat.exists"
-  when: "matrix_mx_puppet_steam_database_engine == 'postgres'"
-
-
diff --git a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2
index 8eda278d..149e08b6 100644
--- a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2
@@ -67,11 +67,11 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: {{ matrix_mx_puppet_steam_database_connString | to_json }}
+  connString: {{ matrix_mx_puppet_steam_database_connection_string|to_json }}
 {% else %}
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: {{ matrix_mx_puppet_steam_database_file }}
+  filename: {{ matrix_mx_puppet_steam_sqlite_database_path_in_container|to_json }}
 {% endif %}
 
 logging:

From ab6563ce4e9273cc4e45fbf1b50339c10aff9452 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 17:09:08 +0200
Subject: [PATCH 128/179] Add support for automatic (Postgres -> SQLite)
 migration to mx-puppet-twitter

---
 .../defaults/main.yml                         | 17 ++--
 .../tasks/setup_install.yml                   | 82 +++++++++++++------
 .../tasks/validate_config.yml                 | 22 -----
 .../templates/config.yaml.j2                  |  4 +-
 4 files changed, 67 insertions(+), 58 deletions(-)

diff --git a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
index 91aa264a..da7a9ae7 100644
--- a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
@@ -62,18 +62,17 @@ matrix_mx_puppet_twitter_login_shared_secret: ''
 
 # Database configuration
 matrix_mx_puppet_twitter_database_engine: sqlite
-matrix_mx_puppet_twitter_database_username: matrix:mx_puppet_twitter
+
+matrix_mx_puppet_twitter_sqlite_database_path_local: "{{ matrix_mx_puppet_twitter_data_path }}/database.db"
+matrix_mx_puppet_twitter_sqlite_database_path_in_container: "/data/database.db"
+
+matrix_mx_puppet_twitter_database_username: mx_puppet_twitter
 matrix_mx_puppet_twitter_database_password: ~
 matrix_mx_puppet_twitter_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_twitter_database_port: 5432
-matrix_mx_puppet_twitter_database_name: matrix_mx_puppet_twitter
-matrix_mx_puppet_twitter_database_file: /data/database.db
-matrix_mx_puppet_twitter_database_connString: >-2
-  {%- if matrix_mx_puppet_twitter_database_engine == 'postgres' -%}
-  postgresql://{{ matrix_mx_puppet_twitter_database_username }}:{{ matrix_mx_puppet_twitter_database_password }}@{{ matrix_mx_puppet_twitter_database_hostname }}:{{ matrix_mx_puppet_twitter_database_port }}/{{ matrix_mx_puppet_twitter_database_name }}?sslmode=disable
-  {%- elif matrix_mx_puppet_twitter_database_engine == 'sqlite' -%}
-  {{ matrix_mx_puppet_twitter_database_engine }}://{{ matrix_mx_puppet_twitter_database_file }}
-  {%- endif -%}
+matrix_mx_puppet_twitter_database_db_name: matrix_mx_puppet_twitter
+
+matrix_mx_puppet_twitter_database_connection_string: 'postgresql://{{ matrix_mx_puppet_twitter_database_username }}:{{ matrix_mx_puppet_twitter_database_password }}@{{ matrix_mx_puppet_twitter_database_hostname }}:{{ matrix_mx_puppet_twitter_database_port }}/{{ matrix_mx_puppet_twitter_database_db_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml
index 663f822c..3893981a 100644
--- a/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml
@@ -8,14 +8,6 @@
       The matrix-bridge-mx-puppet-twitter role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
-- name: Ensure MX Puppet Twitter image is pulled
-  docker_image:
-    name: "{{ matrix_mx_puppet_twitter_docker_image }}"
-    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
-    force_source: "{{ matrix_mx_puppet_twitter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
-    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_twitter_docker_image_force_pull }}"
-  when: matrix_mx_puppet_twitter_enabled|bool and not matrix_mx_puppet_twitter_container_image_self_build
-
 - name: Ensure MX Puppet Twitter paths exist
   file:
     path: "{{ item.path }}"
@@ -30,6 +22,57 @@
     - { path: "{{ matrix_mx_puppet_twitter_docker_src_files_path }}", when: "{{ matrix_mx_puppet_twitter_container_image_self_build }}" }
   when: matrix_mx_puppet_twitter_enabled|bool and item.when|bool
 
+- name: Check if an old database file already exists
+  stat:
+    path: "{{ matrix_mx_puppet_twitter_base_path }}/database.db"
+  register: matrix_mx_puppet_twitter_stat_database
+
+- name: (Data relocation) Ensure matrix-mx-puppet-twitter.service is stopped
+  service:
+    name: matrix-mx-puppet-twitter
+    state: stopped
+    daemon_reload: yes
+  failed_when: false
+  when: "matrix_mx_puppet_twitter_stat_database.stat.exists"
+
+- name: (Data relocation) Move mx-puppet-twitter database file to ./data directory
+  command: "mv {{ matrix_mx_puppet_twitter_base_path }}/database.db {{ matrix_mx_puppet_twitter_data_path }}/database.db"
+  when: "matrix_mx_puppet_twitter_stat_database.stat.exists"
+
+- set_fact:
+    matrix_mx_puppet_twitter_requires_restart: false
+
+- block:
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_mx_puppet_twitter_sqlite_database_path_local }}"
+      register: matrix_mx_puppet_twitter_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_mx_puppet_twitter_sqlite_database_path_local }}"
+              dst: "{{ matrix_mx_puppet_twitter_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_mx_puppet_twitter_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-mx-puppet-twitter.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_mx_puppet_twitter_requires_restart: true
+      when: "matrix_mx_puppet_twitter_sqlite_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_mx_puppet_twitter_database_engine == 'postgres'"
+
+- name: Ensure MX Puppet Twitter image is pulled
+  docker_image:
+    name: "{{ matrix_mx_puppet_twitter_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_mx_puppet_twitter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_twitter_docker_image_force_pull }}"
+  when: matrix_mx_puppet_twitter_enabled|bool and not matrix_mx_puppet_twitter_container_image_self_build
+
 - name: Ensure MX Puppet Twitter repository is present on self build
   git:
     repo: "{{ matrix_mx_puppet_twitter_container_image_self_build_repo }}"
@@ -49,23 +92,6 @@
       pull: yes
   when: "matrix_mx_puppet_twitter_enabled|bool and matrix_mx_puppet_twitter_container_image_self_build"
 
-- name: Check if an old database file already exists
-  stat:
-    path: "{{ matrix_mx_puppet_twitter_base_path }}/database.db"
-  register: matrix_mx_puppet_twitter_stat_database
-
-- name: (Data relocation) Ensure matrix-mx-puppet-twitter.service is stopped
-  service:
-    name: matrix-mx-puppet-twitter
-    state: stopped
-    daemon_reload: yes
-  failed_when: false
-  when: "matrix_mx_puppet_twitter_stat_database.stat.exists"
-
-- name: (Data relocation) Move mx-puppet-twitter database file to ./data directory
-  command: "mv {{ matrix_mx_puppet_twitter_base_path }}/database.db {{ matrix_mx_puppet_twitter_data_path }}/database.db"
-  when: "matrix_mx_puppet_twitter_stat_database.stat.exists"
-
 - name: Ensure mx-puppet-twitter config.yaml installed
   copy:
     content: "{{ matrix_mx_puppet_twitter_configuration|to_nice_yaml }}"
@@ -93,3 +119,9 @@
   service:
     daemon_reload: yes
   when: "matrix_mx_puppet_twitter_systemd_service_result.changed"
+
+- name: Ensure matrix-mx-puppet-twitter.service restarted, if necessary
+  service:
+    name: "matrix-mx-puppet-twitter.service"
+    state: restarted
+  when: "matrix_mx_puppet_twitter_requires_restart|bool"
diff --git a/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml
index 0b3bd8f8..d13a39e1 100644
--- a/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml
@@ -8,25 +8,3 @@
   with_items:
     - "matrix_mx_puppet_twitter_appservice_token"
     - "matrix_mx_puppet_twitter_homeserver_token"
-
-- block:
-  - name: Check if an SQLite database already exists
-    stat:
-      path: "{{ matrix_mx_puppet_twitter_sqlite_data_path }}/database.db"
-    register: matrix_mx_puppet_twitter_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-
-        matrix_mx_puppet_twitter_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_twitter_data_path }}/database.db.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_twitter_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_twitter_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_twitter_database_connString'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_mx_puppet_twitter_sqlite_database_path_local_stat_result.stat.exists"
-  when: "matrix_mx_puppet_twitter_database_engine == 'postgres'"
-
-
diff --git a/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2
index cecb171d..bdecf1dc 100644
--- a/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2
@@ -60,11 +60,11 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: {{ matrix_mx_puppet_twitter_database_connString | to_json }}
+  connString: {{ matrix_mx_puppet_twitter_database_connection_string|to_json }}
 {% else %}
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: {{ matrix_mx_puppet_twitter_database_file }}
+  filename: {{ matrix_mx_puppet_twitter_sqlite_database_path_in_container|to_json }}
 {% endif %}
 
 logging:

From 15f4cc924d30a62256cd30a0c73311223d74eac6 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 17:10:02 +0200
Subject: [PATCH 129/179] Rename variables (_database_db_name ->
 _database_name)

---
 group_vars/matrix_servers                     | 34 +++++++++----------
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  4 +--
 roles/matrix-dimension/defaults/main.yml      |  4 +--
 roles/matrix-ma1sd/defaults/main.yml          |  4 +--
 roles/matrix-ma1sd/templates/ma1sd.yaml.j2    |  2 +-
 roles/matrix-registration/defaults/main.yml   |  4 +--
 .../tasks/setup_install.yml                   |  2 +-
 18 files changed, 49 insertions(+), 49 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 283d5b6e..21846592 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -999,55 +999,55 @@ matrix_postgres_db_name: "homeserver"
 matrix_postgres_additional_databases: |
   {{
     ([{
-      'name': matrix_ma1sd_database_db_name,
+      'name': matrix_ma1sd_database_name,
       'username': matrix_ma1sd_database_username,
       'password': matrix_ma1sd_database_password,
     }] if (matrix_ma1sd_enabled and matrix_ma1sd_database_engine == 'postgres' and matrix_ma1sd_database_hostname == 'matrix-postgres') else [])
     +
     ([{
-      'name': matrix_bot_matrix_reminder_bot_database_db_name,
+      'name': matrix_bot_matrix_reminder_bot_database_name,
       'username': matrix_bot_matrix_reminder_bot_database_username,
       'password': matrix_bot_matrix_reminder_bot_database_password,
     }] if (matrix_bot_matrix_reminder_bot_enabled and matrix_bot_matrix_reminder_bot_database_engine == 'postgres' and matrix_bot_matrix_reminder_bot_database_hostname == 'matrix-postgres') else [])
     +
     ([{
-      'name': matrix_registration_database_db_name,
+      'name': matrix_registration_database_name,
       'username': matrix_registration_database_username,
       'password': matrix_registration_database_password,
     }] if (matrix_registration_enabled and matrix_registration_database_engine == 'postgres' and matrix_registration_database_hostname == 'matrix-postgres') else [])
     +
     ([{
-      'name': matrix_appservice_discord_database_db_name,
+      'name': matrix_appservice_discord_database_name,
       'username': matrix_appservice_discord_database_username,
       'password': matrix_appservice_discord_database_password,
     }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_appservice_slack_database_db_name,
+      'name': matrix_appservice_slack_database_name,
       'username': matrix_appservice_slack_database_username,
       'password': matrix_appservice_slack_database_password,
     }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_appservice_irc_database_db_name,
+      'name': matrix_appservice_irc_database_name,
       'username': matrix_appservice_irc_database_username,
       'password': matrix_appservice_irc_database_password,
     }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_facebook_database_db_name,
+      'name': matrix_mautrix_facebook_database_name,
       'username': matrix_mautrix_facebook_database_username,
       'password': matrix_mautrix_facebook_database_password,
     }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_hangouts_database_db_name,
+      'name': matrix_mautrix_hangouts_database_name,
       'username': matrix_mautrix_hangouts_database_username,
       'password': matrix_mautrix_hangouts_database_password,
     }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_telegram_database_db_name,
+      'name': matrix_mautrix_telegram_database_name,
       'username': matrix_mautrix_telegram_database_username,
       'password': matrix_mautrix_telegram_database_password,
     }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_mautrix_telegram_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_whatsapp_database_db_name,
+      'name': matrix_mautrix_whatsapp_database_name,
       'username': matrix_mautrix_whatsapp_database_username,
       'password': matrix_mautrix_whatsapp_database_password,
     }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres') else [])
@@ -1057,36 +1057,36 @@ matrix_postgres_additional_databases: |
       'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid,
     }] if matrix_sms_bridge_enabled else [])
     + ([{
-      'name': matrix_mx_puppet_skype_database_db_name,
+      'name': matrix_mx_puppet_skype_database_name,
       'username': matrix_mx_puppet_skype_database_username,
       'password': matrix_mx_puppet_skype_database_password,
     }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mx_puppet_slack_database_db_name,
+      'name': matrix_mx_puppet_slack_database_name,
       'username': matrix_mx_puppet_slack_database_username,
       'password': matrix_mx_puppet_slack_database_password,
     }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_twitter_database_db_name,
+      'name': matrix_mx_puppet_twitter_database_name,
       'username': matrix_mx_puppet_twitter_database_username,
       'password': matrix_mx_puppet_twitter_database_password,
     }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_instagram_database_db_name,
+      'name': matrix_mx_puppet_instagram_database_name,
       'username': matrix_mx_puppet_instagram_database_username,
       'password': matrix_mx_puppet_instagram_database_password,
     }] if (matrix_mx_puppet_instagram_enabled
            and matrix_mx_puppet_instagram_database_engine == 'postgres'
            and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_discord_database_db_name,
+      'name': matrix_mx_puppet_discord_database_name,
       'username': matrix_mx_puppet_discord_database_username,
       'password': matrix_mx_puppet_discord_database_password,
     }] if (matrix_mx_puppet_discord_enabled
            and matrix_mx_puppet_discord_database_engine == 'postgres'
            and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_steam_database_db_name,
+      'name': matrix_mx_puppet_steam_database_name,
       'username': matrix_mx_puppet_steam_database_username,
       'password': matrix_mx_puppet_steam_database_password,
     }] if (matrix_mx_puppet_steam_enabled
@@ -1094,7 +1094,7 @@ matrix_postgres_additional_databases: |
            and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else [])
      +
     ([{
-      'name': matrix_dimension_database_db_name,
+      'name': matrix_dimension_database_name,
       'username': matrix_dimension_database_username,
       'password': matrix_dimension_database_password,
     }] if (matrix_dimension_enabled and matrix_dimension_database_engine == 'postgres' and matrix_dimension_database_hostname == 'matrix-postgres') else [])
diff --git a/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml b/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml
index 70e7b2d2..29bc8307 100644
--- a/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml
+++ b/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml
@@ -37,9 +37,9 @@ matrix_bot_matrix_reminder_bot_database_username: 'matrix_reminder_bot'
 matrix_bot_matrix_reminder_bot_database_password: 'some-password'
 matrix_bot_matrix_reminder_bot_database_hostname: 'matrix-postgres'
 matrix_bot_matrix_reminder_bot_database_port: 5432
-matrix_bot_matrix_reminder_bot_database_db_name: 'matrix_reminder_bot'
+matrix_bot_matrix_reminder_bot_database_name: 'matrix_reminder_bot'
 
-matrix_bot_matrix_reminder_bot_database_connection_string: 'postgres://{{ matrix_bot_matrix_reminder_bot_database_username }}:{{ matrix_bot_matrix_reminder_bot_database_password }}@{{ matrix_bot_matrix_reminder_bot_database_hostname }}:{{ matrix_bot_matrix_reminder_bot_database_port }}/{{ matrix_bot_matrix_reminder_bot_database_db_name }}'
+matrix_bot_matrix_reminder_bot_database_connection_string: 'postgres://{{ matrix_bot_matrix_reminder_bot_database_username }}:{{ matrix_bot_matrix_reminder_bot_database_password }}@{{ matrix_bot_matrix_reminder_bot_database_hostname }}:{{ matrix_bot_matrix_reminder_bot_database_port }}/{{ matrix_bot_matrix_reminder_bot_database_name }}'
 
 matrix_bot_matrix_reminder_bot_storage_database: "{{
 	{
diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml
index 8e453a15..c7cdddb6 100644
--- a/roles/matrix-bridge-appservice-discord/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml
@@ -57,12 +57,12 @@ matrix_appservice_discord_database_username: 'matrix_appservice_discord'
 matrix_appservice_discord_database_password: 'some-password'
 matrix_appservice_discord_database_hostname: 'matrix-postgres'
 matrix_appservice_discord_database_port: 5432
-matrix_appservice_discord_database_db_name: 'matrix_appservice_discord'
+matrix_appservice_discord_database_name: 'matrix_appservice_discord'
 
 # These 2 variables are what actually ends up in the bridge configuration.
 # It's best if you don't change them directly, but rather redefine the sub-variables that constitute them.
 matrix_appservice_discord_database_filename: "{{ matrix_appservice_discord_sqlite_database_path_in_container }}"
-matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_username }}:{{ matrix_appservice_discord_database_password }}@{{ matrix_appservice_discord_database_hostname }}:{{ matrix_appservice_discord_database_port }}/{{ matrix_appservice_discord_database_db_name }}'
+matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_username }}:{{ matrix_appservice_discord_database_password }}@{{ matrix_appservice_discord_database_hostname }}:{{ matrix_appservice_discord_database_port }}/{{ matrix_appservice_discord_database_name }}'
 
 
 # Tells whether the bot should make use of "Privileged Gateway Intents".
diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
index c02197a5..580934db 100644
--- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
@@ -49,9 +49,9 @@ matrix_mautrix_facebook_database_username: 'matrix_mautrix_facebook'
 matrix_mautrix_facebook_database_password: 'some-password'
 matrix_mautrix_facebook_database_hostname: 'matrix-postgres'
 matrix_mautrix_facebook_database_port: 5432
-matrix_mautrix_facebook_database_db_name: 'matrix_mautrix_facebook'
+matrix_mautrix_facebook_database_name: 'matrix_mautrix_facebook'
 
-matrix_mautrix_facebook_database_connection_string: 'postgresql://{{ matrix_mautrix_facebook_database_username }}:{{ matrix_mautrix_facebook_database_password }}@{{ matrix_mautrix_facebook_database_hostname }}:{{ matrix_mautrix_facebook_database_port }}/{{ matrix_mautrix_facebook_database_db_name }}'
+matrix_mautrix_facebook_database_connection_string: 'postgresql://{{ matrix_mautrix_facebook_database_username }}:{{ matrix_mautrix_facebook_database_password }}@{{ matrix_mautrix_facebook_database_hostname }}:{{ matrix_mautrix_facebook_database_port }}/{{ matrix_mautrix_facebook_database_name }}'
 
 matrix_mautrix_facebook_appservice_database: "{{
 	{
diff --git a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
index 5d38289b..984bec48 100644
--- a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
@@ -56,9 +56,9 @@ matrix_mautrix_hangouts_database_username: 'matrix_mautrix_hangouts'
 matrix_mautrix_hangouts_database_password: 'some-password'
 matrix_mautrix_hangouts_database_hostname: 'matrix-postgres'
 matrix_mautrix_hangouts_database_port: 5432
-matrix_mautrix_hangouts_database_db_name: 'matrix_mautrix_hangouts'
+matrix_mautrix_hangouts_database_name: 'matrix_mautrix_hangouts'
 
-matrix_mautrix_hangouts_database_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_database_username }}:{{ matrix_mautrix_hangouts_database_password }}@{{ matrix_mautrix_hangouts_database_hostname }}:{{ matrix_mautrix_hangouts_database_port }}/{{ matrix_mautrix_hangouts_database_db_name }}'
+matrix_mautrix_hangouts_database_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_database_username }}:{{ matrix_mautrix_hangouts_database_password }}@{{ matrix_mautrix_hangouts_database_hostname }}:{{ matrix_mautrix_hangouts_database_port }}/{{ matrix_mautrix_hangouts_database_name }}'
 
 matrix_mautrix_hangouts_appservice_database: "{{
 	{
diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
index fcdf95db..ceebd3ec 100644
--- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
@@ -60,9 +60,9 @@ matrix_mautrix_telegram_database_username: 'matrix_mautrix_telegram'
 matrix_mautrix_telegram_database_password: 'some-password'
 matrix_mautrix_telegram_database_hostname: 'matrix-postgres'
 matrix_mautrix_telegram_database_port: 5432
-matrix_mautrix_telegram_database_db_name: 'matrix_mautrix_telegram'
+matrix_mautrix_telegram_database_name: 'matrix_mautrix_telegram'
 
-matrix_mautrix_telegram_database_connection_string: 'postgresql://{{ matrix_mautrix_telegram_database_username }}:{{ matrix_mautrix_telegram_database_password }}@{{ matrix_mautrix_telegram_database_hostname }}:{{ matrix_mautrix_telegram_database_port }}/{{ matrix_mautrix_telegram_database_db_name }}'
+matrix_mautrix_telegram_database_connection_string: 'postgresql://{{ matrix_mautrix_telegram_database_username }}:{{ matrix_mautrix_telegram_database_password }}@{{ matrix_mautrix_telegram_database_hostname }}:{{ matrix_mautrix_telegram_database_port }}/{{ matrix_mautrix_telegram_database_name }}'
 
 matrix_mautrix_telegram_appservice_database: "{{
 	{
diff --git a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
index 7e198b36..0d4f9852 100644
--- a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
@@ -44,9 +44,9 @@ matrix_mautrix_whatsapp_database_username: 'matrix_mautrix_whatsapp'
 matrix_mautrix_whatsapp_database_password: 'some-password'
 matrix_mautrix_whatsapp_database_hostname: 'matrix-postgres'
 matrix_mautrix_whatsapp_database_port: 5432
-matrix_mautrix_whatsapp_database_db_name: 'matrix_mautrix_whatsapp'
+matrix_mautrix_whatsapp_database_name: 'matrix_mautrix_whatsapp'
 
-matrix_mautrix_whatsapp_database_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_database_username }}:{{ matrix_mautrix_whatsapp_database_password }}@{{ matrix_mautrix_whatsapp_database_hostname }}:{{ matrix_mautrix_whatsapp_database_port }}/{{ matrix_mautrix_whatsapp_database_db_name }}'
+matrix_mautrix_whatsapp_database_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_database_username }}:{{ matrix_mautrix_whatsapp_database_password }}@{{ matrix_mautrix_whatsapp_database_hostname }}:{{ matrix_mautrix_whatsapp_database_port }}/{{ matrix_mautrix_whatsapp_database_name }}'
 
 matrix_mautrix_whatsapp_appservice_database_type: "{{
 	{
diff --git a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
index ad86a241..97b20313 100644
--- a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
@@ -65,9 +65,9 @@ matrix_mx_puppet_discord_database_username: matrix_mx_puppet_discord
 matrix_mx_puppet_discord_database_password: ~
 matrix_mx_puppet_discord_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_discord_database_port: 5432
-matrix_mx_puppet_discord_database_db_name: matrix_mx_puppet_discord
+matrix_mx_puppet_discord_database_name: matrix_mx_puppet_discord
 
-matrix_mx_puppet_discord_database_connection_string: 'postgresql://{{ matrix_mx_puppet_discord_database_username }}:{{ matrix_mx_puppet_discord_database_password }}@{{ matrix_mx_puppet_discord_database_hostname }}:{{ matrix_mx_puppet_discord_database_port }}/{{ matrix_mx_puppet_discord_database_db_name }}?sslmode=disable'
+matrix_mx_puppet_discord_database_connection_string: 'postgresql://{{ matrix_mx_puppet_discord_database_username }}:{{ matrix_mx_puppet_discord_database_password }}@{{ matrix_mx_puppet_discord_database_hostname }}:{{ matrix_mx_puppet_discord_database_port }}/{{ matrix_mx_puppet_discord_database_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
index 61bf3dcf..cd08c010 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
@@ -55,9 +55,9 @@ matrix_mx_puppet_instagram_database_username: matrix_mx_puppet_instagram
 matrix_mx_puppet_instagram_database_password: ~
 matrix_mx_puppet_instagram_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_instagram_database_port: 5432
-matrix_mx_puppet_instagram_database_db_name: matrix_mx_puppet_instagram
+matrix_mx_puppet_instagram_database_name: matrix_mx_puppet_instagram
 
-matrix_mx_puppet_instagram_database_connection_string: 'postgresql://{{ matrix_mx_puppet_instagram_database_username }}:{{ matrix_mx_puppet_instagram_database_password }}@{{ matrix_mx_puppet_instagram_database_hostname }}:{{ matrix_mx_puppet_instagram_database_port }}/{{ matrix_mx_puppet_instagram_database_db_name }}?sslmode=disable'
+matrix_mx_puppet_instagram_database_connection_string: 'postgresql://{{ matrix_mx_puppet_instagram_database_username }}:{{ matrix_mx_puppet_instagram_database_password }}@{{ matrix_mx_puppet_instagram_database_hostname }}:{{ matrix_mx_puppet_instagram_database_port }}/{{ matrix_mx_puppet_instagram_database_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
index f3e17a59..83cd3dc5 100644
--- a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
@@ -63,9 +63,9 @@ matrix_mx_puppet_skype_database_username: matrix_mx_puppet_skype
 matrix_mx_puppet_skype_database_password: ~
 matrix_mx_puppet_skype_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_skype_database_port: 5432
-matrix_mx_puppet_skype_database_db_name: matrix_mx_puppet_skype
+matrix_mx_puppet_skype_database_name: matrix_mx_puppet_skype
 
-matrix_mx_puppet_skype_database_connection_string: 'postgresql://{{ matrix_mx_puppet_skype_database_username }}:{{ matrix_mx_puppet_skype_database_password }}@{{ matrix_mx_puppet_skype_database_hostname }}:{{ matrix_mx_puppet_skype_database_port }}/{{ matrix_mx_puppet_skype_database_db_name }}?sslmode=disable'
+matrix_mx_puppet_skype_database_connection_string: 'postgresql://{{ matrix_mx_puppet_skype_database_username }}:{{ matrix_mx_puppet_skype_database_password }}@{{ matrix_mx_puppet_skype_database_hostname }}:{{ matrix_mx_puppet_skype_database_port }}/{{ matrix_mx_puppet_skype_database_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
index 9aad602a..70b98ece 100644
--- a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
@@ -67,9 +67,9 @@ matrix_mx_puppet_slack_database_username: matrix_mx_puppet_slack
 matrix_mx_puppet_slack_database_password: ~
 matrix_mx_puppet_slack_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_slack_database_port: 5432
-matrix_mx_puppet_slack_database_db_name: matrix_mx_puppet_slack
+matrix_mx_puppet_slack_database_name: matrix_mx_puppet_slack
 
-matrix_mx_puppet_slack_database_connection_string: 'postgresql://{{ matrix_mx_puppet_slack_database_username }}:{{ matrix_mx_puppet_slack_database_password }}@{{ matrix_mx_puppet_slack_database_hostname }}:{{ matrix_mx_puppet_slack_database_port }}/{{ matrix_mx_puppet_slack_database_db_name }}?sslmode=disable'
+matrix_mx_puppet_slack_database_connection_string: 'postgresql://{{ matrix_mx_puppet_slack_database_username }}:{{ matrix_mx_puppet_slack_database_password }}@{{ matrix_mx_puppet_slack_database_hostname }}:{{ matrix_mx_puppet_slack_database_port }}/{{ matrix_mx_puppet_slack_database_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
index c1f6a385..15fa889f 100644
--- a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
@@ -64,9 +64,9 @@ matrix_mx_puppet_steam_database_username: matrix_mx_puppet_steam
 matrix_mx_puppet_steam_database_password: ~
 matrix_mx_puppet_steam_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_steam_database_port: 5432
-matrix_mx_puppet_steam_database_db_name: matrix_mx_puppet_steam
+matrix_mx_puppet_steam_database_name: matrix_mx_puppet_steam
 
-matrix_mx_puppet_steam_database_connection_string: 'postgresql://{{ matrix_mx_puppet_steam_database_username }}:{{ matrix_mx_puppet_steam_database_password }}@{{ matrix_mx_puppet_steam_database_hostname }}:{{ matrix_mx_puppet_steam_database_port }}/{{ matrix_mx_puppet_steam_database_db_name }}?sslmode=disable'
+matrix_mx_puppet_steam_database_connection_string: 'postgresql://{{ matrix_mx_puppet_steam_database_username }}:{{ matrix_mx_puppet_steam_database_password }}@{{ matrix_mx_puppet_steam_database_hostname }}:{{ matrix_mx_puppet_steam_database_port }}/{{ matrix_mx_puppet_steam_database_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
index da7a9ae7..28639fda 100644
--- a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
@@ -70,9 +70,9 @@ matrix_mx_puppet_twitter_database_username: mx_puppet_twitter
 matrix_mx_puppet_twitter_database_password: ~
 matrix_mx_puppet_twitter_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_twitter_database_port: 5432
-matrix_mx_puppet_twitter_database_db_name: matrix_mx_puppet_twitter
+matrix_mx_puppet_twitter_database_name: matrix_mx_puppet_twitter
 
-matrix_mx_puppet_twitter_database_connection_string: 'postgresql://{{ matrix_mx_puppet_twitter_database_username }}:{{ matrix_mx_puppet_twitter_database_password }}@{{ matrix_mx_puppet_twitter_database_hostname }}:{{ matrix_mx_puppet_twitter_database_port }}/{{ matrix_mx_puppet_twitter_database_db_name }}?sslmode=disable'
+matrix_mx_puppet_twitter_database_connection_string: 'postgresql://{{ matrix_mx_puppet_twitter_database_username }}:{{ matrix_mx_puppet_twitter_database_password }}@{{ matrix_mx_puppet_twitter_database_hostname }}:{{ matrix_mx_puppet_twitter_database_port }}/{{ matrix_mx_puppet_twitter_database_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-dimension/defaults/main.yml b/roles/matrix-dimension/defaults/main.yml
index a6a3bf5d..8a691946 100644
--- a/roles/matrix-dimension/defaults/main.yml
+++ b/roles/matrix-dimension/defaults/main.yml
@@ -57,9 +57,9 @@ matrix_dimension_database_username: 'matrix_dimension'
 matrix_dimension_database_password: 'some-password'
 matrix_dimension_database_hostname: 'matrix-postgres'
 matrix_dimension_database_port: 5432
-matrix_dimension_database_db_name: 'matrix_dimension'
+matrix_dimension_database_name: 'matrix_dimension'
 
-matrix_dimension_database_connection_string: 'postgres://{{ matrix_dimension_database_username }}:{{ matrix_dimension_database_password }}@{{ matrix_dimension_database_hostname }}:{{ matrix_dimension_database_port }}/{{ matrix_dimension_database_db_name }}'
+matrix_dimension_database_connection_string: 'postgres://{{ matrix_dimension_database_username }}:{{ matrix_dimension_database_password }}@{{ matrix_dimension_database_hostname }}:{{ matrix_dimension_database_port }}/{{ matrix_dimension_database_name }}'
 
 
 # Default Dimension configuration template which covers the generic use case.
diff --git a/roles/matrix-ma1sd/defaults/main.yml b/roles/matrix-ma1sd/defaults/main.yml
index 42e36e92..2932f3ed 100644
--- a/roles/matrix-ma1sd/defaults/main.yml
+++ b/roles/matrix-ma1sd/defaults/main.yml
@@ -56,9 +56,9 @@ matrix_ma1sd_database_username: 'matrix_ma1sd'
 matrix_ma1sd_database_password: 'some-password'
 matrix_ma1sd_database_hostname: 'matrix-postgres'
 matrix_ma1sd_database_port: 5432
-matrix_ma1sd_database_db_name: 'matrix_ma1sd'
+matrix_ma1sd_database_name: 'matrix_ma1sd'
 
-matrix_ma1sd_database_connection_string: 'postgresql://{{ matrix_ma1sd_database_username }}:{{ matrix_ma1sd_database_password }}@{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_db_name }}'
+matrix_ma1sd_database_connection_string: 'postgresql://{{ matrix_ma1sd_database_username }}:{{ matrix_ma1sd_database_password }}@{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_name }}'
 
 
 # ma1sd has serveral supported identity stores.
diff --git a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
index 43b4022d..9a426c47 100644
--- a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
+++ b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
@@ -20,7 +20,7 @@ storage:
     backend: postgresql
     provider:
       postgresql:
-        database: //{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_db_name }}
+        database: //{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_name }}
         username: {{ matrix_ma1sd_database_username|to_json }}
         password: {{ matrix_ma1sd_database_password|to_json }}
   {% endif %}
diff --git a/roles/matrix-registration/defaults/main.yml b/roles/matrix-registration/defaults/main.yml
index d85faf89..065e0c48 100644
--- a/roles/matrix-registration/defaults/main.yml
+++ b/roles/matrix-registration/defaults/main.yml
@@ -48,9 +48,9 @@ matrix_registration_database_username: 'matrix_registration'
 matrix_registration_database_password: 'some-password'
 matrix_registration_database_hostname: 'matrix-postgres'
 matrix_registration_database_port: 5432
-matrix_registration_database_db_name: 'matrix_registration'
+matrix_registration_database_name: 'matrix_registration'
 
-matrix_registration_database_connection_string: 'postgresql://{{ matrix_registration_database_username }}:{{ matrix_registration_database_password }}@{{ matrix_registration_database_hostname }}:{{ matrix_registration_database_port }}/{{ matrix_registration_database_db_name }}'
+matrix_registration_database_connection_string: 'postgresql://{{ matrix_registration_database_username }}:{{ matrix_registration_database_password }}@{{ matrix_registration_database_hostname }}:{{ matrix_registration_database_port }}/{{ matrix_registration_database_name }}'
 
 # For some reason, matrix-registraiton expects the `db` field to be like this: `sqlite:////data/db.sqlite3`.
 # (seems like one too many slashes, but..)
diff --git a/roles/matrix-registration/tasks/setup_install.yml b/roles/matrix-registration/tasks/setup_install.yml
index 708cb1df..9b6d1260 100644
--- a/roles/matrix-registration/tasks/setup_install.yml
+++ b/roles/matrix-registration/tasks/setup_install.yml
@@ -22,7 +22,7 @@
               # which makes matrix-registration choke on it later on when comparing dates.
               additional_psql_statements_list:
                 - ALTER TABLE tokens ALTER COLUMN ex_date TYPE TIMESTAMP WITHOUT TIME ZONE;
-              additional_psql_statements_db_name: "{{ matrix_registration_database_db_name }}"
+              additional_psql_statements_db_name: "{{ matrix_registration_database_name }}"
 
         - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
 

From 815a80c290d2527a74a26acb47e6f1fbbb3f000b Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 17:16:56 +0200
Subject: [PATCH 130/179] Fix alignment issues

---
 group_vars/matrix_servers | 53 ++++++++++++++++++++++-----------------
 1 file changed, 30 insertions(+), 23 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 21846592..318ebac7 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -1021,78 +1021,85 @@ matrix_postgres_additional_databases: |
       'username': matrix_appservice_discord_database_username,
       'password': matrix_appservice_discord_database_password,
     }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_database_hostname == 'matrix-postgres') else [])
-    + ([{
+    +
+    ([{
       'name': matrix_appservice_slack_database_name,
       'username': matrix_appservice_slack_database_username,
       'password': matrix_appservice_slack_database_password,
     }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres') else [])
-    + ([{
+    +
+    ([{
       'name': matrix_appservice_irc_database_name,
       'username': matrix_appservice_irc_database_username,
       'password': matrix_appservice_irc_database_password,
     }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres') else [])
-    + ([{
+    +
+    ([{
       'name': matrix_mautrix_facebook_database_name,
       'username': matrix_mautrix_facebook_database_username,
       'password': matrix_mautrix_facebook_database_password,
     }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_database_hostname == 'matrix-postgres') else [])
-    + ([{
+    +
+    ([{
       'name': matrix_mautrix_hangouts_database_name,
       'username': matrix_mautrix_hangouts_database_username,
       'password': matrix_mautrix_hangouts_database_password,
     }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres') else [])
-    + ([{
+    +
+    ([{
       'name': matrix_mautrix_telegram_database_name,
       'username': matrix_mautrix_telegram_database_username,
       'password': matrix_mautrix_telegram_database_password,
     }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_mautrix_telegram_database_hostname == 'matrix-postgres') else [])
-    + ([{
+    +
+    ([{
       'name': matrix_mautrix_whatsapp_database_name,
       'username': matrix_mautrix_whatsapp_database_username,
       'password': matrix_mautrix_whatsapp_database_password,
     }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres') else [])
-    + ([{
+    +
+    ([{
       'name': 'matrix_bridge_sms',
       'username': 'matrix_bridge_sms',
       'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid,
     }] if matrix_sms_bridge_enabled else [])
-    + ([{
+    +
+    ([{
       'name': matrix_mx_puppet_skype_database_name,
       'username': matrix_mx_puppet_skype_database_username,
       'password': matrix_mx_puppet_skype_database_password,
     }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres') else [])
-    + ([{
+    +
+    ([{
       'name': matrix_mx_puppet_slack_database_name,
       'username': matrix_mx_puppet_slack_database_username,
       'password': matrix_mx_puppet_slack_database_password,
     }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres') else [])
-     + ([{
+    +
+    ([{
       'name': matrix_mx_puppet_twitter_database_name,
       'username': matrix_mx_puppet_twitter_database_username,
       'password': matrix_mx_puppet_twitter_database_password,
     }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres') else [])
-     + ([{
+    +
+    ([{
       'name': matrix_mx_puppet_instagram_database_name,
       'username': matrix_mx_puppet_instagram_database_username,
       'password': matrix_mx_puppet_instagram_database_password,
-    }] if (matrix_mx_puppet_instagram_enabled
-           and matrix_mx_puppet_instagram_database_engine == 'postgres'
-           and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres') else [])
-     + ([{
+    }] if (matrix_mx_puppet_instagram_enabled and matrix_mx_puppet_instagram_database_engine == 'postgres' and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres') else [])
+    +
+    ([{
       'name': matrix_mx_puppet_discord_database_name,
       'username': matrix_mx_puppet_discord_database_username,
       'password': matrix_mx_puppet_discord_database_password,
-    }] if (matrix_mx_puppet_discord_enabled
-           and matrix_mx_puppet_discord_database_engine == 'postgres'
-           and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres') else [])
-     + ([{
+    }] if (matrix_mx_puppet_discord_enabled  and matrix_mx_puppet_discord_database_engine == 'postgres' and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres') else [])
+    +
+    ([{
       'name': matrix_mx_puppet_steam_database_name,
       'username': matrix_mx_puppet_steam_database_username,
       'password': matrix_mx_puppet_steam_database_password,
-    }] if (matrix_mx_puppet_steam_enabled
-           and matrix_mx_puppet_steam_database_engine == 'postgres'
-           and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else [])
-     +
+    }] if (matrix_mx_puppet_steam_enabled and matrix_mx_puppet_steam_database_engine == 'postgres' and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else [])
+    +
     ([{
       'name': matrix_dimension_database_name,
       'username': matrix_dimension_database_username,

From 715bdf2c6487c56be5ad17a4e7b80d94efa46ce2 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 19:32:43 +0200
Subject: [PATCH 131/179] Add support for automatic (nedb -> Postgres)
 migration to mx-appservice-irc

---
 group_vars/matrix_servers                     |  3 +-
 roles/matrix-base/defaults/main.yml           |  1 +
 .../defaults/main.yml                         | 17 +++--
 .../tasks/migrate_nedb_to_postgres.yml        | 53 ++++++++++++++++
 .../tasks/setup_install.yml                   | 63 ++++++++++++-------
 .../tasks/validate_config.yml                 | 20 ------
 .../templates/config.yaml.j2                  |  4 +-
 roles/matrix-postgres/tasks/import_nedb.yml   | 21 +------
 roles/matrix-postgres/tasks/main.yml          |  8 ---
 9 files changed, 109 insertions(+), 81 deletions(-)
 create mode 100644 roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 318ebac7..4ddadb43 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -181,8 +181,7 @@ matrix_appservice_irc_appservice_token: "{{ matrix_synapse_macaroon_secret_key |
 
 matrix_appservice_irc_homeserver_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'irc.hs.token') | to_uuid }}"
 
-# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
-matrix_appservice_irc_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_appservice_irc_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'nedb' }}"
 matrix_appservice_irc_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.irc.db') | to_uuid }}"
 
 
diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml
index e0522ba8..d0fdcdf8 100644
--- a/roles/matrix-base/defaults/main.yml
+++ b/roles/matrix-base/defaults/main.yml
@@ -106,6 +106,7 @@ matrix_docker_package_name: docker-ce
 run_postgres_import: true
 run_postgres_upgrade: true
 run_postgres_import_sqlite_db: true
+run_postgres_import_nedb: true
 run_postgres_vacuum: true
 run_synapse_register_user: true
 run_synapse_update_user_password: true
diff --git a/roles/matrix-bridge-appservice-irc/defaults/main.yml b/roles/matrix-bridge-appservice-irc/defaults/main.yml
index 5110fd85..0b671e76 100644
--- a/roles/matrix-bridge-appservice-irc/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-irc/defaults/main.yml
@@ -23,12 +23,17 @@ matrix_appservice_irc_database_hostname: 'matrix-postgres'
 matrix_appservice_irc_database_port: 5432
 matrix_appservice_irc_database_name: matrix_appservice_irc
 
-matrix_appservice_irc_database_connString: >-2
-  {%- if matrix_appservice_irc_database_engine == 'postgres' -%}
-  postgresql://{{ matrix_appservice_irc_database_username }}:{{ matrix_appservice_irc_database_password }}@{{ matrix_appservice_irc_database_hostname }}:{{ matrix_appservice_irc_database_port }}/{{ matrix_appservice_irc_database_name }}?sslmode=disable
-  {%- elif matrix_appservice_irc_database_engine == 'nedb' -%}
-  {{ matrix_appservice_irc_database_engine }}://{{ matrix_appservice_irc_database_file }}
-  {%- endif -%}
+# This is just the Postgres connection string, if Postgres is used.
+# Naming clashes with `matrix_appservice_irc_database_connectionString` somewhat.
+matrix_appservice_irc_database_connection_string: 'postgresql://{{ matrix_appservice_irc_database_username }}:{{ matrix_appservice_irc_database_password }}@{{ matrix_appservice_irc_database_hostname }}:{{ matrix_appservice_irc_database_port }}/{{ matrix_appservice_irc_database_name }}?sslmode=disable'
+
+# This is what actually goes into `database.connectionString` for the bridge.
+matrix_appservice_irc_database_connectionString: "{{
+	{
+		'nedb': 'nedb:///data',
+		'postgres': matrix_appservice_irc_database_connection_string,
+	}[matrix_appservice_irc_database_engine]
+}}"
 
 matrix_appservice_irc_ircService_servers: []
 
diff --git a/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml b/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml
new file mode 100644
index 00000000..bc6525ec
--- /dev/null
+++ b/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml
@@ -0,0 +1,53 @@
+- name: Fail if Postgres not enabled
+  fail:
+    msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot migrate."
+  when: "not matrix_postgres_enabled|bool"
+
+# Defaults
+
+- name: Set postgres_start_wait_time, if not provided
+  set_fact:
+    postgres_start_wait_time: 15
+  when: "postgres_start_wait_time|default('') == ''"
+
+# Actual import work
+
+- name: Ensure matrix-postgres is started
+  service:
+    name: matrix-postgres
+    state: started
+    daemon_reload: yes
+  register: matrix_postgres_service_start_result
+
+- name: Wait a bit, so that Postgres can start
+  wait_for:
+    timeout: "{{ postgres_start_wait_time }}"
+  delegate_to: 127.0.0.1
+  become: false
+  when: "matrix_postgres_service_start_result.changed|bool"
+
+- name: Ensure matrix-appservice-irc is stopped
+  service:
+    name: matrix-appservice-irc
+    state: stopped
+
+- name: Import appservice-irc NeDB database into Postgres
+  command:
+    cmd: >-
+      {{ matrix_host_command_docker }} run
+      --rm
+      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
+      --cap-drop=ALL
+      --network={{ matrix_docker_network }}
+      --mount type=bind,src={{ matrix_appservice_irc_data_path }},dst=/data
+      --entrypoint=/bin/sh
+      {{ matrix_appservice_irc_docker_image }}
+      -c
+      '/usr/local/bin/node /app/lib/scripts/migrate-db-to-pgres.js --dbdir /data --privateKey /data/passkey.pem --connectionString {{ matrix_appservice_irc_database_connection_string }}'
+
+- name: Archive NeDB database files
+  command:
+    cmd: "mv {{ matrix_appservice_irc_data_path }}/{{ item }} {{ matrix_appservice_irc_data_path }}/{{ item }}.backup"
+  with_items:
+    - rooms.db
+    - users.db
diff --git a/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml b/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml
index 5e313347..a748df96 100644
--- a/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml
+++ b/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml
@@ -1,12 +1,5 @@
 ---
 
-- name: Ensure Appservice IRC image is pulled
-  docker_image:
-    name: "{{ matrix_appservice_irc_docker_image }}"
-    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
-    force_source: "{{ matrix_appservice_irc_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
-    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_irc_docker_image_force_pull }}"
-
 - name: Ensure Appservice IRC paths exist
   file:
     path: "{{ item }}"
@@ -24,25 +17,49 @@
     path: "{{ matrix_appservice_irc_base_path }}/passkey.pem"
   register: matrix_appservice_irc_stat_passkey
 
-- name: (Data relocation) Ensure matrix-appservice-irc.service is stopped
-  service:
-    name: matrix-appservice-irc
-    state: stopped
-    daemon_reload: yes
-  failed_when: false
+- block:
+    - name: (Data relocation) Ensure matrix-appservice-irc.service is stopped
+      service:
+        name: matrix-appservice-irc
+        state: stopped
+        daemon_reload: yes
+      failed_when: false
+
+    - name: (Data relocation) Move AppService IRC passkey.pem file to ./data directory
+      command: "mv {{ matrix_appservice_irc_base_path }}/passkey.pem {{ matrix_appservice_irc_data_path }}/passkey.pem"
+
+    - name: (Data relocation) Move AppService IRC database files to ./data directory
+      command: "mv {{ matrix_appservice_irc_base_path }}/{{ item }} {{ matrix_appservice_irc_data_path }}/{{ item }}"
+      with_items:
+        - rooms.db
+        - users.db
+      failed_when: false
   when: "matrix_appservice_irc_stat_passkey.stat.exists"
 
-- name: (Data relocation) Move AppService IRC passkey.pem file to ./data directory
-  command: "mv {{ matrix_appservice_irc_base_path }}/passkey.pem {{ matrix_appservice_irc_data_path }}/passkey.pem"
-  when: "matrix_appservice_irc_stat_passkey.stat.exists"
 
-- name: (Data relocation) Move AppService IRC database files to ./data directory
-  command: "mv {{ matrix_appservice_irc_base_path }}/{{ item }} {{ matrix_appservice_irc_data_path }}/{{ item }}"
-  with_items:
-    - rooms.db
-    - users.db
-  failed_when: false
-  when: "matrix_appservice_irc_stat_passkey.stat.exists"
+- set_fact:
+    matrix_appservice_irc_requires_restart: false
+
+- block:
+    - name: Check if a nedb database already exists
+      stat:
+        path: "{{ matrix_appservice_irc_data_path }}/users.db"
+      register: matrix_appservice_irc_nedb_database_path_local_stat_result
+
+    - block:
+        - import_tasks: "{{ role_path }}/tasks/migrate_nedb_to_postgres.yml"
+
+        - set_fact:
+            matrix_appservice_irc_requires_restart: true
+      when: "matrix_appservice_irc_nedb_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_appservice_irc_database_engine == 'postgres'"
+
+- name: Ensure Appservice IRC image is pulled
+  docker_image:
+    name: "{{ matrix_appservice_irc_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_appservice_irc_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_irc_docker_image_force_pull }}"
 
 - name: Ensure Matrix Appservice IRC config installed
   copy:
diff --git a/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml b/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml
index 9b89a340..bd08427c 100644
--- a/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml
+++ b/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml
@@ -33,23 +33,3 @@
   when: "item.old in vars"
   with_items:
     - {'old': 'matrix_appservice_irc_container_expose_client_server_api_port', 'new': '<superseded by matrix_appservice_irc_container_http_host_bind_port>'}
-
-- block:
-  - name: Check if a neDB database already exists
-    stat:
-      path: "{{ matrix_appservice_irc_data_path }}/"
-    register: matrix_appservice_irc_nedb_stat_result
-
-  - name: Fail if an neDB database already exists when using Postgres
-    fail:
-      msg: >-2
-        matrix_appservice_irc_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing neDB database in {{ matrix_appservice_irc_data_path }}/.
-        It appears that you've been using this bridge with the neDB engine until now.
-        To continue using neDB, opt into it explicitly: add `matrix_appservice_irc_database_engine: nedb` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing neDB database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-irc-nedb --extra-vars='nedb_database_path={{ matrix_appservice_irc_data_path }} postgres_connection_string_variable_name=matrix_appservice_irc_database_connString'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_appservice_irc_nedb_stat_result.stat.exists"
-  when: "matrix_appservice_irc_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2
index 0da28403..94bbda7b 100644
--- a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2
+++ b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2
@@ -127,8 +127,8 @@ advanced:
 # Use an external database to store bridge state.
 database:
   # database engine (must be 'postgres' or 'nedb'). Default: nedb
-  engine: {{ matrix_appservice_irc_database_engine }}
+  engine: {{ matrix_appservice_irc_database_engine|to_json }}
   # Either a PostgreSQL connection string, or a path to the NeDB storage directory.
   # For postgres, it must start with postgres://
   # For NeDB, it must start with nedb://. The path is relative to the project directory.
-  connectionString: {{ matrix_appservice_irc_database_connString | to_json }}
+  connectionString: {{ matrix_appservice_irc_database_connectionString|to_json }}
diff --git a/roles/matrix-postgres/tasks/import_nedb.yml b/roles/matrix-postgres/tasks/import_nedb.yml
index cc1f9d78..2a3dd587 100644
--- a/roles/matrix-postgres/tasks/import_nedb.yml
+++ b/roles/matrix-postgres/tasks/import_nedb.yml
@@ -67,22 +67,6 @@
   become: false
   when: "matrix_postgres_service_start_result.changed|bool"
 
-# See https://github.com/matrix-org/matrix-appservice-irc/wiki/Migrating-from-NEdB-to-PostgreSQL
-- name: Import appservice_irc NeDB database from {{ sqlite_database_path }} into Postgres
-  when: database == 'appservice_irc'
-  command:
-    cmd: >-
-      {{ matrix_host_command_docker }} run
-      --rm
-      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
-      --cap-drop=ALL
-      --network={{ matrix_docker_network }}
-      --mount type=bind,src={{ matrix_appservice_irc_data_path }}:/data:ro
-      --entrypoint=/bin/sh
-      {{ matrix_appservice_irc_docker_image }}
-      -c
-      './scripts/migrate-db-to-pgres.sh -d /data -p passkey.pem -c {{ postgres_db_connection_string }}'
-
 # No migration.sh available, but found this:
 # https://github.com/matrix-org/matrix-appservice-slack/blob/develop/src/scripts/migrateToPostgres.ts
 # Usage should be similar to appservice_irc
@@ -95,15 +79,12 @@
       --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
       --cap-drop=ALL
       --network={{ matrix_docker_network }}
-      --mount type=bind,src={{ matrix_appservice_irc_data_path }}:/data:ro
+      --mount type=bind,src={{ matrix_appservice_irc_data_path }},dst=/data,ro
       --entrypoint=/bin/sh
       {{ matrix_appservice_slack_docker_image }}
       -c
       'node /lib/scripts/migrate-db-to-pgres.js -d /data -p passkey.pem -c {{ postgres_db_connection_string }}'
 
-- name: Archive NeDB database ({{ sqlite_database_path }} -> {{ sqlite_database_path }}.backup)
-  command:
-    cmd: "mv {{ sqlite_database_path }} {{ sqlite_database_path }}.backup"
 
 - name: Inject result
   set_fact:
diff --git a/roles/matrix-postgres/tasks/main.yml b/roles/matrix-postgres/tasks/main.yml
index 51801314..86b8f872 100644
--- a/roles/matrix-postgres/tasks/main.yml
+++ b/roles/matrix-postgres/tasks/main.yml
@@ -32,14 +32,6 @@
   tags:
     - import-generic-sqlite-db
 
-# Imports appservice-irc NeDB into postgres
-- import_tasks: "{{ role_path }}/tasks/import_nedb.yml"
-  vars:
-    database: appservice_irc
-  when: run_postgres_import_nedb|bool
-  tags:
-    - import-irc-nedb
-
 # Imports slacks neDB to postgres.
 - import_tasks: "{{ role_path }}/tasks/import_nedb.yml"
   vars:

From 9b95e1937c61a2b21d3d11b3d497e16c6176d593 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 19:34:08 +0200
Subject: [PATCH 132/179] Auto-restart matrix-appservice-irc after (nedb ->
 Postgres) migration

---
 roles/matrix-bridge-appservice-irc/tasks/setup_install.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml b/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml
index a748df96..00568c0d 100644
--- a/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml
+++ b/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml
@@ -36,7 +36,6 @@
       failed_when: false
   when: "matrix_appservice_irc_stat_passkey.stat.exists"
 
-
 - set_fact:
     matrix_appservice_irc_requires_restart: false
 
@@ -164,3 +163,9 @@
   service:
     daemon_reload: yes
   when: "matrix_appservice_irc_systemd_service_result.changed"
+
+- name: Ensure matrix-appservice-irc.service restarted, if necessary
+  service:
+    name: "matrix-appservice-irc.service"
+    state: restarted
+  when: "matrix_appservice_irc_requires_restart|bool"

From 8675dedbdb310425d309f6bf1725d7a6c415279e Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 19:56:52 +0200
Subject: [PATCH 133/179] Add support for automatic (nedb -> Postgres)
 migration to matrix-appservice-slack

---
 roles/matrix-base/defaults/main.yml           |  1 -
 .../tasks/migrate_nedb_to_postgres.yml        | 11 +++
 .../defaults/main.yml                         | 19 ++--
 .../tasks/migrate_nedb_to_postgres.yml        | 66 +++++++++++++
 .../tasks/setup_install.yml                   | 37 +++++--
 .../tasks/validate_config.yml                 | 20 ----
 .../templates/config.yaml.j2                  |  7 +-
 roles/matrix-postgres/tasks/import_nedb.yml   | 98 -------------------
 roles/matrix-postgres/tasks/main.yml          |  8 --
 9 files changed, 123 insertions(+), 144 deletions(-)
 create mode 100644 roles/matrix-bridge-appservice-slack/tasks/migrate_nedb_to_postgres.yml
 delete mode 100644 roles/matrix-postgres/tasks/import_nedb.yml

diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml
index d0fdcdf8..e0522ba8 100644
--- a/roles/matrix-base/defaults/main.yml
+++ b/roles/matrix-base/defaults/main.yml
@@ -106,7 +106,6 @@ matrix_docker_package_name: docker-ce
 run_postgres_import: true
 run_postgres_upgrade: true
 run_postgres_import_sqlite_db: true
-run_postgres_import_nedb: true
 run_postgres_vacuum: true
 run_synapse_register_user: true
 run_synapse_update_user_password: true
diff --git a/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml b/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml
index bc6525ec..3fab195a 100644
--- a/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml
+++ b/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml
@@ -51,3 +51,14 @@
   with_items:
     - rooms.db
     - users.db
+
+- name: Inject result
+  set_fact:
+    matrix_playbook_runtime_results: |
+      {{
+        matrix_playbook_runtime_results|default([])
+        +
+        [
+          "NOTE: Your appservice-irc database files have been imported into Postgres. The original database files have been moved from `{{ matrix_appservice_irc_data_path }}/*.db` to `{{ matrix_appservice_irc_data_path }}/*.db.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete these files."
+        ]
+      }}
diff --git a/roles/matrix-bridge-appservice-slack/defaults/main.yml b/roles/matrix-bridge-appservice-slack/defaults/main.yml
index 365493ff..28646a3c 100644
--- a/roles/matrix-bridge-appservice-slack/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-slack/defaults/main.yml
@@ -51,13 +51,18 @@ matrix_appservice_slack_database_password: ~
 matrix_appservice_slack_database_hostname: 'matrix-postgres'
 matrix_appservice_slack_database_port: 5432
 matrix_appservice_slack_database_name: matrix_appservice_slack
-matrix_appservice_slack_database_file: /data
-matrix_appservice_slack_database_connString: >-2
-  {%- if matrix_appservice_slack_database_engine == 'postgres' -%}
-  postgresql://{{ matrix_appservice_slack_database_username }}:{{ matrix_appservice_slack_database_password }}@{{ matrix_appservice_slack_database_hostname }}:{{ matrix_appservice_slack_database_port }}/{{ matrix_appservice_slack_database_name }}?sslmode=disable
-  {%- elif matrix_appservice_slack_database_engine == 'nedb' -%}
-  {{ matrix_appservice_slack_database_engine }}://{{ matrix_appservice_slack_database_file }}
-  {%- endif -%}
+
+# This is just the Postgres connection string, if Postgres is used.
+# Naming clashes with `matrix_appservice_slack_database_connectionString` somewhat.
+matrix_appservice_slack_database_connection_string: 'postgresql://{{ matrix_appservice_slack_database_username }}:{{ matrix_appservice_slack_database_password }}@{{ matrix_appservice_slack_database_hostname }}:{{ matrix_appservice_slack_database_port }}/{{ matrix_appservice_slack_database_name }}?sslmode=disable'
+
+# This is what actually goes into `database.connectionString` for the bridge.
+matrix_appservice_slack_database_connectionString: "{{
+	{
+		'nedb': 'nedb:///data',
+		'postgres': matrix_appservice_slack_database_connection_string,
+	}[matrix_appservice_slack_database_engine]
+}}"
 
 
 matrix_appservice_slack_configuration_yaml: "{{ lookup('template', 'templates/config.yaml.j2') }}"
diff --git a/roles/matrix-bridge-appservice-slack/tasks/migrate_nedb_to_postgres.yml b/roles/matrix-bridge-appservice-slack/tasks/migrate_nedb_to_postgres.yml
new file mode 100644
index 00000000..fedad977
--- /dev/null
+++ b/roles/matrix-bridge-appservice-slack/tasks/migrate_nedb_to_postgres.yml
@@ -0,0 +1,66 @@
+- name: Fail if Postgres not enabled
+  fail:
+    msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot migrate."
+  when: "not matrix_postgres_enabled|bool"
+
+# Defaults
+
+- name: Set postgres_start_wait_time, if not provided
+  set_fact:
+    postgres_start_wait_time: 15
+  when: "postgres_start_wait_time|default('') == ''"
+
+# Actual import work
+
+- name: Ensure matrix-postgres is started
+  service:
+    name: matrix-postgres
+    state: started
+    daemon_reload: yes
+  register: matrix_postgres_service_start_result
+
+- name: Wait a bit, so that Postgres can start
+  wait_for:
+    timeout: "{{ postgres_start_wait_time }}"
+  delegate_to: 127.0.0.1
+  become: false
+  when: "matrix_postgres_service_start_result.changed|bool"
+
+- name: Ensure matrix-appservice-slack is stopped
+  service:
+    name: matrix-appservice-slack
+    state: stopped
+
+- name: Import appservice-slack NeDB database into Postgres
+  command:
+    cmd: >-
+      {{ matrix_host_command_docker }} run
+      --rm
+      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
+      --cap-drop=ALL
+      --network={{ matrix_docker_network }}
+      --mount type=bind,src={{ matrix_appservice_slack_data_path }},dst=/data
+      --entrypoint=/bin/sh
+      {{ matrix_appservice_slack_docker_image }}
+      -c
+      '/usr/local/bin/node /usr/src/app/lib/scripts/migrateToPostgres.js --dbdir /data --connectionString {{ matrix_appservice_slack_database_connection_string }}'
+
+- name: Archive NeDB database files
+  command:
+    cmd: "mv {{ matrix_appservice_slack_data_path }}/{{ item }} {{ matrix_appservice_slack_data_path }}/{{ item }}.backup"
+  with_items:
+    - teams.db
+    - room-store.db
+    - user-store.db
+    - event-store.db
+
+- name: Inject result
+  set_fact:
+    matrix_playbook_runtime_results: |
+      {{
+        matrix_playbook_runtime_results|default([])
+        +
+        [
+          "NOTE: Your appservice-slack database files have been imported into Postgres. The original database files have been moved from `{{ matrix_appservice_slack_data_path }}/*.db` to `{{ matrix_appservice_slack_data_path }}/*.db.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete these files."
+        ]
+      }}
diff --git a/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml b/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml
index 94e0fedf..29b0f39e 100644
--- a/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml
+++ b/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml
@@ -1,12 +1,5 @@
 ---
 
-- name: Ensure Appservice Slack image is pulled
-  docker_image:
-    name: "{{ matrix_appservice_slack_docker_image }}"
-    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
-    force_source: "{{ matrix_appservice_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
-    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_slack_docker_image_force_pull }}"
-
 - name: Ensure AppService Slack paths exist
   file:
     path: "{{ item }}"
@@ -19,6 +12,30 @@
     - "{{ matrix_appservice_slack_config_path }}"
     - "{{ matrix_appservice_slack_data_path }}"
 
+- set_fact:
+    matrix_appservice_slack_requires_restart: false
+
+- block:
+    - name: Check if a nedb database already exists
+      stat:
+        path: "{{ matrix_appservice_slack_data_path }}/teams.db"
+      register: matrix_appservice_slack_nedb_database_path_local_stat_result
+
+    - block:
+        - import_tasks: "{{ role_path }}/tasks/migrate_nedb_to_postgres.yml"
+
+        - set_fact:
+            matrix_appservice_slack_requires_restart: true
+      when: "matrix_appservice_slack_nedb_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_appservice_slack_database_engine == 'postgres'"
+
+- name: Ensure Appservice Slack image is pulled
+  docker_image:
+    name: "{{ matrix_appservice_slack_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_appservice_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_slack_docker_image_force_pull }}"
+
 - name: Ensure Matrix Appservice Slack config installed
   copy:
     content: "{{ matrix_appservice_slack_configuration|to_nice_yaml }}"
@@ -46,3 +63,9 @@
   service:
     daemon_reload: yes
   when: "matrix_appservice_slack_systemd_service_result.changed"
+
+- name: Ensure matrix-appservice-slack.service restarted, if necessary
+  service:
+    name: "matrix-appservice-slack.service"
+    state: restarted
+  when: "matrix_appservice_slack_requires_restart|bool"
diff --git a/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml b/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml
index 5421b112..8af10f2f 100644
--- a/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml
+++ b/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml
@@ -10,23 +10,3 @@
     - "matrix_appservice_slack_appservice_token"
     - "matrix_appservice_slack_homeserver_token"
     - "matrix_appservice_slack_id_token"
-
-- block:
-  - name: Check if a neDB database already exists
-    stat:
-      path: "{{ matrix_appservice_slack_data_path }}/"
-    register: matrix_appservice_slack_nedb_stat_result
-
-  - name: Fail if an neDB database already exists when using Postgres
-    fail:
-      msg: >-2
-        matrix_appservice_slack_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing neDB database in {{ matrix_appservice_slack_data_path }}/.
-        It appears that you've been using this bridge with the neDB engine until now.
-        To continue using neDB, opt into it explicitly: add `matrix_appservice_slack_database_engine: nedb` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing neDB database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_slack_data_path }} postgres_connection_string_variable_name=matrix_appservice_slack_database_connString'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_appservice_slack_nedb_stat_result.stat.exists"
-  when: "matrix_appservice_slack_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2
index 0ae13738..bf8072c1 100644
--- a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2
+++ b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2
@@ -11,9 +11,10 @@ homeserver:
 
 {% if matrix_appservice_slack_database_engine == 'nedb' %}
 dbdir: "/data"
-{% endif %}
+{% else %}
 db:
-    engine: "{{ matrix_appservice_slack_database_engine }}"
-    connectionString: {{ matrix_appservice_slack_database_connString | to_json }}
+    engine: {{ matrix_appservice_slack_database_engine|to_json }}
+    connectionString: {{ matrix_appservice_slack_database_connectionString|to_json }}
+{% endif %}
 
 matrix_admin_room: "{{ matrix_appservice_slack_control_room_id }}"
diff --git a/roles/matrix-postgres/tasks/import_nedb.yml b/roles/matrix-postgres/tasks/import_nedb.yml
deleted file mode 100644
index 2a3dd587..00000000
--- a/roles/matrix-postgres/tasks/import_nedb.yml
+++ /dev/null
@@ -1,98 +0,0 @@
----
-
-# Pre-checks
-
-- name: Fail if Postgres not enabled
-  fail:
-    msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot import."
-  when: "not matrix_postgres_enabled|bool"
-
-- name: Fail if playbook called incorrectly
-  fail:
-    msg: "The `nedb_database_path` variable needs to be provided to this playbook, via --extra-vars"
-  when: "nedb_database_path is not defined or nedb_database_path.startswith('<')"
-
-- name: Check if the provided nedb database file exists
-  stat:
-    path: "{{ nedb_database_path }}"
-  register: nedb_database_path_stat_result
-
-- name: Fail if provided SQLite database file doesn't exist
-  fail:
-    msg: "File cannot be found on the server at {{ nedb_database_path }}"
-  when: "not nedb_database_path_stat_result.stat.exists"
-
-# We either expect `postgres_db_connection_string` specifying a full Postgres database connection string,
-# or `postgres_connection_string_variable_name`, specifying a name of a variable, which contains a valid connection string.
-
-- block:
-    - name: Fail if postgres_connection_string_variable_name points to an undefined variable
-      fail: msg="postgres_connection_string_variable_name is defined, but there is no variable with the name `{{ postgres_connection_string_variable_name }}`"
-      when: "postgres_connection_string_variable_name not in vars"
-
-    - name: Get Postgres connection string from variable
-      set_fact:
-        postgres_db_connection_string: "{{ lookup('vars', postgres_connection_string_variable_name) }}"
-  when: 'postgres_connection_string_variable_name is defined'
-
-- name: Fail if playbook called incorrectly
-  fail:
-    msg: >-
-      Either a `postgres_db_connection_string` variable or a `postgres_connection_string_variable_name` needs to be provided to this playbook, via `--extra-vars`.
-      Example: `--extra-vars="postgres_db_connection_string=postgresql://username:password@localhost:<port>/database_name"` or `--extra-vars="postgres_connection_string_variable_name=matrix_appservice_discord_database_connString"`
-  when: "postgres_db_connection_string is not defined or not postgres_db_connection_string.startswith('postgresql://')"
-
-
-# Defaults
-
-- name: Set postgres_start_wait_time, if not provided
-  set_fact:
-    postgres_start_wait_time: 15
-  when: "postgres_start_wait_time|default('') == ''"
-
-
-# Actual import work
-
-- name: Ensure matrix-postgres is started
-  service:
-    name: matrix-postgres
-    state: started
-    daemon_reload: yes
-  register: matrix_postgres_service_start_result
-
-- name: Wait a bit, so that Postgres can start
-  wait_for:
-    timeout: "{{ postgres_start_wait_time }}"
-  delegate_to: 127.0.0.1
-  become: false
-  when: "matrix_postgres_service_start_result.changed|bool"
-
-# No migration.sh available, but found this:
-# https://github.com/matrix-org/matrix-appservice-slack/blob/develop/src/scripts/migrateToPostgres.ts
-# Usage should be similar to appservice_irc
-- name: Import appservice_slack NeDB database from {{ sqlite_database_path }} into Postgres
-  when: database == 'appservice_slack'
-  command:
-    cmd: >-
-      {{ matrix_host_command_docker }} run
-      --rm
-      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
-      --cap-drop=ALL
-      --network={{ matrix_docker_network }}
-      --mount type=bind,src={{ matrix_appservice_irc_data_path }},dst=/data,ro
-      --entrypoint=/bin/sh
-      {{ matrix_appservice_slack_docker_image }}
-      -c
-      'node /lib/scripts/migrate-db-to-pgres.js -d /data -p passkey.pem -c {{ postgres_db_connection_string }}'
-
-
-- name: Inject result
-  set_fact:
-    matrix_playbook_runtime_results: |
-      {{
-        matrix_playbook_runtime_results|default([])
-        +
-        [
-          "NOTE: Your NeDB database file has been imported into Postgres. The original directory has been moved from `{{ nedb_database_path }}` to `{{ nedb_database_path }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file."
-        ]
-      }}
diff --git a/roles/matrix-postgres/tasks/main.yml b/roles/matrix-postgres/tasks/main.yml
index 86b8f872..b9c2ae7c 100644
--- a/roles/matrix-postgres/tasks/main.yml
+++ b/roles/matrix-postgres/tasks/main.yml
@@ -32,14 +32,6 @@
   tags:
     - import-generic-sqlite-db
 
-# Imports slacks neDB to postgres.
-- import_tasks: "{{ role_path }}/tasks/import_nedb.yml"
-  vars:
-    database: appservice_slack
-  when: run_postgres_import_nedb|bool
-  tags:
-    - import-slack-nedb
-
 - import_tasks: "{{ role_path }}/tasks/upgrade_postgres.yml"
   when: run_postgres_upgrade|bool
   tags:

From ad1425eee4f8d9eb1f92f5f0acecef8da3447280 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Wed, 23 Dec 2020 09:08:54 +0200
Subject: [PATCH 134/179] Add pgloader self-building support (for ARM)

---
 group_vars/matrix_servers                     |  2 +
 roles/matrix-postgres/defaults/main.yml       | 13 ++++-
 .../tasks/util/migrate_db_to_postgres.yml     | 55 +++++++++++++++----
 3 files changed, 58 insertions(+), 12 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 4ddadb43..c8af3cf2 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -995,6 +995,8 @@ matrix_postgres_connection_username: "synapse"
 matrix_postgres_connection_password: "synapse-password"
 matrix_postgres_db_name: "homeserver"
 
+matrix_postgres_pgloader_container_image_self_build: "{{ matrix_architecture != 'amd64' }}"
+
 matrix_postgres_additional_databases: |
   {{
     ([{
diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml
index 0c516281..8f1d0d78 100644
--- a/roles/matrix-postgres/defaults/main.yml
+++ b/roles/matrix-postgres/defaults/main.yml
@@ -65,4 +65,15 @@ matrix_postgres_import_databases_ignore_regex: "^CREATE DATABASE ({{ matrix_post
 # For most (subsequent) runs, Postgres would already be running, so no waiting will be happening at all.
 matrix_postgres_additional_databases_postgres_start_wait_timeout_seconds: 15
 
-matrix_postgres_pgloader_docker_image: "docker.io/illagrenan/pgloader:3.6.2"
+
+matrix_postgres_pgloader_container_image_self_build: false
+matrix_postgres_pgloader_container_image_self_build_repo: "https://github.com/illagrenan/pgloader-docker.git"
+matrix_postgres_pgloader_container_image_self_build_repo_branch: "v{{ matrix_postgres_pgloader_docker_image_tag }}"
+matrix_postgres_pgloader_container_image_self_build_src_path: "{{ matrix_postgres_base_path }}/pgloader-container-src"
+
+# We use illagrenan/pgloader, instead of the more official dimitri/pgloader image,
+# because the official one only provides a `latest` tag.
+matrix_postgres_pgloader_docker_image: "{{ matrix_postgres_pgloader_docker_image_name_prefix }}illagrenan/pgloader:{{ matrix_postgres_pgloader_docker_image_tag }}"
+matrix_postgres_pgloader_docker_image_name_prefix: "{{ 'localhost/' if matrix_postgres_pgloader_container_image_self_build else 'docker.io/' }}"
+matrix_postgres_pgloader_docker_image_tag: "3.6.2"
+matrix_postgres_pgloader_docker_image_force_pull: "{{ matrix_postgres_pgloader_docker_image.endswith(':latest') }}"
diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
index d01611ef..0da48c64 100644
--- a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
+++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
@@ -31,17 +31,50 @@
     msg: "File cannot be found on the server at {{ matrix_postgres_db_migration_request.src }}"
   when: "not matrix_postgres_db_migration_request_src_stat_result.stat.exists"
 
-- name: Fail if we cannot migrate on the current architecture ({{ matrix_architecture }})
-  fail:
-    msg: >-
-      {{ matrix_postgres_db_migration_request.engine_variable_name }} (part of {{ matrix_postgres_db_migration_request.caller }}) has been set to `postgres` (which is our new default now).
-      However, we've discovered an existing file-based database ({{ matrix_postgres_db_migration_request.engine_old }}) in {{ matrix_postgres_db_migration_request.src }}.
-      It appears that you've been using this bridge with a file-based database engine until now.
-      To continue using {{ matrix_postgres_db_migration_request.engine_old }}, opt into it explicitly: add `{{ matrix_postgres_db_migration_request.engine_variable_name }}: {{ matrix_postgres_db_migration_request.engine_old }}` to your vars.yml file and re-run this same command.
-      We'd normally auto-migrate you to Postgres, but we can't do it on the {{ matrix_architecture }} architecture. Our pgloader container image only supports amd64 (for now).
-      Learn more here: https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/740
-  when: "matrix_architecture != 'amd64'"
-
+- block:
+    - name: Ensure pgloader repository is present on self-build
+      git:
+        repo: "{{ matrix_postgres_pgloader_container_image_self_build_repo }}"
+        dest: "{{ matrix_postgres_pgloader_container_image_self_build_src_path }}"
+        version: "{{ matrix_postgres_pgloader_container_image_self_build_repo_branch }}"
+        force: "yes"
+      register: matrix_postgres_pgloader_git_pull_results
+
+    # If `stable` is used, we hit an error when processing /opt/src/pgloader/build/quicklisp/dists/quicklisp/software/uax-15-20201220-git/data/CompositionExclusions.txt:
+    # > the octet sequence #(194) cannot be decoded
+    #
+    # The issue is described here and is not getting fixed for months: https://github.com/dimitri/pgloader/pull/1179
+    #
+    # Although we're not using the dimitri/pgloader image, the one we're using suffers from the same problem.
+    - name: Switch pgloader base image from Debian stable (likely 10.x/Buster) to Bullseye
+      lineinfile:
+        path: "{{ matrix_postgres_pgloader_container_image_self_build_src_path }}/Dockerfile"
+        regexp: "{{ item.match }}"
+        line: "{{ item.replace }}"
+      with_items:
+        - match: '^FROM debian:stable-slim as builder$'
+          replace: 'FROM debian:bullseye-slim as builder'
+        - match: '^FROM debian:stable-slim$'
+          replace: 'FROM debian:bullseye-slim'
+
+    - name: Ensure pgloader Docker image is built
+      docker_image:
+        name: "{{ matrix_postgres_pgloader_docker_image }}"
+        source: build
+        force_source: "{{ matrix_postgres_pgloader_git_pull_results.changed }}"
+        build:
+          dockerfile: Dockerfile
+          path: "{{ matrix_postgres_pgloader_container_image_self_build_src_path }}"
+          pull: yes
+  when: "matrix_postgres_pgloader_container_image_self_build|bool"
+
+- name: Ensure pgloader Docker image is pulled
+  docker_image:
+    name: "{{ matrix_postgres_pgloader_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_postgres_pgloader_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_postgres_pgloader_docker_image_force_pull }}"
+  when: "not matrix_postgres_pgloader_container_image_self_build"
 
 # Defaults
 

From 3475b98b76ff27af5250dceba1e90195e2825ed7 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Wed, 23 Dec 2020 11:02:02 +0200
Subject: [PATCH 135/179] Announce the big move to all-on-Postgres

Related to
- https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/740
- https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/686
---
 CHANGELOG.md | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 946ec5b0..efc5c536 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,31 @@
+# 2020-12-23
+
+## The big move to all-on-Postgres (potentially dangerous)
+
+**TLDR**: all your bridges (and other services) will likely be auto-migrated from SQLite/nedb to Postgres, hopefully without trouble. You can opt-out (see how below), if too worried about breakage.
+
+Until now, we've only used Postgres as a database for Synapse. All other services (bridges, bots, etc.) were kept simple and used a file-based database (SQLite or nedb).
+
+Since [this huge pull request](https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/740), **all of our services now use Postgres by default**. Thanks to [Johanna Dorothea Reichmann](https://github.com/jdreichmann) for starting the work on it and for providing great input!
+
+Moving all services to Postgres brings a few **benefits** to us:
+
+- **improved performance**
+- **improved compatibility**. Most bridges are deprecating SQLite/nedb support or offer less features when not on Postgres.
+- **easier backups**. It's still some effort to take a proper backup (Postgres dump + various files, keys), but a Postgres dump now takes you much further.
+- we're now **more prepared to introduce other services** that need a Postgres database - [Dendrite](https://github.com/matrix-org/dendrite), the [mautrix-signal](https://github.com/tulir/mautrix-signal) bridge (existing [pull request](https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/686)), etc.
+
+### Key takeway
+
+- existing installations that use an [external Postgres](https://github.com/spantaleev/matrix-docker-ansible-deploy/blob/master/docs/configuring-playbook-external-postgres.md) server should be unaffected (they remain on SQLite/nedb for all services, except Synapse)
+
+- for existing installations which use our integrated Postgres database server (`matrix-postgres`, which is the default), **we automatically migrate data** from SQLite/nedb to Postgres and **archive the database files** (`something.db` -> `something.db.backup`), so you can restore them if you need to go back (see how below).
+
+- this is a **very large and somewhat untested change** (potentially dangerous), so **if you're not feeling confident/experimental, opt-out** of it for now (see below). Still, it's the new default and what we (and various bridges) will focus on going forward, so don't stick to old ways for too long.
+
+- you can remain on SQLite/nedb (at least for now) by adding a variable like this to your `vars.yml` file for each service you use: `matrix_COMPONENT_database_engine: sqlite` (e.g. `matrix_mautrix_facebook_database_engine: sqlite`). Some services (like `appservice-irc` and `appservice-slack`) don't use SQLite, so use `nedb`, instead of `sqlite` for them. If the playbook had already migrated you to Postgres, you will need to rename back the database files (`something.db.backup` -> `something.db`).
+
+
 # 2020-12-11
 
 ## synapse-janitor support removed

From 4fe1248d95b116177315edb986a2c86122423c86 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Wed, 23 Dec 2020 11:21:06 +0200
Subject: [PATCH 136/179] Update changelog entry to be more informative

---
 CHANGELOG.md | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index efc5c536..c4a586e8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -21,9 +21,23 @@ Moving all services to Postgres brings a few **benefits** to us:
 
 - for existing installations which use our integrated Postgres database server (`matrix-postgres`, which is the default), **we automatically migrate data** from SQLite/nedb to Postgres and **archive the database files** (`something.db` -> `something.db.backup`), so you can restore them if you need to go back (see how below).
 
-- this is a **very large and somewhat untested change** (potentially dangerous), so **if you're not feeling confident/experimental, opt-out** of it for now (see below). Still, it's the new default and what we (and various bridges) will focus on going forward, so don't stick to old ways for too long.
+### Opting-out of the Postgres migration
 
-- you can remain on SQLite/nedb (at least for now) by adding a variable like this to your `vars.yml` file for each service you use: `matrix_COMPONENT_database_engine: sqlite` (e.g. `matrix_mautrix_facebook_database_engine: sqlite`). Some services (like `appservice-irc` and `appservice-slack`) don't use SQLite, so use `nedb`, instead of `sqlite` for them. If the playbook had already migrated you to Postgres, you will need to rename back the database files (`something.db.backup` -> `something.db`).
+This is a **very large and somewhat untested change** (potentially dangerous), so **if you're not feeling confident/experimental, opt-out** of it for now. Still, it's the new default and what we (and various bridges) will focus on going forward, so don't stick to old ways for too long.
+
+You can remain on SQLite/nedb (at least for now) by adding a variable like this to your `vars.yml` file for each service you use: `matrix_COMPONENT_database_engine: sqlite` (e.g. `matrix_mautrix_facebook_database_engine: sqlite`).
+
+Some services (like `appservice-irc` and `appservice-slack`) don't use SQLite, so use `nedb`, instead of `sqlite` for them.
+
+### Going back to SQLite/nedb if things went wrong
+
+If you went with the Postgres migration and it went badly for you (some bridge not working as expected or not working at all), do this:
+
+- stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+- SSH into the server and rename the old database files (`something.db.backup` -> `something.db`). Example: `mv /matrix/mautrix-facebook/data/mautrix-facebook.db.backup /matrix/mautrix-facebook/data/mautrix-facebook.db`
+- switch the affected service back to SQLite (e.g. `matrix_mautrix_facebook_database_engine: sqlite`). Some services (like `appservice-irc` and `appservice-slack`) don't use SQLite, so use `nedb`, instead of `sqlite` for them.
+- re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+- [get in touch](README.md#support) with us
 
 
 # 2020-12-11

From c5f8b1f61bf544bdcae0cd916c3d48d17886d277 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Wed, 23 Dec 2020 11:40:22 +0200
Subject: [PATCH 137/179] Fix mautrix-whatsapp Postgres connection string to
 not use SSL by default

---
 roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
index 0d4f9852..beda6d7d 100644
--- a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
@@ -46,7 +46,7 @@ matrix_mautrix_whatsapp_database_hostname: 'matrix-postgres'
 matrix_mautrix_whatsapp_database_port: 5432
 matrix_mautrix_whatsapp_database_name: 'matrix_mautrix_whatsapp'
 
-matrix_mautrix_whatsapp_database_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_database_username }}:{{ matrix_mautrix_whatsapp_database_password }}@{{ matrix_mautrix_whatsapp_database_hostname }}:{{ matrix_mautrix_whatsapp_database_port }}/{{ matrix_mautrix_whatsapp_database_name }}'
+matrix_mautrix_whatsapp_database_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_database_username }}:{{ matrix_mautrix_whatsapp_database_password }}@{{ matrix_mautrix_whatsapp_database_hostname }}:{{ matrix_mautrix_whatsapp_database_port }}/{{ matrix_mautrix_whatsapp_database_name }}?sslmode=disable'
 
 matrix_mautrix_whatsapp_appservice_database_type: "{{
 	{

From 4675c8a715bae4da5288d14badf5d18f9e7a30e0 Mon Sep 17 00:00:00 2001
From: Jan <janbruckner@gmail.com>
Date: Wed, 23 Dec 2020 11:23:09 +0100
Subject: [PATCH 138/179] Update updating-users-passwords.md

Fixed markdown error
---
 docs/updating-users-passwords.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/updating-users-passwords.md b/docs/updating-users-passwords.md
index 785bc17c..7d2f2832 100644
--- a/docs/updating-users-passwords.md
+++ b/docs/updating-users-passwords.md
@@ -26,7 +26,7 @@ and then connecting to the postgres server and executing:
 ```
 UPDATE users SET password_hash = '<password-hash>' WHERE name = '@someone:server.com'
 ```
-`
+
 where `<password-hash>` is the hash returned by the docker command above.
 
 

From 9f00970c90795c92dad9d1e83ea7fa4f8e2afa80 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Wed, 23 Dec 2020 12:31:52 +0200
Subject: [PATCH 139/179] Do not use salts longer than 16 characters

We've hit this problem before as well. Certain Ansible installations
choke on it.
---
 group_vars/matrix_servers | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 904b2291..44848ae5 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -208,7 +208,7 @@ matrix_mautrix_facebook_bridge_presence: "{{ matrix_synapse_use_presence if matr
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_facebook_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_facebook_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_facebook.db') | to_uuid }}"
+matrix_mautrix_facebook_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau_fb.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -247,7 +247,7 @@ matrix_mautrix_hangouts_login_shared_secret: "{{ matrix_synapse_ext_password_pro
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_hangouts_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_hangouts_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_hangouts.db') | to_uuid }}"
+matrix_mautrix_hangouts_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau_hangouts.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -289,7 +289,7 @@ matrix_mautrix_telegram_login_shared_secret: "{{ matrix_synapse_ext_password_pro
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_telegram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_telegram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_telegram.db') | to_uuid }}"
+matrix_mautrix_telegram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau_telegram.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -323,7 +323,7 @@ matrix_mautrix_whatsapp_login_shared_secret: "{{ matrix_synapse_ext_password_pro
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_whatsapp_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_whatsapp_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_whatsapp.db') | to_uuid }}"
+matrix_mautrix_whatsapp_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau_whatsapp.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -385,7 +385,7 @@ matrix_mx_puppet_skype_login_shared_secret: "{{ matrix_synapse_ext_password_prov
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_skype_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_skype_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_skype.db') | to_uuid }}"
+matrix_mx_puppet_skype_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_skype.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -422,7 +422,7 @@ matrix_mx_puppet_slack_login_shared_secret: "{{ matrix_synapse_ext_password_prov
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_slack.db') | to_uuid }}"
+matrix_mx_puppet_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_slack.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -460,7 +460,7 @@ matrix_mx_puppet_twitter_container_http_host_bind_port: "{{ '' if matrix_nginx_p
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_twitter_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_twitter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_twitter.db') | to_uuid }}"
+matrix_mx_puppet_twitter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_twitter.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -497,7 +497,7 @@ matrix_mx_puppet_instagram_login_shared_secret: "{{ matrix_synapse_ext_password_
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_instagram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_instagram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_instagram.db') | to_uuid }}"
+matrix_mx_puppet_instagram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_ig.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -533,7 +533,7 @@ matrix_mx_puppet_discord_login_shared_secret: "{{ matrix_synapse_ext_password_pr
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_discord_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_discord.db') | to_uuid }}"
+matrix_mx_puppet_discord_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_dsc.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -569,7 +569,7 @@ matrix_mx_puppet_steam_login_shared_secret: "{{ matrix_synapse_ext_password_prov
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_steam_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_steam.db') | to_uuid }}"
+matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_steam.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -1048,7 +1048,7 @@ matrix_postgres_additional_databases: |
     ([{
       'name': 'matrix_bridge_sms',
       'username': 'matrix_bridge_sms',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db') | to_uuid,
     }] if matrix_sms_bridge_enabled else [])
     +
     ([{

From 8b6174786b86f3ec09246228df7ad21e9a178f8f Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Wed, 23 Dec 2020 12:57:30 +0200
Subject: [PATCH 140/179] Fixup Dimension database schema a bit after pgloader
 import

---
 roles/matrix-dimension/tasks/setup_install.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/roles/matrix-dimension/tasks/setup_install.yml b/roles/matrix-dimension/tasks/setup_install.yml
index 9a264449..b7f9b8e0 100644
--- a/roles/matrix-dimension/tasks/setup_install.yml
+++ b/roles/matrix-dimension/tasks/setup_install.yml
@@ -19,6 +19,12 @@
               engine_old: 'sqlite'
               systemd_services_to_stop: ['matrix-dimension.service']
               pgloader_options: ['--with "quote identifiers"']
+              # pgloader makes `isSelfBot` of type `smallint`, instead of `boolean`.
+              # We need to fix it up
+              additional_psql_statements_list:
+                - 'ALTER TABLE dimension_users ALTER COLUMN "isSelfBot" DROP default;'
+                - 'ALTER TABLE dimension_users ALTER COLUMN "isSelfBot" TYPE BOOLEAN USING("isSelfBot"::text::boolean);'
+                - 'ALTER TABLE dimension_users ALTER COLUMN "isSelfBot" SET default false;'
 
         - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
 

From ea804f2f9fc16cdfbd9d1e30ff22f10944b8f28a Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Wed, 23 Dec 2020 12:59:48 +0200
Subject: [PATCH 141/179] Do not use underscore in salts passed to sha512

Some Ansible installations choke on it, it seems.
Similar to 9f00970c907
---
 group_vars/matrix_servers | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 44848ae5..a54d8baa 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -208,7 +208,7 @@ matrix_mautrix_facebook_bridge_presence: "{{ matrix_synapse_use_presence if matr
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_facebook_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_facebook_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau_fb.db') | to_uuid }}"
+matrix_mautrix_facebook_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau.fb.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -247,7 +247,7 @@ matrix_mautrix_hangouts_login_shared_secret: "{{ matrix_synapse_ext_password_pro
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_hangouts_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_hangouts_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau_hangouts.db') | to_uuid }}"
+matrix_mautrix_hangouts_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau.hangouts.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -289,7 +289,7 @@ matrix_mautrix_telegram_login_shared_secret: "{{ matrix_synapse_ext_password_pro
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_telegram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_telegram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau_telegram.db') | to_uuid }}"
+matrix_mautrix_telegram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau.telegram.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -323,7 +323,7 @@ matrix_mautrix_whatsapp_login_shared_secret: "{{ matrix_synapse_ext_password_pro
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_whatsapp_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_whatsapp_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau_whatsapp.db') | to_uuid }}"
+matrix_mautrix_whatsapp_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mauwhatsapp.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -385,7 +385,7 @@ matrix_mx_puppet_skype_login_shared_secret: "{{ matrix_synapse_ext_password_prov
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_skype_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_skype_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_skype.db') | to_uuid }}"
+matrix_mx_puppet_skype_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup.skype.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -422,7 +422,7 @@ matrix_mx_puppet_slack_login_shared_secret: "{{ matrix_synapse_ext_password_prov
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_slack.db') | to_uuid }}"
+matrix_mx_puppet_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup.slack.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -460,7 +460,7 @@ matrix_mx_puppet_twitter_container_http_host_bind_port: "{{ '' if matrix_nginx_p
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_twitter_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_twitter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_twitter.db') | to_uuid }}"
+matrix_mx_puppet_twitter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup.twitter.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -497,7 +497,7 @@ matrix_mx_puppet_instagram_login_shared_secret: "{{ matrix_synapse_ext_password_
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_instagram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_instagram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_ig.db') | to_uuid }}"
+matrix_mx_puppet_instagram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup.ig.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -533,7 +533,7 @@ matrix_mx_puppet_discord_login_shared_secret: "{{ matrix_synapse_ext_password_pr
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_discord_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_dsc.db') | to_uuid }}"
+matrix_mx_puppet_discord_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup.dsc.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -569,7 +569,7 @@ matrix_mx_puppet_steam_login_shared_secret: "{{ matrix_synapse_ext_password_prov
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_steam_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_steam.db') | to_uuid }}"
+matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup.steam.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -1048,7 +1048,7 @@ matrix_postgres_additional_databases: |
     ([{
       'name': 'matrix_bridge_sms',
       'username': 'matrix_bridge_sms',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db') | to_uuid,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge.sms.db') | to_uuid,
     }] if matrix_sms_bridge_enabled else [])
     +
     ([{

From be0c59956525f6219aa510344d03d09e850fa7ca Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Wed, 23 Dec 2020 13:33:25 +0200
Subject: [PATCH 142/179] Feed more slashes to mautrix bridges when using
 SQLite

This makes the `sqlite://` URI match what we were using before
and what the config expects.
---
 roles/matrix-bridge-mautrix-facebook/defaults/main.yml | 2 +-
 roles/matrix-bridge-mautrix-hangouts/defaults/main.yml | 2 +-
 roles/matrix-bridge-mautrix-telegram/defaults/main.yml | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
index 580934db..80e54e0e 100644
--- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
@@ -55,7 +55,7 @@ matrix_mautrix_facebook_database_connection_string: 'postgresql://{{ matrix_maut
 
 matrix_mautrix_facebook_appservice_database: "{{
 	{
-		'sqlite': ('sqlite://' + matrix_mautrix_facebook_sqlite_database_path_in_container),
+		'sqlite': ('sqlite:///' + matrix_mautrix_facebook_sqlite_database_path_in_container),
 		'postgres': matrix_mautrix_facebook_database_connection_string,
 	}[matrix_mautrix_facebook_database_engine]
 }}"
diff --git a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
index 984bec48..e266bcbb 100644
--- a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
@@ -62,7 +62,7 @@ matrix_mautrix_hangouts_database_connection_string: 'postgresql://{{ matrix_maut
 
 matrix_mautrix_hangouts_appservice_database: "{{
 	{
-		'sqlite': ('sqlite://' + matrix_mautrix_hangouts_sqlite_database_path_in_container),
+		'sqlite': ('sqlite:///' + matrix_mautrix_hangouts_sqlite_database_path_in_container),
 		'postgres': matrix_mautrix_hangouts_database_connection_string,
 	}[matrix_mautrix_hangouts_database_engine]
 }}"
diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
index ceebd3ec..539f0a9c 100644
--- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
@@ -66,7 +66,7 @@ matrix_mautrix_telegram_database_connection_string: 'postgresql://{{ matrix_maut
 
 matrix_mautrix_telegram_appservice_database: "{{
 	{
-		'sqlite': ('sqlite://' + matrix_mautrix_telegram_sqlite_database_path_in_container),
+		'sqlite': ('sqlite:///' + matrix_mautrix_telegram_sqlite_database_path_in_container),
 		'postgres': matrix_mautrix_telegram_database_connection_string,
 	}[matrix_mautrix_telegram_database_engine]
 }}"

From 019a4d7dcd774776aa9c7f6fef80cb9c58cfba97 Mon Sep 17 00:00:00 2001
From: Stuart Mumford <stuart@cadair.com>
Date: Wed, 23 Dec 2020 10:38:47 +0000
Subject: [PATCH 143/179] Use role relative paths for things

---
 roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml    | 2 +-
 roles/matrix-bridge-appservice-discord/tasks/setup_install.yml  | 2 +-
 roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml    | 2 +-
 roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml    | 2 +-
 roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml    | 2 +-
 roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml    | 2 +-
 roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml   | 2 +-
 roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml | 2 +-
 roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml     | 2 +-
 roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml     | 2 +-
 roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml     | 2 +-
 roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml   | 2 +-
 roles/matrix-dimension/tasks/setup_install.yml                  | 2 +-
 roles/matrix-ma1sd/tasks/setup_install.yml                      | 2 +-
 roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml     | 2 +-
 roles/matrix-registration/tasks/setup_install.yml               | 2 +-
 16 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml
index fea1e00d..195485e4 100644
--- a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml
+++ b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml
@@ -19,7 +19,7 @@
               engine_old: 'sqlite'
               systemd_services_to_stop: ['matrix-bot-matrix-reminder-bot.service']
 
-        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+        - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
 
         - set_fact:
             matrix_bot_matrix_reminder_bot_requires_restart: true
diff --git a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
index 8bf50e28..6d3fdd0f 100644
--- a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
+++ b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
@@ -19,7 +19,7 @@
               engine_old: 'sqlite'
               systemd_services_to_stop: ['matrix-appservice-discord.service']
 
-        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+        - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
 
         - set_fact:
             matrix_appservice_discord_requires_restart: true
diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
index 7f310446..59998463 100644
--- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
@@ -27,7 +27,7 @@
               engine_old: 'sqlite'
               systemd_services_to_stop: ['matrix-mautrix-facebook.service']
 
-        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+        - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
 
         - set_fact:
             matrix_mautrix_facebook_requires_restart: true
diff --git a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml
index 15af9626..2ce8a441 100644
--- a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml
@@ -27,7 +27,7 @@
               engine_old: 'sqlite'
               systemd_services_to_stop: ['matrix-mautrix-hangouts.service']
 
-        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+        - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
 
         - set_fact:
             matrix_mautrix_hangouts_requires_restart: true
diff --git a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml
index 6a37974a..e9a93c72 100644
--- a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml
@@ -27,7 +27,7 @@
               engine_old: 'sqlite'
               systemd_services_to_stop: ['matrix-mautrix-telegram.service']
 
-        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+        - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
 
         - set_fact:
             matrix_mautrix_telegram_requires_restart: true
diff --git a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml
index 8d894a84..1cfa60f8 100644
--- a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml
@@ -27,7 +27,7 @@
               engine_old: 'sqlite'
               systemd_services_to_stop: ['matrix-mautrix-whatsapp.service']
 
-        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+        - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
 
         - set_fact:
             matrix_mautrix_whatsapp_requires_restart: true
diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
index 3e3b2f94..c7865e98 100644
--- a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
@@ -58,7 +58,7 @@
               engine_old: 'sqlite'
               systemd_services_to_stop: ['matrix-mx-puppet-discord.service']
 
-        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+        - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
 
         - set_fact:
             matrix_mx_puppet_discord_requires_restart: true
diff --git a/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml
index 76bbd629..5701a916 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml
@@ -28,7 +28,7 @@
               engine_old: 'sqlite'
               systemd_services_to_stop: ['matrix-mx-puppet-instagram.service']
 
-        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+        - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
 
         - set_fact:
             matrix_mx_puppet_instagram_requires_restart: true
diff --git a/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml
index 9289a793..68a1d7f4 100644
--- a/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml
@@ -58,7 +58,7 @@
               engine_old: 'sqlite'
               systemd_services_to_stop: ['matrix-mx-puppet-skype.service']
 
-        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+        - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
 
         - set_fact:
             matrix_mx_puppet_skype_requires_restart: true
diff --git a/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml
index 5d68435c..04eab20a 100644
--- a/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml
@@ -54,7 +54,7 @@
               engine_old: 'sqlite'
               systemd_services_to_stop: ['matrix-mx-puppet-slack.service']
 
-        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+        - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
 
         - set_fact:
             matrix_mx_puppet_slack_requires_restart: true
diff --git a/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml
index 71f6d889..6b574656 100644
--- a/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml
@@ -58,7 +58,7 @@
               engine_old: 'sqlite'
               systemd_services_to_stop: ['matrix-mx-puppet-steam.service']
 
-        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+        - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
 
         - set_fact:
             matrix_mx_puppet_steam_requires_restart: true
diff --git a/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml
index 3893981a..1c48c030 100644
--- a/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml
@@ -58,7 +58,7 @@
               engine_old: 'sqlite'
               systemd_services_to_stop: ['matrix-mx-puppet-twitter.service']
 
-        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+        - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
 
         - set_fact:
             matrix_mx_puppet_twitter_requires_restart: true
diff --git a/roles/matrix-dimension/tasks/setup_install.yml b/roles/matrix-dimension/tasks/setup_install.yml
index b7f9b8e0..ca12f367 100644
--- a/roles/matrix-dimension/tasks/setup_install.yml
+++ b/roles/matrix-dimension/tasks/setup_install.yml
@@ -26,7 +26,7 @@
                 - 'ALTER TABLE dimension_users ALTER COLUMN "isSelfBot" TYPE BOOLEAN USING("isSelfBot"::text::boolean);'
                 - 'ALTER TABLE dimension_users ALTER COLUMN "isSelfBot" SET default false;'
 
-        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+        - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
 
         - set_fact:
             matrix_dimension_requires_restart: true
diff --git a/roles/matrix-ma1sd/tasks/setup_install.yml b/roles/matrix-ma1sd/tasks/setup_install.yml
index 9ae5f077..a0a32728 100644
--- a/roles/matrix-ma1sd/tasks/setup_install.yml
+++ b/roles/matrix-ma1sd/tasks/setup_install.yml
@@ -38,7 +38,7 @@
               systemd_services_to_stop: ['matrix-ma1sd.service']
               pgloader_options: ['--with "quote identifiers"']
 
-        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+        - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
 
         - set_fact:
             matrix_ma1sd_requires_restart: true
diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
index 0da48c64..de0327b7 100644
--- a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
+++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
@@ -135,7 +135,7 @@
 - block:
     # We can't use `{{ role_path }}` here, neither with `import_tasks`, nor with `include_tasks`,
     # because it refers to the role that included this util, and not to the role this file belongs to.
-    - import_tasks: "roles/matrix-postgres/tasks/util/detect_existing_postgres_version.yml"
+    - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/detect_existing_postgres_version.yml"
 
     - set_fact:
         matrix_postgres_docker_image_to_use: "{{ matrix_postgres_docker_image_latest if matrix_postgres_detected_version_corresponding_docker_image == '' else matrix_postgres_detected_version_corresponding_docker_image }}"
diff --git a/roles/matrix-registration/tasks/setup_install.yml b/roles/matrix-registration/tasks/setup_install.yml
index 9b6d1260..2b806fe0 100644
--- a/roles/matrix-registration/tasks/setup_install.yml
+++ b/roles/matrix-registration/tasks/setup_install.yml
@@ -24,7 +24,7 @@
                 - ALTER TABLE tokens ALTER COLUMN ex_date TYPE TIMESTAMP WITHOUT TIME ZONE;
               additional_psql_statements_db_name: "{{ matrix_registration_database_name }}"
 
-        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+        - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
 
         - set_fact:
             matrix_registration_requires_restart: true

From 21662af3be5c13fe03ed304a29860935a7a72365 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Wed, 23 Dec 2020 14:11:21 +0200
Subject: [PATCH 144/179] Archive database only after
 additional_psql_statements_list had executed

---
 .../matrix-postgres/tasks/util/migrate_db_to_postgres.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
index de0327b7..af95815f 100644
--- a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
+++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
@@ -128,10 +128,6 @@
       -c
       'pgloader {{ matrix_postgres_db_migration_request.pgloader_options|default([])|join(' ') }} /in.db {{ matrix_postgres_db_migration_request.dst }}'
 
-- name: Archive {{ matrix_postgres_db_migration_request.engine_old }} database ({{ matrix_postgres_db_migration_request.src }} -> {{ matrix_postgres_db_migration_request.src }}.backup)
-  command:
-    cmd: "mv {{ matrix_postgres_db_migration_request.src }} {{ matrix_postgres_db_migration_request.src }}.backup"
-
 - block:
     # We can't use `{{ role_path }}` here, neither with `import_tasks`, nor with `include_tasks`,
     # because it refers to the role that included this util, and not to the role this file belongs to.
@@ -155,6 +151,10 @@
 
   when: "matrix_postgres_db_migration_request.additional_psql_statements_list|default([])|length > 0"
 
+- name: Archive {{ matrix_postgres_db_migration_request.engine_old }} database ({{ matrix_postgres_db_migration_request.src }} -> {{ matrix_postgres_db_migration_request.src }}.backup)
+  command:
+    cmd: "mv {{ matrix_postgres_db_migration_request.src }} {{ matrix_postgres_db_migration_request.src }}.backup"
+
 - name: Inject result
   set_fact:
     matrix_playbook_runtime_results: |

From 80c72615c7bfdcf47644d55f033e88f4e610cf25 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Wed, 23 Dec 2020 14:11:47 +0200
Subject: [PATCH 145/179] Fixup all Dimension boolean fields after pgloader
 import

This is 8b6174786b86f3 done right. There were many more fields
that we had to account for.
---
 .../matrix-dimension/tasks/setup_install.yml  | 46 ++++++++++++++++---
 1 file changed, 40 insertions(+), 6 deletions(-)

diff --git a/roles/matrix-dimension/tasks/setup_install.yml b/roles/matrix-dimension/tasks/setup_install.yml
index ca12f367..26a75bcb 100644
--- a/roles/matrix-dimension/tasks/setup_install.yml
+++ b/roles/matrix-dimension/tasks/setup_install.yml
@@ -10,6 +10,44 @@
       register: matrix_dimension_sqlite_database_path_local_stat_result
 
     - block:
+        # pgloader makes a few columns `smallint`, instead of `boolean`.
+        # We need to fix them up.
+        - set_fact:
+            matrix_dimension_pgloader_additional_psql_statements_list: []
+
+        - set_fact:
+            matrix_dimension_pgloader_additional_psql_statements_list: |
+              {{
+                matrix_dimension_pgloader_additional_psql_statements_list
+                +
+                ([] if item.default == '' else ['ALTER TABLE ' + item.table + ' ALTER COLUMN "' + item.column + '" DROP default;'])
+                +
+                (['ALTER TABLE ' + item.table + ' ALTER COLUMN "' + item.column + '" TYPE BOOLEAN USING("' + item.column + '"::text::boolean);'])
+                +
+                ([] if item.default == '' else ['ALTER TABLE ' + item.table + ' ALTER COLUMN "' + item.column + '" SET default ' + item.default + ';'])
+              }}
+          with_items:
+            - {'table': 'dimension_widgets', 'column': 'isEnabled', 'default': ''}
+            - {'table': 'dimension_widgets', 'column': 'isPublic', 'default': ''}
+            - {'table': 'dimension_webhook_bridges', 'column': 'isEnabled', 'default': ''}
+            - {'table': 'dimension_user_sticker_packs', 'column': 'isSelected', 'default': ''}
+            - {'table': 'dimension_scalar_tokens', 'column': 'isDimensionToken', 'default': ''}
+            - {'table': 'dimension_users', 'column': 'isSelfBot', 'default': 'false'}
+            - {'table': 'dimension_telegram_bridges', 'column': 'allowTgPuppets', 'default': ''}
+            - {'table': 'dimension_telegram_bridges', 'column': 'allowMxPuppets', 'default': ''}
+            - {'table': 'dimension_telegram_bridges', 'column': 'isEnabled', 'default': ''}
+            - {'table': 'dimension_sticker_packs', 'column': 'isEnabled', 'default': ''}
+            - {'table': 'dimension_sticker_packs', 'column': 'isPublic', 'default': ''}
+            - {'table': 'dimension_slack_bridges', 'column': 'isEnabled', 'default': ''}
+            - {'table': 'dimension_neb_integrations', 'column': 'isPublic', 'default': ''}
+            - {'table': 'dimension_irc_bridges', 'column': 'isEnabled', 'default': ''}
+            - {'table': 'dimension_irc_bridge_networks', 'column': 'isEnabled', 'default': ''}
+            - {'table': 'dimension_gitter_bridges', 'column': 'isEnabled', 'default': ''}
+            - {'table': 'dimension_custom_simple_bots', 'column': 'isEnabled', 'default': ''}
+            - {'table': 'dimension_custom_simple_bots', 'column': 'isPublic', 'default': ''}
+            - {'table': 'dimension_bridges', 'column': 'isEnabled', 'default': ''}
+            - {'table': 'dimension_bridges', 'column': 'isPublic', 'default': ''}
+
         - set_fact:
             matrix_postgres_db_migration_request:
               src: "{{ matrix_dimension_sqlite_database_path_local }}"
@@ -19,12 +57,8 @@
               engine_old: 'sqlite'
               systemd_services_to_stop: ['matrix-dimension.service']
               pgloader_options: ['--with "quote identifiers"']
-              # pgloader makes `isSelfBot` of type `smallint`, instead of `boolean`.
-              # We need to fix it up
-              additional_psql_statements_list:
-                - 'ALTER TABLE dimension_users ALTER COLUMN "isSelfBot" DROP default;'
-                - 'ALTER TABLE dimension_users ALTER COLUMN "isSelfBot" TYPE BOOLEAN USING("isSelfBot"::text::boolean);'
-                - 'ALTER TABLE dimension_users ALTER COLUMN "isSelfBot" SET default false;'
+              additional_psql_statements_list: "{{ matrix_dimension_pgloader_additional_psql_statements_list }}"
+              additional_psql_statements_db_name: "{{ matrix_dimension_database_name }}"
 
         - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
 

From a2a4218e9573ad99a8c87a9edece1c4453cc3bff Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Wed, 23 Dec 2020 15:39:12 +0200
Subject: [PATCH 146/179] Make mautrix-python-based bridges E2EE happier

Fixes a problem like this:
> File "/usr/lib/python3.8/site-packages/mautrix/bridge/e2ee.py", line 79, in __init__
> raise RuntimeError("Unsupported database scheme")

mautrix-python's e2ee.py module expects to find `postgres://` instead of
`postgresql://`.
---
 roles/matrix-bridge-mautrix-facebook/defaults/main.yml | 2 +-
 roles/matrix-bridge-mautrix-hangouts/defaults/main.yml | 2 +-
 roles/matrix-bridge-mautrix-telegram/defaults/main.yml | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
index 80e54e0e..da9b1889 100644
--- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
@@ -51,7 +51,7 @@ matrix_mautrix_facebook_database_hostname: 'matrix-postgres'
 matrix_mautrix_facebook_database_port: 5432
 matrix_mautrix_facebook_database_name: 'matrix_mautrix_facebook'
 
-matrix_mautrix_facebook_database_connection_string: 'postgresql://{{ matrix_mautrix_facebook_database_username }}:{{ matrix_mautrix_facebook_database_password }}@{{ matrix_mautrix_facebook_database_hostname }}:{{ matrix_mautrix_facebook_database_port }}/{{ matrix_mautrix_facebook_database_name }}'
+matrix_mautrix_facebook_database_connection_string: 'postgres://{{ matrix_mautrix_facebook_database_username }}:{{ matrix_mautrix_facebook_database_password }}@{{ matrix_mautrix_facebook_database_hostname }}:{{ matrix_mautrix_facebook_database_port }}/{{ matrix_mautrix_facebook_database_name }}'
 
 matrix_mautrix_facebook_appservice_database: "{{
 	{
diff --git a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
index e266bcbb..8dfee030 100644
--- a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
@@ -58,7 +58,7 @@ matrix_mautrix_hangouts_database_hostname: 'matrix-postgres'
 matrix_mautrix_hangouts_database_port: 5432
 matrix_mautrix_hangouts_database_name: 'matrix_mautrix_hangouts'
 
-matrix_mautrix_hangouts_database_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_database_username }}:{{ matrix_mautrix_hangouts_database_password }}@{{ matrix_mautrix_hangouts_database_hostname }}:{{ matrix_mautrix_hangouts_database_port }}/{{ matrix_mautrix_hangouts_database_name }}'
+matrix_mautrix_hangouts_database_connection_string: 'postgres://{{ matrix_mautrix_hangouts_database_username }}:{{ matrix_mautrix_hangouts_database_password }}@{{ matrix_mautrix_hangouts_database_hostname }}:{{ matrix_mautrix_hangouts_database_port }}/{{ matrix_mautrix_hangouts_database_name }}'
 
 matrix_mautrix_hangouts_appservice_database: "{{
 	{
diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
index 539f0a9c..3f81617a 100644
--- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
@@ -62,7 +62,7 @@ matrix_mautrix_telegram_database_hostname: 'matrix-postgres'
 matrix_mautrix_telegram_database_port: 5432
 matrix_mautrix_telegram_database_name: 'matrix_mautrix_telegram'
 
-matrix_mautrix_telegram_database_connection_string: 'postgresql://{{ matrix_mautrix_telegram_database_username }}:{{ matrix_mautrix_telegram_database_password }}@{{ matrix_mautrix_telegram_database_hostname }}:{{ matrix_mautrix_telegram_database_port }}/{{ matrix_mautrix_telegram_database_name }}'
+matrix_mautrix_telegram_database_connection_string: 'postgres://{{ matrix_mautrix_telegram_database_username }}:{{ matrix_mautrix_telegram_database_password }}@{{ matrix_mautrix_telegram_database_hostname }}:{{ matrix_mautrix_telegram_database_port }}/{{ matrix_mautrix_telegram_database_name }}'
 
 matrix_mautrix_telegram_appservice_database: "{{
 	{

From befffa926bd0f71ab20c702fcbb6d0aeb58d730f Mon Sep 17 00:00:00 2001
From: David Gnedt <dgnedt@sba-research.org>
Date: Wed, 23 Dec 2020 19:24:45 +0100
Subject: [PATCH 147/179] Fix concatenation of additional databases

Otherwise the postgres upgrade fails with the following error:

Unexpected templating type error occurred on ({{
  [matrix_postgres_connection_username]
  +
  matrix_postgres_additional_databases|map(attribute='username')
}}
): can only concatenate list (not "generator") to list
---
 group_vars/matrix_servers | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index a54d8baa..b8fc9eb4 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -1098,14 +1098,14 @@ matrix_postgres_import_roles_to_ignore: |
   {{
     [matrix_postgres_connection_username]
     +
-    matrix_postgres_additional_databases|map(attribute='username')
+    matrix_postgres_additional_databases|map(attribute='username')|list
   }}
 
 matrix_postgres_import_databases_to_ignore: |
   {{
     [matrix_postgres_db_name]
     +
-    matrix_postgres_additional_databases|map(attribute='name')
+    matrix_postgres_additional_databases|map(attribute='name')|list
   }}
 
 ######################################################################

From 6403904f596bfacc27c73fcd432d3bd3450a4b35 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Thu, 24 Dec 2020 11:38:06 +0200
Subject: [PATCH 148/179] Try to make DNS SRV setup sound scarier (which it is)

We've had people who get the impression that well-known = bad,
DNS SRV = good, and who try to use DNS SRV for server delegation.

While it's true that DNS SRV can be superior for high-availability
scenarios, it's much harder to set up and comes with its own potential
downsides.

Using the well-known method is more straightforward and is enough
for almost all of us. Throwing people into the deep for no good
reason is not nice. Hopefully wording is better after this patch.
---
 docs/howto-server-delegation.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/howto-server-delegation.md b/docs/howto-server-delegation.md
index 1ea1091a..7bb86ec0 100644
--- a/docs/howto-server-delegation.md
+++ b/docs/howto-server-delegation.md
@@ -28,14 +28,14 @@ Server Delegation by means of a `/.well-known/matrix/server` file is the most st
 
 > Errors are recommended to be cached for up to an hour, and servers are encouraged to exponentially back off for repeated failures.
 
-If this is not a concern for you, feel free to not read ahead.
+**For most people, this is a reasonable tradeoff** given that it's easy and straightforward to set up. We recommend you stay on this path.
 
-Otherwise, you can decide to go against the default for this playbook, and instead set up [Server Delegation via a DNS SRV record (advanced)](#server-delegation-via-a-dns-srv-record-advanced).
+Otherwise, you can decide to go against the default for this playbook, and instead set up [Server Delegation via a DNS SRV record (advanced)](#server-delegation-via-a-dns-srv-record-advanced) (much more complicated).
 
 
 ## Server Delegation via a DNS SRV record (advanced)
 
-**NOTE**: doing Server Delegation via a DNS SRV record is a more advanced way to do it and is not the default for this playbook.
+**NOTE**: doing Server Delegation via a DNS SRV record is a more **advanced** way to do it and is not the default for this playbook. This is usually **much more complicated** to set up, so **we don't recommend it**. If you're not an experience sysadmi, you'd better stay away from this.
 
 As per the [Server-Server spec](https://matrix.org/docs/spec/server_server/r0.1.0.html#server-discovery), it's possible to do Server Delegation using only a SRV record (without a `/.well-known/matrix/server` file).
 
@@ -47,7 +47,7 @@ To use DNS SRV record validation, you need to:
 
 - ensure that you have a `_matrix._tcp` DNS SRV record for your base domain (`<your-domain>`) with a value of `10 0 8448 matrix.<your-domain>`
 
-- ensure that you are serving the Matrix Federation API (tcp/8448) with a certificate for `<your-domain>` (not `matrix.<your-domain>`!). See below.
+- ensure that you are serving the Matrix Federation API (tcp/8448) with a certificate for `<your-domain>` (not `matrix.<your-domain>`!). Getting this certiface to the `matrix.<your-domain>` server may be complicated. The playbook's automatic SSL obtaining/renewal flow will likely not work and you'll need to copy certificates around manually. See below.
 
 
 ### Obtaining certificates

From d023355d2e65a3a6cb3ef2018699e061caf2656f Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Thu, 24 Dec 2020 11:43:35 +0200
Subject: [PATCH 149/179] Fix typo

---
 docs/howto-server-delegation.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/howto-server-delegation.md b/docs/howto-server-delegation.md
index 7bb86ec0..9f0f53c6 100644
--- a/docs/howto-server-delegation.md
+++ b/docs/howto-server-delegation.md
@@ -35,7 +35,7 @@ Otherwise, you can decide to go against the default for this playbook, and inste
 
 ## Server Delegation via a DNS SRV record (advanced)
 
-**NOTE**: doing Server Delegation via a DNS SRV record is a more **advanced** way to do it and is not the default for this playbook. This is usually **much more complicated** to set up, so **we don't recommend it**. If you're not an experience sysadmi, you'd better stay away from this.
+**NOTE**: doing Server Delegation via a DNS SRV record is a more **advanced** way to do it and is not the default for this playbook. This is usually **much more complicated** to set up, so **we don't recommend it**. If you're not an experience sysadmin, you'd better stay away from this.
 
 As per the [Server-Server spec](https://matrix.org/docs/spec/server_server/r0.1.0.html#server-discovery), it's possible to do Server Delegation using only a SRV record (without a `/.well-known/matrix/server` file).
 

From c0a62d6c17a17248757db5f71b0431f5ee44c0e4 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Thu, 24 Dec 2020 11:56:36 +0200
Subject: [PATCH 150/179] Mention base domain serving

---
 docs/howto-server-delegation.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/howto-server-delegation.md b/docs/howto-server-delegation.md
index 9f0f53c6..b89bd2f2 100644
--- a/docs/howto-server-delegation.md
+++ b/docs/howto-server-delegation.md
@@ -22,7 +22,7 @@ If this is okay with you, feel free to not read ahead.
 
 Server Delegation by means of a `/.well-known/matrix/server` file is the most straightforward, but suffers from the following downsides:
 
-- you need to have a working HTTPS server for the base domain (`<your-domain>`)
+- you need to have a working HTTPS server for the base domain (`<your-domain>`). If you don't have any server for the base domain at all, you can easily solve it by making the playbook [serve the base domain from the Matrix server](configuring-playbook-base-domain-serving.md).
 
 - any downtime on the base domain (`<your-domain>`) or network trouble between the matrix subdomain (`matrix.<your-domain>`) and the base `<domain>` may cause Matrix Federation outages. As the [Server-Server spec says](https://matrix.org/docs/spec/server_server/r0.1.0.html#server-discovery):
 
@@ -47,7 +47,7 @@ To use DNS SRV record validation, you need to:
 
 - ensure that you have a `_matrix._tcp` DNS SRV record for your base domain (`<your-domain>`) with a value of `10 0 8448 matrix.<your-domain>`
 
-- ensure that you are serving the Matrix Federation API (tcp/8448) with a certificate for `<your-domain>` (not `matrix.<your-domain>`!). Getting this certiface to the `matrix.<your-domain>` server may be complicated. The playbook's automatic SSL obtaining/renewal flow will likely not work and you'll need to copy certificates around manually. See below.
+- ensure that you are serving the Matrix Federation API (tcp/8448) with a certificate for `<your-domain>` (not `matrix.<your-domain>`!). Getting this certificate to the `matrix.<your-domain>` server may be complicated. The playbook's automatic SSL obtaining/renewal flow will likely not work and you'll need to copy certificates around manually. See below.
 
 
 ### Obtaining certificates

From 3dd0517f0472d22efc7fc4bad6caf46dbc0e7ab2 Mon Sep 17 00:00:00 2001
From: Aaron Raimist <aaron@raim.ist>
Date: Sat, 26 Dec 2020 20:13:49 -0600
Subject: [PATCH 151/179] Check for buggy version of Ansible that Ubuntu 20.04
 provides

---
 docs/ansible.md                          | 2 +-
 roles/matrix-base/tasks/sanity_check.yml | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/docs/ansible.md b/docs/ansible.md
index 776147bd..b5135639 100644
--- a/docs/ansible.md
+++ b/docs/ansible.md
@@ -11,7 +11,7 @@ If your local computer cannot run Ansible, you can also run Ansible on some serv
 
 Ansible 2.7.0 or newer is required.
 
-Ubuntu (at least 20.04) ships with a buggy version (see this [bug](https://bugs.launchpad.net/ubuntu/+source/ansible/+bug/1880359)), which can't be used in combination with a host running new systemd (more detaisl in [#517](https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/517), [#669]([669](https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/669))). If this problem affects you, you can: avoid running Ubuntu 20.04 on your host; run Ansible from another machine targeting your host; or try to upgrade to a newer Ansible version (see below).
+Note: Ubuntu 20.04 ships with Ansible 2.9.6 which is a buggy version (see this [bug](https://bugs.launchpad.net/ubuntu/+source/ansible/+bug/1880359)), which can't be used in combination with a host running new systemd (more details in [#517](https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/517), [#669](https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/669)). If this problem affects you, you can: avoid running Ubuntu 20.04 on your host; run Ansible from another machine targeting your host; or try to upgrade to a newer Ansible version (see below).
 
 
 ## Checking your Ansible version
diff --git a/roles/matrix-base/tasks/sanity_check.yml b/roles/matrix-base/tasks/sanity_check.yml
index e504bfe9..22dde5f3 100644
--- a/roles/matrix-base/tasks/sanity_check.yml
+++ b/roles/matrix-base/tasks/sanity_check.yml
@@ -6,6 +6,14 @@
     msg: "You are running on Ansible {{ ansible_version.string }}, which is not supported. See our guide about Ansible: https://github.com/spantaleev/matrix-docker-ansible-deploy/blob/master/docs/ansible.md"
   when: "(ansible_version.major < 2) or (ansible_version.major <= 2 and ansible_version.minor < 7)"
 
+# Though we do not support Ansible 2.9.6 which is buggy
+- name: Fail if running on Ansible 2.9.6 on Ubuntu
+  fail:
+    msg: "You are running on Ansible {{ ansible_version.string }}, which is not supported. See our guide about Ansible: https://github.com/spantaleev/matrix-docker-ansible-deploy/blob/master/docs/ansible.md"
+  when:
+    - ansible_distribution == 'Ubuntu'
+    - "ansible_version.major = 2 and ansible_version.major == 9 and ansible_version.minor = 6"
+
 - name: (Deprecation) Catch and report renamed settings
   fail:
     msg: >-

From 8827a49e21b90c2e9fdafc27f2ab293ea29fedd0 Mon Sep 17 00:00:00 2001
From: Aaron Raimist <aaron@raim.ist>
Date: Sat, 26 Dec 2020 20:20:00 -0600
Subject: [PATCH 152/179] Check equality properly

---
 roles/matrix-base/tasks/sanity_check.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-base/tasks/sanity_check.yml b/roles/matrix-base/tasks/sanity_check.yml
index 22dde5f3..59ab8b93 100644
--- a/roles/matrix-base/tasks/sanity_check.yml
+++ b/roles/matrix-base/tasks/sanity_check.yml
@@ -12,7 +12,7 @@
     msg: "You are running on Ansible {{ ansible_version.string }}, which is not supported. See our guide about Ansible: https://github.com/spantaleev/matrix-docker-ansible-deploy/blob/master/docs/ansible.md"
   when:
     - ansible_distribution == 'Ubuntu'
-    - "ansible_version.major = 2 and ansible_version.major == 9 and ansible_version.minor = 6"
+    - "ansible_version.major == 2 and ansible_version.major == 9 and ansible_version.minor == 6"
 
 - name: (Deprecation) Catch and report renamed settings
   fail:

From 86da489b9b5c2349b8943509db152a5641bc93dd Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 29 Dec 2020 10:31:20 +0200
Subject: [PATCH 153/179] Never fail when stopping systemd service during
 (SQLite -> Postgres) migration

We need to suppress systemd service-stopping requests in certain rare
cases like https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/771

That issue seems to describe a case, where a migration from mxisd to
ma1sd was happening (DB files had just been moved), and then we were
attemping to stop `matrix-ma1sd.service` so we could import that database into
Postgres. However, there's neither `matrix-mxisd.service`, nor
`matrix-ma1sd.service` after `migrate_mxisd.yml` had just run, so
stopping `matrix-ma1sd.service` was failing.
---
 roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
index af95815f..d3429c97 100644
--- a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
+++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
@@ -112,6 +112,7 @@
   service:
     name: "{{ item }}"
     state: stopped
+  failed_when: false
   with_items: "{{ matrix_postgres_db_migration_request.systemd_services_to_stop }}"
 
 - name: Import {{ matrix_postgres_db_migration_request.engine_old }} database from {{ matrix_postgres_db_migration_request.src }} into Postgres

From a1ecaf54ef5de44e9d3cac37416612fae35653f9 Mon Sep 17 00:00:00 2001
From: 0hlov3 <0hlov3@protonmail.com>
Date: Tue, 29 Dec 2020 12:00:01 +0100
Subject: [PATCH 154/179] Adds Example for Caddy2 Jitsi

---
 ansible.cfg               |  2 +-
 examples/caddy2/Caddyfile | 63 +++++++++++++++++++++++++++++++++++++--
 examples/caddy2/README.md | 12 ++++++++
 3 files changed, 74 insertions(+), 3 deletions(-)
 create mode 100644 examples/caddy2/README.md

diff --git a/ansible.cfg b/ansible.cfg
index 0ae274f7..6c2838d3 100644
--- a/ansible.cfg
+++ b/ansible.cfg
@@ -1,5 +1,5 @@
 [defaults]
 retry_files_enabled = False
-
+ansible_python_interpreter=/usr/bin/python3
 [connection]
 pipelining = True
diff --git a/examples/caddy2/Caddyfile b/examples/caddy2/Caddyfile
index 1438a50b..82728f96 100644
--- a/examples/caddy2/Caddyfile
+++ b/examples/caddy2/Caddyfile
@@ -1,4 +1,8 @@
 matrix.DOMAIN.tld {
+
+  # creates letsencrypt certificate
+  # tls your@email.com
+
   @identity {
         path /_matrix/identity/*
   }
@@ -94,7 +98,11 @@ matrix.DOMAIN.tld:8448 {
 }
 
 dimension.DOMAIN.tld {
-header {
+      
+      # creates letsencrypt certificate
+      # tls your@email.com
+      
+      header {
          	# Enable HTTP Strict Transport Security (HSTS) to force clients to always connect via HTTPS
         	Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
         	# Enable cross-site filter (XSS) and tell browser to block detected attacks
@@ -121,7 +129,11 @@ header {
 }
 
 element.DOMAIN.tld {
- 	header {
+
+      # creates letsencrypt certificate
+      # tls your@email.com
+ 	
+      header {
          	# Enable HTTP Strict Transport Security (HSTS) to force clients to always connect via HTTPS
         	Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
         	# Enable cross-site filter (XSS) and tell browser to block detected attacks
@@ -145,3 +157,50 @@ element.DOMAIN.tld {
                      header_up X-Forwarded-HttpsProto {proto}
         }
 }
+
+#jitsi.DOMAIN.tld {
+#  log {
+#        output discard
+#  }
+#  
+#  creates letsencrypt certificate
+#  tls your@email.com
+#
+#  header {
+#        # Enable HTTP Strict Transport Security (HSTS) to force clients to always connect via HTTPS
+#        Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
+#
+#        # Enable cross-site filter (XSS) and tell browser to block detected attacks
+#        X-XSS-Protection "1; mode=block"
+#
+#        # Prevent some browsers from MIME-sniffing a response away from the declared Content-Type
+#        X-Content-Type-Options "nosniff"
+#
+#        # Disallow the site to be rendered within a frame (clickjacking protection)
+#        X-Frame-Options "SAMEORIGIN"
+#
+#        # Disable some features
+#        Feature-Policy "accelerometer 'none';ambient-light-sensor 'none'; autoplay 'none';camera 'none';encrypted-media 'none';focus-without-user-activation 'none'; geolocation 'none';gyroscope #'none';magnetometer 'none';microphone 'none';midi 'none';payment 'none';picture-in-picture 'none'; speaker 'none';sync-xhr 'none';usb 'none';vr 'none'"
+#
+#        # Referer
+#        Referrer-Policy "no-referrer"
+#
+#        # X-Robots-Tag
+#        X-Robots-Tag "none"
+#
+#        # Remove Server header
+#        -Server
+#  }
+#
+#  handle {
+#        encode zstd gzip
+#
+#        reverse_proxy 127.0.0.1:12080 {
+#               header_up X-Forwarded-Port {http.request.port}
+#               header_up X-Forwarded-Proto {http.request.scheme}
+#               header_up X-Forwarded-TlsProto {tls_protocol}
+#               header_up X-Forwarded-TlsCipher {tls_cipher}
+#               header_up X-Forwarded-HttpsProto {proto}
+#        }
+#  }
+#}
\ No newline at end of file
diff --git a/examples/caddy2/README.md b/examples/caddy2/README.md
new file mode 100644
index 00000000..4752140f
--- /dev/null
+++ b/examples/caddy2/README.md
@@ -0,0 +1,12 @@
+# Caddyfile
+
+This directory contains sample files that show you how to do reverse-proxying using Caddy2.
+
+## Config
+
+| Variable           | Function |
+| ------------------ | -------- |
+| tls your@email.com | Specify an email address for your [ACME account](https://caddyserver.com/docs/caddyfile/directives/tls) (but if only one email is used for all sites, we recommend the email [global option](https://caddyserver.com/docs/caddyfile/options) instead) | 
+| tls                | To enable [tls](https://caddyserver.com/docs/caddyfile/directives/tls) support uncomment the lines for tls |
+| Jitsi              | To enable Jitsi support uncomment the lines for Jitsi and set your data |
+| log {output discard }         | No output. You can find the Options in the [Documentaton](https://caddyserver.com/docs/caddyfile/directives/log) for logging |
\ No newline at end of file

From 63c92fc2def26a96aea7e6c3b797c4ffa01f98f9 Mon Sep 17 00:00:00 2001
From: 0hlov3 <0hlov3@protonmail.com>
Date: Tue, 29 Dec 2020 12:02:12 +0100
Subject: [PATCH 155/179] Adds Example for Caddy2 Jitsi

---
 ansible.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible.cfg b/ansible.cfg
index 6c2838d3..0ae274f7 100644
--- a/ansible.cfg
+++ b/ansible.cfg
@@ -1,5 +1,5 @@
 [defaults]
 retry_files_enabled = False
-ansible_python_interpreter=/usr/bin/python3
+
 [connection]
 pipelining = True

From f86d781244d8220a88bb55c55f25d8dd7c374942 Mon Sep 17 00:00:00 2001
From: 0hlov3 <0hlov3@protonmail.com>
Date: Tue, 29 Dec 2020 14:07:32 +0100
Subject: [PATCH 156/179] Updates the the caddy2 Readme and comment lines for
 dimension

---
 examples/caddy2/Caddyfile | 66 +++++++++++++++++++--------------------
 examples/caddy2/README.md |  4 +--
 2 files changed, 34 insertions(+), 36 deletions(-)

diff --git a/examples/caddy2/Caddyfile b/examples/caddy2/Caddyfile
index 82728f96..d13b9788 100644
--- a/examples/caddy2/Caddyfile
+++ b/examples/caddy2/Caddyfile
@@ -97,37 +97,6 @@ matrix.DOMAIN.tld:8448 {
     }
 }
 
-dimension.DOMAIN.tld {
-      
-      # creates letsencrypt certificate
-      # tls your@email.com
-      
-      header {
-         	# Enable HTTP Strict Transport Security (HSTS) to force clients to always connect via HTTPS
-        	Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
-        	# Enable cross-site filter (XSS) and tell browser to block detected attacks
-        	X-XSS-Protection "1; mode=block"
-        	# Prevent some browsers from MIME-sniffing a response away from the declared Content-Type
-        	X-Content-Type-Options "nosniff"
-        	# Disallow the site to be rendered within a frame (clickjacking protection)
-        	X-Frame-Options "DENY"
-        	# X-Robots-Tag
-        	X-Robots-Tag "noindex, noarchive, nofollow"
-  	}
-
-    	handle {
-        	encode zstd gzip
-
-        	reverse_proxy localhost:8184  {
-               		header_up X-Forwarded-Port {http.request.port}
-               		header_up X-Forwarded-Proto {http.request.scheme}
-               		header_up X-Forwarded-TlsProto {tls_protocol}
-               		header_up X-Forwarded-TlsCipher {tls_cipher}
-               		header_up X-Forwarded-HttpsProto {proto}
-        	}
-  	}
-}
-
 element.DOMAIN.tld {
 
       # creates letsencrypt certificate
@@ -158,10 +127,39 @@ element.DOMAIN.tld {
         }
 }
 
+#dimension.DOMAIN.tld {
+#      
+#      # creates letsencrypt certificate
+#      # tls your@email.com
+#      
+#      header {
+#          # Enable HTTP Strict Transport Security (HSTS) to force clients to always connect via HTTPS
+#          Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
+#          # Enable cross-site filter (XSS) and tell browser to block detected attacks
+#          X-XSS-Protection "1; mode=block"
+#          # Prevent some browsers from MIME-sniffing a response away from the declared Content-Type
+#          X-Content-Type-Options "nosniff"
+#          # Disallow the site to be rendered within a frame (clickjacking protection)
+#          X-Frame-Options "DENY"
+#          # X-Robots-Tag
+#          X-Robots-Tag "noindex, noarchive, nofollow"
+#    }
+#
+#      handle {
+#          encode zstd gzip
+#
+#          reverse_proxy localhost:8184  {
+#                  header_up X-Forwarded-Port {http.request.port}
+#                  header_up X-Forwarded-Proto {http.request.scheme}
+#                  header_up X-Forwarded-TlsProto {tls_protocol}
+#                  header_up X-Forwarded-TlsCipher {tls_cipher}
+#                  header_up X-Forwarded-HttpsProto {proto}
+#          }
+#    }
+#}
+
+
 #jitsi.DOMAIN.tld {
-#  log {
-#        output discard
-#  }
 #  
 #  creates letsencrypt certificate
 #  tls your@email.com
diff --git a/examples/caddy2/README.md b/examples/caddy2/README.md
index 4752140f..022a4fec 100644
--- a/examples/caddy2/README.md
+++ b/examples/caddy2/README.md
@@ -8,5 +8,5 @@ This directory contains sample files that show you how to do reverse-proxying us
 | ------------------ | -------- |
 | tls your@email.com | Specify an email address for your [ACME account](https://caddyserver.com/docs/caddyfile/directives/tls) (but if only one email is used for all sites, we recommend the email [global option](https://caddyserver.com/docs/caddyfile/options) instead) | 
 | tls                | To enable [tls](https://caddyserver.com/docs/caddyfile/directives/tls) support uncomment the lines for tls |
-| Jitsi              | To enable Jitsi support uncomment the lines for Jitsi and set your data |
-| log {output discard }         | No output. You can find the Options in the [Documentaton](https://caddyserver.com/docs/caddyfile/directives/log) for logging |
\ No newline at end of file
+| Dimnension         | To enable Dimension support uncomment the lines for Dimension and set your data |
+| Jitsi              | To enable Jitsi support uncomment the lines for Jitsi and set your data |
\ No newline at end of file

From ffb837d4bc069748819079390b319313820d2d99 Mon Sep 17 00:00:00 2001
From: Sabine Laszakovits <sabinbox@laszakovits.net>
Date: Sat, 2 Jan 2021 00:39:11 +0100
Subject: [PATCH 157/179] made the bridge use the default postgres db

---
 group_vars/matrix_servers                     | 17 +++++++---
 .../defaults/main.yml                         | 28 +++++++++-------
 .../tasks/setup_install.yml                   | 17 +---------
 .../tasks/setup_uninstall.yml                 | 21 +-----------
 .../tasks/validate_config.yml                 |  9 +++--
 .../templates/config.yaml.j2                  |  2 +-
 .../matrix-mautrix-signal-db.service.j2       | 33 -------------------
 7 files changed, 39 insertions(+), 88 deletions(-)
 delete mode 100644 roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal-db.service.j2

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 09253dae..c9065420 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -262,6 +262,7 @@ matrix_mautrix_hangouts_database_password: "{{ matrix_synapse_macaroon_secret_ke
 #
 ######################################################################
 
+# We don't enable bridges by default.
 matrix_mautrix_signal_enabled: false
 
 # TODO: unclear whether to put this into the role or keep it here
@@ -271,7 +272,9 @@ matrix_mautrix_signal_systemd_required_services_list: |
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
     +
-    ['matrix-mautrix-signal-daemon.service', 'matrix-mautrix-signal-db.service']
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
+    +
+    ['matrix-mautrix-signal-daemon.service']
   }}
 
 matrix_mautrix_signal_homeserver_domain: '{{ matrix_domain }}'
@@ -284,9 +287,9 @@ matrix_mautrix_signal_appservice_token: "{{ matrix_synapse_macaroon_secret_key |
 
 matrix_mautrix_signal_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
-matrix_mautrix_signal_db_user: 'signal-db-user'
-matrix_mautrix_signal_db_password: 'signal-db-password'
-matrix_mautrix_signal_db_database: 'signal-db'
+matrix_mautrix_signal_database_engine: 'postgres'
+matrix_mautrix_signal_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau.signal.db') | to_uuid }}" 
+
 
 ######################################################################
 #
@@ -1069,6 +1072,12 @@ matrix_postgres_additional_databases: |
       'password': matrix_mautrix_hangouts_database_password,
     }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres') else [])
     +
+    ([{
+      'name': matrix_mautrix_signal_database_name, 
+      'username': matrix_mautrix_signal_database_username, 
+      'password': matrix_mautrix_signal_database_password, 
+    }] if (matrix_mautrix_signal_enabled and matrix_mautrix_signal_database_engine == 'postgres' and matrix_mautrix_signal_database_hostname == 'matrix-postgres') else [])
+    +
     ([{
       'name': matrix_mautrix_telegram_database_name,
       'username': matrix_mautrix_telegram_database_username,
diff --git a/roles/matrix-bridge-mautrix-signal/defaults/main.yml b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
index 5d6bb7d3..d26513df 100644
--- a/roles/matrix-bridge-mautrix-signal/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
@@ -30,9 +30,7 @@ matrix_mautrix_signal_container_extra_arguments: []
 # TODO: unclear whether to put this into group_vars or keep it here
 matrix_mautrix_signal_systemd_required_services_list:
   - 'docker.service'
-  - "{{ 'matrix-synapse.service' if matrix_synapse_enabled else [] }}"
   - 'matrix-mautrix-signal-daemon.service'
-  - 'matrix-mautrix-signal-db.service'
 
 # List of systemd services that matrix-mautrix-signal.service wants
 matrix_mautrix_signal_systemd_wanted_services_list: []
@@ -40,17 +38,25 @@ matrix_mautrix_signal_systemd_wanted_services_list: []
 matrix_mautrix_signal_appservice_token: ''
 matrix_mautrix_signal_homeserver_token: ''
 
-matrix_mautrix_signal_db_docker_image: "postgres:13.0-alpine"
-matrix_mautrix_signal_db_docker_image_force_pull: "{{ matrix_mautrix_signal_db_docker_image.endswith(':latest') }}"
-matrix_mautrix_signal_db_storage_path: "{{ matrix_mautrix_signal_base_path }}/database"
+# Database-related configuration fields
+#
+# This bridge only supports postgres. 
+#
+matrix_mautrix_signal_database_engine: 'postgres'
+
+matrix_mautrix_signal_database_username: 'matrix_mautrix_signal'
+matrix_mautrix_signal_database_password: 'some-password'
+matrix_mautrix_signal_database_hostname: 'matrix-postgres'
+matrix_mautrix_signal_database_port: 5432
+matrix_mautrix_signal_database_name: 'matrix_mautrix_signal'
 
-matrix_mautrix_signal_db_user: ''
-matrix_mautrix_signal_db_password: ''
-matrix_mautrix_signal_db_host: 'matrix-mautrix-signal-db'
-matrix_mautrix_signal_db_port: '5432'
-matrix_mautrix_signal_db_database: ''
+matrix_mautrix_signal_database_connection_string: 'postgres://{{ matrix_mautrix_signal_database_username }}:{{ matrix_mautrix_signal_database_password }}@{{ matrix_mautrix_signal_database_hostname }}:{{ matrix_mautrix_signal_database_port }}/{{ matrix_mautrix_signal_database_name }}' 
 
-matrix_mautrix_signal_db_url: "postgres://{{ matrix_mautrix_signal_db_user }}:{{ matrix_mautrix_signal_db_password }}@{{ matrix_mautrix_signal_db_host }}:{{ matrix_mautrix_signal_db_port }}/{{ matrix_mautrix_signal_db_database }}"
+matrix_mautrix_signal_appservice_database: "{{ 
+ 	{ 
+ 		'postgres': matrix_mautrix_facebook_database_connection_string, 
+ 	}[matrix_mautrix_signal_database_engine] 
+ }}" 
 
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mautrix_signal_login_shared_secret: ''
diff --git a/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml
index 16b68d25..9b049987 100644
--- a/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml
@@ -22,13 +22,6 @@
     force_source: "{{ matrix_mautrix_signal_daemon_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
     force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mautrix_signal_docker_image_force_pull }}"
 
-- name: Ensure Mautrix Signal database image is pulled
-  docker_image:
-    name: "{{ matrix_mautrix_signal_db_docker_image }}"
-    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
-    force_source: "{{ matrix_mautrix_signal_db_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
-    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mautrix_signal_docker_image_force_pull }}"
-
 - name: Ensure Mautrix Signal paths exist
   file:
     path: "{{ item }}"
@@ -40,7 +33,6 @@
     - "{{ matrix_mautrix_signal_base_path }}"
     - "{{ matrix_mautrix_signal_config_path }}"
     - "{{ matrix_mautrix_signal_daemon_path }}"
-    - "{{ matrix_mautrix_signal_db_storage_path }}"
 
 - name: Ensure mautrix-signal config.yaml installed
   copy:
@@ -65,13 +57,6 @@
     mode: 0644
   register: matrix_mautrix_signal_daemon_systemd_service_result
 
-- name: Ensure matrix-mautrix-signal-db.service installed
-  template:
-    src: "{{ role_path }}/templates/systemd/matrix-mautrix-signal-db.service.j2"
-    dest: "{{ matrix_systemd_path }}/matrix-mautrix-signal-db.service"
-    mode: 0644
-  register: matrix_mautrix_signal_db_systemd_service_result
-
 - name: Ensure matrix-mautrix-signal.service installed
   template:
     src: "{{ role_path }}/templates/systemd/matrix-mautrix-signal.service.j2"
@@ -82,4 +67,4 @@
 - name: Ensure systemd reloaded after matrix-mautrix-signal.service installation
   service:
     daemon_reload: yes
-  when: "matrix_mautrix_signal_systemd_service_result.changed or matrix_mautrix_signal_daemon_systemd_service_result.changed or matrix_mautrix_signal_db_systemd_service_result.changed"
+  when: "matrix_mautrix_signal_systemd_service_result.changed or matrix_mautrix_signal_daemon_systemd_service_result.changed"
diff --git a/roles/matrix-bridge-mautrix-signal/tasks/setup_uninstall.yml b/roles/matrix-bridge-mautrix-signal/tasks/setup_uninstall.yml
index 51581874..2ca6a9a9 100644
--- a/roles/matrix-bridge-mautrix-signal/tasks/setup_uninstall.yml
+++ b/roles/matrix-bridge-mautrix-signal/tasks/setup_uninstall.yml
@@ -1,24 +1,5 @@
 ---
 
-# Signal database service
-- name: Check existence of matrix-mautrix-signal-db service
-  stat:
-    path: "{{ matrix_systemd_path }}/matrix-mautrix-signal-db.service"
-  register: matrix_mautrix_signal_db_service_stat
-
-- name: Ensure matrix-mautrix-signal-db is stopped
-  service:
-    name: matrix-mautrix-signal-db
-    state: stopped
-    daemon_reload: yes
-  when: "matrix_mautrix_signal_db_service_stat.stat.exists"
-
-- name: Ensure matrix-mautrix-signal-db.service doesn't exist
-  file:
-    path: "{{ matrix_systemd_path }}/matrix-mautrix-signal-db.service"
-    state: absent
-  when: "matrix_mautrix_signal_db_service_stat.stat.exists"
-
 # Signal daemon service
 - name: Check existence of matrix-mautrix-signal-daemon service
   stat:
@@ -61,4 +42,4 @@
 - name: Ensure systemd reloaded after matrix-mautrix-signal_X.service removal
   service:
     daemon_reload: yes
-  when: "matrix_mautrix_signal_service_stat.stat.exists or matrix_mautrix_signal_daemon_service_stat.stat.exists or matrix_mautrix_signal_db_service_stat.stat.exists"
+  when: "matrix_mautrix_signal_service_stat.stat.exists or matrix_mautrix_signal_daemon_service_stat.stat.exists"
diff --git a/roles/matrix-bridge-mautrix-signal/tasks/validate_config.yml b/roles/matrix-bridge-mautrix-signal/tasks/validate_config.yml
index 6663c641..397ef4d8 100644
--- a/roles/matrix-bridge-mautrix-signal/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mautrix-signal/tasks/validate_config.yml
@@ -10,9 +10,6 @@
     - "matrix_mautrix_signal_homeserver_address"
     - "matrix_mautrix_signal_homeserver_token"
     - "matrix_mautrix_signal_appservice_token"
-    - "matrix_mautrix_signal_db_user"
-    - "matrix_mautrix_signal_db_password"
-    - "matrix_mautrix_signal_db_database"
 
 - name: (Deprecation) Catch and report renamed Signal variables
   fail:
@@ -22,3 +19,9 @@
   when: "item.old in vars"
   with_items:
     - {'old': 'matrix_mautrix_signal_container_exposed_port_number', 'new': '<superseded by matrix_mautrix_signal_container_http_host_bind_port>'}
+    - {'old': 'matrix_mautrix_signal_db_user', 'new': 'matrix_mautrix_signal_database_username'}
+    - {'old': 'matrix_mautrix_signal_db_password', 'new': 'matrix_mautrix_signal_database_password'}
+    - {'old': 'matrix_mautrix_signal_db_database', 'new': 'matrix_mautrix_signal_database_name'}
+    - {'old': 'matrix_mautrix_signal_db_host', 'new': 'matrix_mautrix_signal_database_hostname'}
+    - {'old': 'matrix_mautrix_signal_db_port', 'new': 'matrix_mautrix_signal_database_port'}
+    - {'old': 'matrix_mautrix_signal_db_url', 'new': 'matrix_mautrix_signal_database_connection_string'}
diff --git a/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2
index ec2a4764..64a820b5 100644
--- a/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2
@@ -27,7 +27,7 @@ appservice:
     max_body_size: 1
 
     # The full URI to the database. Only Postgres is currently supported.
-    database: {{ matrix_mautrix_signal_db_url }}
+    database: {{ matrix_mautrix_signal_database_connection_string }}
 
     # Provisioning API part of the web server for automated portal creation and fetching information.
     # Used by things like mautrix-manager (https://github.com/tulir/mautrix-manager).
diff --git a/roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal-db.service.j2 b/roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal-db.service.j2
deleted file mode 100644
index b4baed4c..00000000
--- a/roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal-db.service.j2
+++ /dev/null
@@ -1,33 +0,0 @@
-#jinja2: lstrip_blocks: "True"
-[Unit]
-Description=Matrix Mautrix Signal database server
-
-[Service]
-Type=simple
-
-ExecStartPre=-{{ matrix_host_command_docker }} kill {{ matrix_mautrix_signal_db_host }}
-ExecStartPre=-{{ matrix_host_command_docker }} rm {{ matrix_mautrix_signal_db_host }}
-
-# Intentional delay, so that the homeserver (we likely depend on) can manage to start.
-ExecStartPre={{ matrix_host_command_sleep }} 5
-
-ExecStart={{ matrix_host_command_docker }} run --rm --name {{ matrix_mautrix_signal_db_host }} \
-			--log-driver=none \
-			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
-			--network={{ matrix_docker_network }} \
-			--env POSTGRES_USER={{ matrix_mautrix_signal_db_user }} \
-			--env POSTGRES_PASSWORD={{ matrix_mautrix_signal_db_password }} \
-			--env POSTGRES_DB={{ matrix_mautrix_signal_db_database }} \
-			-v {{ matrix_mautrix_signal_db_storage_path }}:/var/lib/postgresql/data:z \
-			-v /etc/passwd:/etc/passwd:ro \
-			{{ matrix_mautrix_signal_db_docker_image }}
-
-ExecStop=-{{ matrix_host_command_docker }} kill {{ matrix_mautrix_signal_db_host }}
-ExecStop=-{{ matrix_host_command_docker }} rm {{ matrix_mautrix_signal_db_host }}
-
-Restart=always
-RestartSec=30
-SyslogIdentifier=matrix-mautrix-signal-db
-
-[Install]
-WantedBy=multi-user.target

From 89f7f3c3b8290ea13e0bf53c837cfe45a7cb6a0f Mon Sep 17 00:00:00 2001
From: Sabine Laszakovits <sabinbox@laszakovits.net>
Date: Sat, 2 Jan 2021 00:55:55 +0100
Subject: [PATCH 158/179] added log level configuration

---
 roles/matrix-bridge-mautrix-signal/defaults/main.yml        | 4 +++-
 roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2 | 4 ++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/roles/matrix-bridge-mautrix-signal/defaults/main.yml b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
index d26513df..d366ed57 100644
--- a/roles/matrix-bridge-mautrix-signal/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
@@ -28,7 +28,7 @@ matrix_mautrix_signal_container_extra_arguments: []
 
 # List of systemd services that matrix-mautrix-signal.service depends on.
 # TODO: unclear whether to put this into group_vars or keep it here
-matrix_mautrix_signal_systemd_required_services_list:
+matrix_mautrix_signal_systemd_required_services_list: 
   - 'docker.service'
   - 'matrix-mautrix-signal-daemon.service'
 
@@ -95,3 +95,5 @@ matrix_mautrix_signal_configuration: "{{ matrix_mautrix_signal_configuration_yam
 matrix_mautrix_signal_registration_yaml: "{{ lookup('template', 'templates/registration.yaml.j2') }}"
 
 matrix_mautrix_signal_registration: "{{ matrix_mautrix_signal_registration_yaml|from_yaml }}"
+
+matrix_mautrix_signal_log_level: 'DEBUG'
\ No newline at end of file
diff --git a/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2
index 64a820b5..df57915a 100644
--- a/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2
@@ -197,9 +197,9 @@ logging:
             formatter: colored
     loggers:
         mau:
-            level: DEBUG
+            level: {{ matrix_mautrix_signal_log_level }}
         aiohttp:
             level: INFO
     root:
-        level: DEBUG
+        level: {{ matrix_mautrix_signal_log_level }}
         handlers: [file, console]

From 56af2b1a8cce6a46d401c4556f3faa209b41b5ed Mon Sep 17 00:00:00 2001
From: Sabine Laszakovits <sabinbox@laszakovits.net>
Date: Sat, 2 Jan 2021 00:56:45 +0100
Subject: [PATCH 159/179] small fixes

---
 roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml
index 9b049987..29555116 100644
--- a/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml
@@ -14,6 +14,7 @@
     source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
     force_source: "{{ matrix_mautrix_signal_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
     force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mautrix_signal_docker_image_force_pull }}"
+  when: matrix_mautrix_signal_enabled|bool
 
 - name: Ensure Mautrix Signal Daemon image is pulled
   docker_image:
@@ -21,6 +22,7 @@
     source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
     force_source: "{{ matrix_mautrix_signal_daemon_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
     force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mautrix_signal_docker_image_force_pull }}"
+  when: matrix_mautrix_signal_enabled|bool
 
 - name: Ensure Mautrix Signal paths exist
   file:

From 84cac25c11baad5828d6655817281cb990b8159a Mon Sep 17 00:00:00 2001
From: Sabine Laszakovits <sabinbox@laszakovits.net>
Date: Sat, 2 Jan 2021 19:01:21 +0100
Subject: [PATCH 160/179] added config data_dir (else in ~, which isn't set)

---
 roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2 | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2
index df57915a..4335b023 100644
--- a/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2
@@ -73,6 +73,8 @@ signal:
     outgoing_attachment_dir: /signald/attachments
     # Directory where signald stores avatars for groups.
     avatar_dir: /signald/avatars
+    # Directory where signald stores auth data. Used to delete data when logging out.
+    data_dir: /signald/data
     # Whether or not message attachments should be removed from disk after they're bridged.
     remove_file_after_handling: true
 

From fb83eccf99cbe2d3f3411633eda2bd02f8cffdcd Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Fri, 1 Jan 2021 16:39:07 +0200
Subject: [PATCH 161/179] Relocate SQL template file

---
 roles/matrix-postgres/tasks/util/create_additional_database.yml | 2 +-
 .../templates/{ => sql}/init-additional-db-user-and-role.sql.j2 | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename roles/matrix-postgres/templates/{ => sql}/init-additional-db-user-and-role.sql.j2 (100%)

diff --git a/roles/matrix-postgres/tasks/util/create_additional_database.yml b/roles/matrix-postgres/tasks/util/create_additional_database.yml
index ce064d59..22b3c9a2 100644
--- a/roles/matrix-postgres/tasks/util/create_additional_database.yml
+++ b/roles/matrix-postgres/tasks/util/create_additional_database.yml
@@ -13,7 +13,7 @@
 # This way, we avoid passing sensitive data around in CLI commands that other users on the system can see.
 - name: Create additional database initialization SQL file for {{ additional_db.name }}
   template:
-    src: "{{ role_path }}/templates/init-additional-db-user-and-role.sql.j2"
+    src: "{{ role_path }}/templates/sql/init-additional-db-user-and-role.sql.j2"
     dest: "/tmp/matrix-postgres-init-additional-db-user-and-role.sql"
     mode: 0600
     owner: "{{ matrix_user_uid }}"
diff --git a/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2 b/roles/matrix-postgres/templates/sql/init-additional-db-user-and-role.sql.j2
similarity index 100%
rename from roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2
rename to roles/matrix-postgres/templates/sql/init-additional-db-user-and-role.sql.j2

From b5812b539b25b2453676cd7334fedda9691ddd19 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 3 Jan 2021 00:58:09 +0200
Subject: [PATCH 162/179] Rename ma1sd custom email template variable

Keeps up with https://github.com/ma1uta/ma1sd/commit/a1f64f515919e40708a4be59e508ac0dbf86f12f#diff-0ccf69eb4d59a7645eb4d0a0b077e693948edb33ad06df043bba3fb30122879b
---
 roles/matrix-ma1sd/defaults/main.yml         | 2 +-
 roles/matrix-ma1sd/tasks/setup_install.yml   | 2 +-
 roles/matrix-ma1sd/tasks/validate_config.yml | 1 +
 roles/matrix-ma1sd/templates/ma1sd.yaml.j2   | 6 +++---
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/roles/matrix-ma1sd/defaults/main.yml b/roles/matrix-ma1sd/defaults/main.yml
index 2932f3ed..321292d1 100644
--- a/roles/matrix-ma1sd/defaults/main.yml
+++ b/roles/matrix-ma1sd/defaults/main.yml
@@ -92,7 +92,7 @@ matrix_ma1sd_dns_overwrite_homeserver_client_value: "http://matrix-synapse:8008"
 matrix_ma1sd_threepid_medium_email_custom_templates_enabled: false
 matrix_ma1sd_threepid_medium_email_custom_invite_template: ""
 matrix_ma1sd_threepid_medium_email_custom_session_validation_template: ""
-matrix_ma1sd_threepid_medium_email_custom_unbind_fraudulent_template: ""
+matrix_ma1sd_threepid_medium_email_custom_session_unbind_notification_template: ""
 matrix_ma1sd_threepid_medium_email_custom_matrixid_template: ""
 
 # Controls whether the self-check feature should validate SSL certificates.
diff --git a/roles/matrix-ma1sd/tasks/setup_install.yml b/roles/matrix-ma1sd/tasks/setup_install.yml
index a0a32728..7cd6b72c 100644
--- a/roles/matrix-ma1sd/tasks/setup_install.yml
+++ b/roles/matrix-ma1sd/tasks/setup_install.yml
@@ -117,7 +117,7 @@
   with_items:
     - {value: "{{ matrix_ma1sd_threepid_medium_email_custom_invite_template }}", location: 'invite-template.eml'}
     - {value: "{{ matrix_ma1sd_threepid_medium_email_custom_session_validation_template }}", location: 'validate-template.eml'}
-    - {value: "{{ matrix_ma1sd_threepid_medium_email_custom_unbind_fraudulent_template }}", location: 'unbind-fraudulent.eml'}
+    - {value: "{{ matrix_ma1sd_threepid_medium_email_custom_session_unbind_notification_template }}", location: 'unbind-notification.eml'}
     - {value: "{{ matrix_ma1sd_threepid_medium_email_custom_matrixid_template }}", location: 'mxid-template.eml'}
   when: "matrix_ma1sd_threepid_medium_email_custom_templates_enabled|bool and item.value"
 
diff --git a/roles/matrix-ma1sd/tasks/validate_config.yml b/roles/matrix-ma1sd/tasks/validate_config.yml
index 691fe0ac..4ca25e7e 100644
--- a/roles/matrix-ma1sd/tasks/validate_config.yml
+++ b/roles/matrix-ma1sd/tasks/validate_config.yml
@@ -54,6 +54,7 @@
   when: "vars | dict2items | selectattr('key', 'match', item.old) | list | items2dict"
   with_items:
     - {'old': 'matrix_ma1sd_container_expose_port', 'new': '<superseded by matrix_ma1sd_container_http_host_bind_port>'}
+    - {'old': 'matrix_ma1sd_threepid_medium_email_custom_unbind_fraudulent_template', 'new': 'matrix_ma1sd_threepid_medium_email_custom_session_unbind_notification_template'}
 
 - name: (Deprecation) Catch and report mxisd variables
   fail:
diff --git a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
index 9a426c47..4579b839 100644
--- a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
+++ b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
@@ -57,14 +57,14 @@ threepid:
           {% if matrix_ma1sd_threepid_medium_email_custom_invite_template %}
           invite: '/var/ma1sd/invite-template.eml'
           {% endif %}
-          {% if matrix_ma1sd_threepid_medium_email_custom_session_validation_template or matrix_ma1sd_threepid_medium_email_custom_unbind_fraudulent_template %}
+          {% if matrix_ma1sd_threepid_medium_email_custom_session_validation_template or matrix_ma1sd_threepid_medium_email_custom_session_unbind_notification_template %}
           session:
             {% if matrix_ma1sd_threepid_medium_email_custom_session_validation_template %}
             validation: '/var/ma1sd/validate-template.eml'
             {% endif %}
-            {% if matrix_ma1sd_threepid_medium_email_custom_unbind_fraudulent_template %}
+            {% if matrix_ma1sd_threepid_medium_email_custom_session_unbind_notification_template %}
             unbind:
-              frandulent: '/var/ma1sd/unbind-fraudulent.eml'
+              notification: '/var/ma1sd/unbind-notification.eml'
             {% endif %}
           {% endif %}
           {% if matrix_ma1sd_threepid_medium_email_custom_matrixid_template %}

From f84c69c16415b8b37a47077be7c556a712eded4b Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 3 Jan 2021 07:35:13 +0200
Subject: [PATCH 163/179] Relocate custom ma1sd threepid email templates to
 config/

We used to store them in data/, but that seems inappropriate,
since it's just static configuration that the playbook can recreate.
---
 roles/matrix-ma1sd/defaults/main.yml       |  4 ++++
 roles/matrix-ma1sd/tasks/setup_install.yml | 16 ++++++++++++++--
 roles/matrix-ma1sd/templates/ma1sd.yaml.j2 |  8 ++++----
 3 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/roles/matrix-ma1sd/defaults/main.yml b/roles/matrix-ma1sd/defaults/main.yml
index 321292d1..c0faea67 100644
--- a/roles/matrix-ma1sd/defaults/main.yml
+++ b/roles/matrix-ma1sd/defaults/main.yml
@@ -90,9 +90,13 @@ matrix_ma1sd_dns_overwrite_homeserver_client_value: "http://matrix-synapse:8008"
 # https://github.com/ma1uta/ma1sd/blob/master/docs/threepids/notification/template-generator.md
 # https://github.com/ma1uta/ma1sd/tree/master/src/main/resources/threepids/email
 matrix_ma1sd_threepid_medium_email_custom_templates_enabled: false
+# Defaults to: https://github.com/ma1uta/ma1sd/blob/master/src/main/resources/threepids/email/invite-template.eml
 matrix_ma1sd_threepid_medium_email_custom_invite_template: ""
+# Defaults to: https://github.com/ma1uta/ma1sd/blob/master/src/main/resources/threepids/email/validate-template.eml
 matrix_ma1sd_threepid_medium_email_custom_session_validation_template: ""
+# Defaults to: https://github.com/ma1uta/ma1sd/blob/master/src/main/resources/threepids/email/unbind-notification.eml
 matrix_ma1sd_threepid_medium_email_custom_session_unbind_notification_template: ""
+# Defaults to: https://github.com/ma1uta/ma1sd/blob/master/src/main/resources/threepids/email/mxid-template.eml
 matrix_ma1sd_threepid_medium_email_custom_matrixid_template: ""
 
 # Controls whether the self-check feature should validate SSL certificates.
diff --git a/roles/matrix-ma1sd/tasks/setup_install.yml b/roles/matrix-ma1sd/tasks/setup_install.yml
index 7cd6b72c..58fef962 100644
--- a/roles/matrix-ma1sd/tasks/setup_install.yml
+++ b/roles/matrix-ma1sd/tasks/setup_install.yml
@@ -107,10 +107,10 @@
     owner: "{{ matrix_user_username }}"
     group: "{{ matrix_user_groupname }}"
 
-- name: Ensure custom templates are installed if any
+- name: Ensure custom email templates are installed, if any
   copy:
     content: "{{ item.value }}"
-    dest: "{{ matrix_ma1sd_data_path }}/{{ item.location }}"
+    dest: "{{ matrix_ma1sd_config_path }}/{{ item.location }}"
     mode: 0644
     owner: "{{ matrix_user_username }}"
     group: "{{ matrix_user_groupname }}"
@@ -121,6 +121,18 @@
     - {value: "{{ matrix_ma1sd_threepid_medium_email_custom_matrixid_template }}", location: 'mxid-template.eml'}
   when: "matrix_ma1sd_threepid_medium_email_custom_templates_enabled|bool and item.value"
 
+# Only cleaning up for people who define the respective templates
+- name: (Cleanup) Ensure custom email templates are not in data/ anymore (we've put them in config/)
+  file:
+    path: "{{ matrix_ma1sd_data_path }}/{{ item.location }}"
+    state: absent
+  with_items:
+    - {value: "{{ matrix_ma1sd_threepid_medium_email_custom_invite_template }}", location: 'invite-template.eml'}
+    - {value: "{{ matrix_ma1sd_threepid_medium_email_custom_session_validation_template }}", location: 'validate-template.eml'}
+    - {value: "{{ matrix_ma1sd_threepid_medium_email_custom_session_unbind_notification_template }}", location: 'unbind-notification.eml'}
+    - {value: "{{ matrix_ma1sd_threepid_medium_email_custom_matrixid_template }}", location: 'mxid-template.eml'}
+  when: "matrix_ma1sd_threepid_medium_email_custom_templates_enabled|bool and item.value"
+
 - name: Ensure matrix-ma1sd.service installed
   template:
     src: "{{ role_path }}/templates/systemd/matrix-ma1sd.service.j2"
diff --git a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
index 4579b839..59fb1744 100644
--- a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
+++ b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
@@ -55,21 +55,21 @@ threepid:
       generators:
         template:
           {% if matrix_ma1sd_threepid_medium_email_custom_invite_template %}
-          invite: '/var/ma1sd/invite-template.eml'
+          invite: '/etc/ma1sd/invite-template.eml'
           {% endif %}
           {% if matrix_ma1sd_threepid_medium_email_custom_session_validation_template or matrix_ma1sd_threepid_medium_email_custom_session_unbind_notification_template %}
           session:
             {% if matrix_ma1sd_threepid_medium_email_custom_session_validation_template %}
-            validation: '/var/ma1sd/validate-template.eml'
+            validation: '/etc/ma1sd/validate-template.eml'
             {% endif %}
             {% if matrix_ma1sd_threepid_medium_email_custom_session_unbind_notification_template %}
             unbind:
-              notification: '/var/ma1sd/unbind-notification.eml'
+              notification: '/etc/ma1sd/unbind-notification.eml'
             {% endif %}
           {% endif %}
           {% if matrix_ma1sd_threepid_medium_email_custom_matrixid_template %}
           generic:
-            matrixId: '/var/ma1sd/mxid-template.eml'
+            matrixId: '/etc/ma1sd/mxid-template.eml'
           {% endif %}
 {% endif %}
 

From 480563718129d63647b22da8b4a29d85b2086763 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 3 Jan 2021 07:36:09 +0200
Subject: [PATCH 164/179] Add support for custom ma1sd view sesion templates

---
 roles/matrix-ma1sd/defaults/main.yml       | 11 +++++++++++
 roles/matrix-ma1sd/tasks/setup_install.yml | 12 ++++++++++++
 roles/matrix-ma1sd/templates/ma1sd.yaml.j2 | 12 ++++++++++++
 3 files changed, 35 insertions(+)

diff --git a/roles/matrix-ma1sd/defaults/main.yml b/roles/matrix-ma1sd/defaults/main.yml
index c0faea67..28e3405c 100644
--- a/roles/matrix-ma1sd/defaults/main.yml
+++ b/roles/matrix-ma1sd/defaults/main.yml
@@ -83,6 +83,17 @@ matrix_ma1sd_dns_overwrite_enabled: false
 matrix_ma1sd_dns_overwrite_homeserver_client_name: "{{ matrix_server_fqn_matrix }}"
 matrix_ma1sd_dns_overwrite_homeserver_client_value: "http://matrix-synapse:8008"
 
+# Override the default session templates
+# To use this, fill in the template variables with the full desired template as a multi-line YAML variable
+#
+# More info:
+# https://github.com/ma1uta/ma1sd/blob/master/docs/threepids/session/session-views.md
+matrix_ma1sd_view_session_custom_templates_enabled: false
+# Defaults to: https://github.com/ma1uta/ma1sd/blob/master/src/main/resources/templates/session/tokenSubmitSuccess.html
+matrix_ma1sd_view_session_custom_onTokenSubmit_success_template: ""
+# Defaults to: https://github.com/ma1uta/ma1sd/blob/master/src/main/resources/templates/session/tokenSubmitFailure.html
+matrix_ma1sd_view_session_custom_onTokenSubmit_failure_template: ""
+
 # Override the default email templates
 # To use this, fill in the template variables with the full desired template as a multi-line YAML variable
 #
diff --git a/roles/matrix-ma1sd/tasks/setup_install.yml b/roles/matrix-ma1sd/tasks/setup_install.yml
index 58fef962..a4a70ea4 100644
--- a/roles/matrix-ma1sd/tasks/setup_install.yml
+++ b/roles/matrix-ma1sd/tasks/setup_install.yml
@@ -107,6 +107,18 @@
     owner: "{{ matrix_user_username }}"
     group: "{{ matrix_user_groupname }}"
 
+- name: Ensure custom view templates are installed, if any
+  copy:
+    content: "{{ item.value }}"
+    dest: "{{ matrix_ma1sd_config_path }}/{{ item.location }}"
+    mode: 0644
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+  with_items:
+    - {value: "{{ matrix_ma1sd_view_session_custom_onTokenSubmit_success_template }}", location: 'tokenSubmitSuccess.html'}
+    - {value: "{{ matrix_ma1sd_view_session_custom_onTokenSubmit_failure_template }}", location: 'tokenSubmitFailure.html'}
+  when: "matrix_ma1sd_view_session_custom_templates_enabled|bool and item.value"
+
 - name: Ensure custom email templates are installed, if any
   copy:
     content: "{{ item.value }}"
diff --git a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
index 59fb1744..a4100adc 100644
--- a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
+++ b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
@@ -73,6 +73,18 @@ threepid:
           {% endif %}
 {% endif %}
 
+{% if matrix_ma1sd_view_session_custom_templates_enabled %}
+view:
+  session:
+    onTokenSubmit:
+      {% if matrix_ma1sd_view_session_custom_onTokenSubmit_success_template %}
+      success: '/etc/ma1sd/tokenSubmitSuccess.html'
+      {% endif %}
+      {% if matrix_ma1sd_view_session_custom_onTokenSubmit_failure_template %}
+      failure: '/etc/ma1sd/tokenSubmitFailure.html'
+      {% endif %}
+{% endif %}
+
 {% if matrix_ma1sd_hashing_enabled %}
 hashing:
   enabled: true # enable or disable the hash lookup MSC2140 (default is false)

From df8d9cfd3406433b1a6e9b234c818b70c6460daa Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 3 Jan 2021 07:46:55 +0200
Subject: [PATCH 165/179] Remove some TODOs

The answer to these is: it's good to have them in both places.
The role defines the obvious things it depends on (not knowing
what setup it will find itself into), and then
`group_vars/matrix_servers` "extends" it based on everything else it
knows (the homeserver being Synapse, whether or not the internal
Postgres server is being used, etc.)
---
 group_vars/matrix_servers                     | 10 ++++------
 .../defaults/main.yml                         | 19 +++++++++----------
 2 files changed, 13 insertions(+), 16 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index c9065420..45260688 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -265,7 +265,6 @@ matrix_mautrix_hangouts_database_password: "{{ matrix_synapse_macaroon_secret_ke
 # We don't enable bridges by default.
 matrix_mautrix_signal_enabled: false
 
-# TODO: unclear whether to put this into the role or keep it here
 matrix_mautrix_signal_systemd_required_services_list: |
   {{
     ['docker.service']
@@ -288,8 +287,7 @@ matrix_mautrix_signal_appservice_token: "{{ matrix_synapse_macaroon_secret_key |
 matrix_mautrix_signal_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
 matrix_mautrix_signal_database_engine: 'postgres'
-matrix_mautrix_signal_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau.signal.db') | to_uuid }}" 
-
+matrix_mautrix_signal_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau.signal.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -1073,9 +1071,9 @@ matrix_postgres_additional_databases: |
     }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres') else [])
     +
     ([{
-      'name': matrix_mautrix_signal_database_name, 
-      'username': matrix_mautrix_signal_database_username, 
-      'password': matrix_mautrix_signal_database_password, 
+      'name': matrix_mautrix_signal_database_name,
+      'username': matrix_mautrix_signal_database_username,
+      'password': matrix_mautrix_signal_database_password,
     }] if (matrix_mautrix_signal_enabled and matrix_mautrix_signal_database_engine == 'postgres' and matrix_mautrix_signal_database_hostname == 'matrix-postgres') else [])
     +
     ([{
diff --git a/roles/matrix-bridge-mautrix-signal/defaults/main.yml b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
index d366ed57..d46c2c50 100644
--- a/roles/matrix-bridge-mautrix-signal/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
@@ -27,8 +27,7 @@ matrix_mautrix_signal_container_http_host_bind_port: ''
 matrix_mautrix_signal_container_extra_arguments: []
 
 # List of systemd services that matrix-mautrix-signal.service depends on.
-# TODO: unclear whether to put this into group_vars or keep it here
-matrix_mautrix_signal_systemd_required_services_list: 
+matrix_mautrix_signal_systemd_required_services_list:
   - 'docker.service'
   - 'matrix-mautrix-signal-daemon.service'
 
@@ -40,7 +39,7 @@ matrix_mautrix_signal_homeserver_token: ''
 
 # Database-related configuration fields
 #
-# This bridge only supports postgres. 
+# This bridge only supports postgres.
 #
 matrix_mautrix_signal_database_engine: 'postgres'
 
@@ -50,13 +49,13 @@ matrix_mautrix_signal_database_hostname: 'matrix-postgres'
 matrix_mautrix_signal_database_port: 5432
 matrix_mautrix_signal_database_name: 'matrix_mautrix_signal'
 
-matrix_mautrix_signal_database_connection_string: 'postgres://{{ matrix_mautrix_signal_database_username }}:{{ matrix_mautrix_signal_database_password }}@{{ matrix_mautrix_signal_database_hostname }}:{{ matrix_mautrix_signal_database_port }}/{{ matrix_mautrix_signal_database_name }}' 
+matrix_mautrix_signal_database_connection_string: 'postgres://{{ matrix_mautrix_signal_database_username }}:{{ matrix_mautrix_signal_database_password }}@{{ matrix_mautrix_signal_database_hostname }}:{{ matrix_mautrix_signal_database_port }}/{{ matrix_mautrix_signal_database_name }}'
 
-matrix_mautrix_signal_appservice_database: "{{ 
- 	{ 
- 		'postgres': matrix_mautrix_facebook_database_connection_string, 
- 	}[matrix_mautrix_signal_database_engine] 
- }}" 
+matrix_mautrix_signal_appservice_database: "{{
+ 	{
+ 		'postgres': matrix_mautrix_facebook_database_connection_string,
+ 	}[matrix_mautrix_signal_database_engine]
+ }}"
 
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mautrix_signal_login_shared_secret: ''
@@ -96,4 +95,4 @@ matrix_mautrix_signal_registration_yaml: "{{ lookup('template', 'templates/regis
 
 matrix_mautrix_signal_registration: "{{ matrix_mautrix_signal_registration_yaml|from_yaml }}"
 
-matrix_mautrix_signal_log_level: 'DEBUG'
\ No newline at end of file
+matrix_mautrix_signal_log_level: 'DEBUG'

From aac400664afeda5f93c13e93a75feacc01964add Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 3 Jan 2021 07:59:49 +0200
Subject: [PATCH 166/179] Announce mautrix-signal bridge

---
 CHANGELOG.md                                       | 7 +++++++
 README.md                                          | 4 ++++
 docs/configuring-playbook-bridge-mautrix-signal.md | 2 ++
 3 files changed, 13 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c4a586e8..b6d1f414 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,10 @@
+# 2021-01-03
+
+## Signal bridging support via mautrix-signal
+
+Thanks to [laszabine](https://github.com/laszabine)'s efforts, the playbook now supports bridging to [Signal](https://www.signal.org/) via the [mautrix-signal](https://github.com/tulir/mautrix-signal) bridge. See our [Setting up Mautrix Signal bridging](docs/configuring-playbook-bridge-mautrix-signal.md) documentation page for getting started.
+
+
 # 2020-12-23
 
 ## The big move to all-on-Postgres (potentially dangerous)
diff --git a/README.md b/README.md
index ac2b7ca9..1ddc5016 100644
--- a/README.md
+++ b/README.md
@@ -46,6 +46,8 @@ Using this playbook, you can get the following services configured on your serve
 
 - (optional) the [mautrix-hangouts](https://github.com/tulir/mautrix-hangouts) bridge for bridging your Matrix server to [Google Hangouts](https://en.wikipedia.org/wiki/Google_Hangouts)
 
+- (optional) the [mautrix-signal](https://github.com/tulir/mautrix-signal) bridge for bridging your Matrix server to [Signal](https://www.signal.org/)
+
 - (optional) the [matrix-appservice-irc](https://github.com/matrix-org/matrix-appservice-irc) bridge for bridging your Matrix server to [IRC](https://wikipedia.org/wiki/Internet_Relay_Chat)
 
 - (optional) the [matrix-appservice-discord](https://github.com/Half-Shot/matrix-appservice-discord) bridge for bridging your Matrix server to [Discord](https://discordapp.com/)
@@ -158,6 +160,8 @@ This playbook sets up your server using the following Docker images:
 
 - [tulir/mautrix-hangouts](https://hub.docker.com/r/tulir/mautrix-hangouts/) - the [mautrix-hangouts](https://github.com/tulir/mautrix-hangouts) bridge to [Google Hangouts](https://en.wikipedia.org/wiki/Google_Hangouts) (optional)
 
+- [tulir/mautrix-signal](https://mau.dev/tulir/mautrix-signal/container_registry) - the [mautrix-signal](https://github.com/tulir/mautrix-signal) bridge to [Signal](https://www.signal.org/) (optional)
+
 - [matrixdotorg/matrix-appservice-irc](https://hub.docker.com/r/matrixdotorg/matrix-appservice-irc) - the [matrix-appservice-irc](https://github.com/matrix-org/matrix-appservice-irc) bridge to [IRC](https://wikipedia.org/wiki/Internet_Relay_Chat) (optional)
 
 - [halfshot/matrix-appservice-discord](https://hub.docker.com/r/halfshot/matrix-appservice-discord) - the [matrix-appservice-discord](https://github.com/Half-Shot/matrix-appservice-discord) bridge to [Discord](https://discordapp.com/) (optional)
diff --git a/docs/configuring-playbook-bridge-mautrix-signal.md b/docs/configuring-playbook-bridge-mautrix-signal.md
index a5410665..cc6541b5 100644
--- a/docs/configuring-playbook-bridge-mautrix-signal.md
+++ b/docs/configuring-playbook-bridge-mautrix-signal.md
@@ -4,6 +4,8 @@ The playbook can install and configure [mautrix-signal](https://github.com/tulir
 
 See the project's [documentation](https://github.com/tulir/mautrix-signal/wiki) to learn what it does and why it might be useful to you.
 
+**Note/Prerequisite**: If you're running with the Postgres database server integrated by the playbook (which is the default), you don't need to do anything special and can easily proceed with installing. However, if you're [using an external Postgres server](configuring-playbook-external-postgres.md), you'd need to manually prepare a Postgres database for this bridge and adjust the variables related to that (`matrix_mautrix_signal_database_*`).
+
 Use the following playbook configuration:
 
 ```yaml

From da2a6682b39699d0e572a2fd5bcf0350844cd22b Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 3 Jan 2021 08:06:32 +0200
Subject: [PATCH 167/179] Get rid of
 matrix_mautrix_signal_configuration_permissions

While it's kind of nice having it, it's also somewhat raw
and unnecessary.

Having a good default and not even mentioning it seems better
for most users.

People who need a more exposed bridge (rare) can use
override the default configuration using
`matrix_mautrix_signal_configuration_extension_yaml`.
---
 ...nfiguring-playbook-bridge-mautrix-signal.md | 18 ------------------
 .../defaults/main.yml                          |  9 ---------
 .../tasks/validate_config.yml                  |  1 +
 .../templates/config.yaml.j2                   |  3 ++-
 4 files changed, 3 insertions(+), 28 deletions(-)

diff --git a/docs/configuring-playbook-bridge-mautrix-signal.md b/docs/configuring-playbook-bridge-mautrix-signal.md
index cc6541b5..164b06de 100644
--- a/docs/configuring-playbook-bridge-mautrix-signal.md
+++ b/docs/configuring-playbook-bridge-mautrix-signal.md
@@ -12,24 +12,6 @@ Use the following playbook configuration:
 matrix_mautrix_signal_enabled: true
 ```
 
-To specify which users have access to the bridge, use the variable `matrix_mautrix_signal_configuration_permissions`.
-Refer to the documentation for
-```yaml
-bridge:
-  permissions:
-```
-in [the example config in mautrix-signal](https://github.com/tulir/mautrix-signal/blob/master/mautrix_signal/example-config.yaml).
-For instance, use
-```yaml
-matrix_mautrix_signal_configuration_permissions: |
-  {
-    '{{ matrix_domain }}': 'user'
-  }
-```
-to allow all users registered to the current host's matrix domain access to the bridge, or hard-code whatever you like.
-(See [this issue](https://github.com/ansible/ansible/issues/17324#issuecomment-449642731) on how to use variable names as dictionary keys.)
-
-
 ## Set up Double Puppeting
 
 If you'd like to use [Double Puppeting](https://github.com/tulir/mautrix-whatsapp/wiki/Authentication#replacing-whatsapp-accounts-matrix-puppet-with-matrix-account) (hint: you most likely do), you have 2 ways of going about it.
diff --git a/roles/matrix-bridge-mautrix-signal/defaults/main.yml b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
index d46c2c50..d15ba6e3 100644
--- a/roles/matrix-bridge-mautrix-signal/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
@@ -67,15 +67,6 @@ matrix_mautrix_signal_login_shared_secret: ''
 # or completely replace this variable with your own template.
 matrix_mautrix_signal_configuration_yaml: "{{ lookup('template', 'templates/config.yaml.j2') }}"
 
-# Permitted values:
-#       user - Use the bridge with puppeting.
-#      admin - Use and administrate the bridge.
-# Permitted keys:
-#        * - All Matrix users
-#   domain - All users on that homeserver
-#     mxid - Specific user
-matrix_mautrix_signal_configuration_permissions: {}
-
 matrix_mautrix_signal_configuration_extension_yaml: |
   # Your custom YAML configuration goes here.
   # This configuration extends the default starting configuration (`matrix_mautrix_signal_configuration_yaml`).
diff --git a/roles/matrix-bridge-mautrix-signal/tasks/validate_config.yml b/roles/matrix-bridge-mautrix-signal/tasks/validate_config.yml
index 397ef4d8..100af3f8 100644
--- a/roles/matrix-bridge-mautrix-signal/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mautrix-signal/tasks/validate_config.yml
@@ -25,3 +25,4 @@
     - {'old': 'matrix_mautrix_signal_db_host', 'new': 'matrix_mautrix_signal_database_hostname'}
     - {'old': 'matrix_mautrix_signal_db_port', 'new': 'matrix_mautrix_signal_database_port'}
     - {'old': 'matrix_mautrix_signal_db_url', 'new': 'matrix_mautrix_signal_database_connection_string'}
+    - {'old': 'matrix_mautrix_signal_configuration_permissions', 'new': '<superseded by matrix_mautrix_signal_configuration_extension_yaml>'}
diff --git a/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2
index 4335b023..68ed1d6d 100644
--- a/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2
@@ -172,7 +172,8 @@ bridge:
     #        * - All Matrix users
     #   domain - All users on that homeserver
     #     mxid - Specific user
-    permissions: {{ matrix_mautrix_signal_configuration_permissions }} 
+    permissions:
+      '{{ matrix_mautrix_signal_homeserver_domain }}': user
 
 
 # Python logging configuration.

From 274f23f66868b72eea753569ee31c6374a4c7686 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 3 Jan 2021 08:16:49 +0200
Subject: [PATCH 168/179] Make matrix-mautrix-signal-daemon.service depend on
 docker.service

---
 roles/matrix-bridge-mautrix-signal/defaults/main.yml     | 6 ++++++
 .../systemd/matrix-mautrix-signal-daemon.service.j2      | 9 +++++++++
 2 files changed, 15 insertions(+)

diff --git a/roles/matrix-bridge-mautrix-signal/defaults/main.yml b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
index d15ba6e3..962140c9 100644
--- a/roles/matrix-bridge-mautrix-signal/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-signal/defaults/main.yml
@@ -34,6 +34,12 @@ matrix_mautrix_signal_systemd_required_services_list:
 # List of systemd services that matrix-mautrix-signal.service wants
 matrix_mautrix_signal_systemd_wanted_services_list: []
 
+# List of systemd services that matrix-mautrix-signal-daemon.service depends on.
+matrix_mautrix_signal_daemon_systemd_required_services_list: ['docker.service']
+
+# List of systemd services that matrix-mautrix-signal-daemon.service wants
+matrix_mautrix_signal_daemon_systemd_wanted_services_list: []
+
 matrix_mautrix_signal_appservice_token: ''
 matrix_mautrix_signal_homeserver_token: ''
 
diff --git a/roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal-daemon.service.j2 b/roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal-daemon.service.j2
index 499ee309..d8952992 100644
--- a/roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal-daemon.service.j2
+++ b/roles/matrix-bridge-mautrix-signal/templates/systemd/matrix-mautrix-signal-daemon.service.j2
@@ -2,6 +2,15 @@
 [Unit]
 Description=Matrix Mautrix Signal daemon
 
+{% for service in matrix_mautrix_signal_daemon_systemd_required_services_list %}
+Requires={{ service }}
+After={{ service }}
+{% endfor %}
+
+{% for service in matrix_mautrix_signal_daemon_systemd_wanted_services_list %}
+Wants={{ service }}
+{% endfor %}
+
 [Service]
 Type=simple
 

From 3b524ee815c5123a9d15f5bf8478bc40b705d80a Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 3 Jan 2021 08:20:43 +0200
Subject: [PATCH 169/179] Make mautrix-signal bridge not log to files

We try to only use console logging (going to journald) for everything,
instead of logging things twice (or more).
---
 .../matrix-bridge-mautrix-signal/templates/config.yaml.j2 | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2
index 68ed1d6d..28fff6f0 100644
--- a/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mautrix-signal/templates/config.yaml.j2
@@ -189,12 +189,6 @@ logging:
         normal:
             format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s"
     handlers:
-        file:
-            class: logging.handlers.RotatingFileHandler
-            formatter: normal
-            filename: ./mautrix-signal.log
-            maxBytes: 10485760
-            backupCount: 10
         console:
             class: logging.StreamHandler
             formatter: colored
@@ -205,4 +199,4 @@ logging:
             level: INFO
     root:
         level: {{ matrix_mautrix_signal_log_level }}
-        handlers: [file, console]
+        handlers: [console]

From 6e652e10ad33027425ff5ea97b5e3089cbccf1ea Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 3 Jan 2021 08:48:29 +0200
Subject: [PATCH 170/179] Bump minimium Ansible version to 2.7.1

Provoked by https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/743
---
 docs/ansible.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/ansible.md b/docs/ansible.md
index b5135639..3c062a75 100644
--- a/docs/ansible.md
+++ b/docs/ansible.md
@@ -9,7 +9,7 @@ If your local computer cannot run Ansible, you can also run Ansible on some serv
 
 ## Supported Ansible versions
 
-Ansible 2.7.0 or newer is required.
+Ansible 2.7.1 or newer is required ([last discussion about Ansible versions](https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/743)).
 
 Note: Ubuntu 20.04 ships with Ansible 2.9.6 which is a buggy version (see this [bug](https://bugs.launchpad.net/ubuntu/+source/ansible/+bug/1880359)), which can't be used in combination with a host running new systemd (more details in [#517](https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/517), [#669](https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/669)). If this problem affects you, you can: avoid running Ubuntu 20.04 on your host; run Ansible from another machine targeting your host; or try to upgrade to a newer Ansible version (see below).
 

From 2c09111a3a3dba37f849a40022d7d710593f0c9d Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 3 Jan 2021 08:53:54 +0200
Subject: [PATCH 171/179] Actually enforce that we run on Ansible >= 2.7.1

Related to 6e652e10ad33
---
 roles/matrix-base/tasks/sanity_check.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/roles/matrix-base/tasks/sanity_check.yml b/roles/matrix-base/tasks/sanity_check.yml
index 59ab8b93..3372d749 100644
--- a/roles/matrix-base/tasks/sanity_check.yml
+++ b/roles/matrix-base/tasks/sanity_check.yml
@@ -1,10 +1,11 @@
 ---
 
-# We generally support Ansible 2.7.0 and above.
-- name: Fail if running on Ansible < 2.7
+# We generally support Ansible 2.7.1 and above.
+- name: Fail if running on Ansible < 2.7.1
   fail:
     msg: "You are running on Ansible {{ ansible_version.string }}, which is not supported. See our guide about Ansible: https://github.com/spantaleev/matrix-docker-ansible-deploy/blob/master/docs/ansible.md"
-  when: "(ansible_version.major < 2) or (ansible_version.major <= 2 and ansible_version.minor < 7)"
+  when:
+    - "(ansible_version.major < 2) or (ansible_version.major == 2 and ansible_version.minor < 7) or (ansible_version.major == 2 and ansible_version.minor == 7 and ansible_version.revision < 1)"
 
 # Though we do not support Ansible 2.9.6 which is buggy
 - name: Fail if running on Ansible 2.9.6 on Ubuntu

From 6cce5383bcf1f34c06af2213c16954246060f565 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 3 Jan 2021 08:55:30 +0200
Subject: [PATCH 172/179] Fix Ansible 2.9.6 check

Fixup for https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/769
---
 roles/matrix-base/tasks/sanity_check.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-base/tasks/sanity_check.yml b/roles/matrix-base/tasks/sanity_check.yml
index 3372d749..c31d9137 100644
--- a/roles/matrix-base/tasks/sanity_check.yml
+++ b/roles/matrix-base/tasks/sanity_check.yml
@@ -13,7 +13,7 @@
     msg: "You are running on Ansible {{ ansible_version.string }}, which is not supported. See our guide about Ansible: https://github.com/spantaleev/matrix-docker-ansible-deploy/blob/master/docs/ansible.md"
   when:
     - ansible_distribution == 'Ubuntu'
-    - "ansible_version.major == 2 and ansible_version.major == 9 and ansible_version.minor == 6"
+    - "ansible_version.major == 2 and ansible_version.minor == 9 and ansible_version.revision == 6"
 
 - name: (Deprecation) Catch and report renamed settings
   fail:

From 23f246b0ad45fe414c4f88f8246928b6f7ab1cc6 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 3 Jan 2021 09:19:06 +0200
Subject: [PATCH 173/179] Mention mautrix-signal migration steps to early
 adopters

---
 CHANGELOG.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b6d1f414..1aa8c9b5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,8 @@
 
 Thanks to [laszabine](https://github.com/laszabine)'s efforts, the playbook now supports bridging to [Signal](https://www.signal.org/) via the [mautrix-signal](https://github.com/tulir/mautrix-signal) bridge. See our [Setting up Mautrix Signal bridging](docs/configuring-playbook-bridge-mautrix-signal.md) documentation page for getting started.
 
+If you had installed the mautrix-signal bridge while its Pull Request was still work-in-progress, you can migrate your data to the new and final setup by referring to [this comment](https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/686#issuecomment-753510789).
+
 
 # 2020-12-23
 

From bcb408497d0ceb88dc13cbd4a534839be6e3b3a0 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 3 Jan 2021 10:18:53 +0200
Subject: [PATCH 174/179] Fix broken links (mautrix bridges are not on Docker
 Hub)

---
 README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 1ddc5016..5a9278f3 100644
--- a/README.md
+++ b/README.md
@@ -152,13 +152,13 @@ This playbook sets up your server using the following Docker images:
 
 - [certbot/certbot](https://hub.docker.com/r/certbot/certbot/) - the [certbot](https://certbot.eff.org/) tool for obtaining SSL certificates from [Let's Encrypt](https://letsencrypt.org/) (optional)
 
-- [tulir/mautrix-telegram](https://hub.docker.com/r/tulir/mautrix-telegram/) - the [mautrix-telegram](https://github.com/tulir/mautrix-telegram) bridge to [Telegram](https://telegram.org/) (optional)
+- [tulir/mautrix-telegram](https://mau.dev/tulir/mautrix-telegram/container_registry) - the [mautrix-telegram](https://github.com/tulir/mautrix-telegram) bridge to [Telegram](https://telegram.org/) (optional)
 
-- [tulir/mautrix-whatsapp](https://hub.docker.com/r/tulir/mautrix-whatsapp/) - the [mautrix-whatsapp](https://github.com/tulir/mautrix-whatsapp) bridge to [Whatsapp](https://www.whatsapp.com/) (optional)
+- [tulir/mautrix-whatsapp](https://mau.dev/tulir/mautrix-whatsapp/container_registry) - the [mautrix-whatsapp](https://github.com/tulir/mautrix-whatsapp) bridge to [Whatsapp](https://www.whatsapp.com/) (optional)
 
-- [tulir/mautrix-facebook](https://hub.docker.com/r/tulir/mautrix-facebook/) - the [mautrix-facebook](https://github.com/tulir/mautrix-facebook) bridge to [Facebook](https://facebook.com/) (optional)
+- [tulir/mautrix-facebook](https://mau.dev/tulir/mautrix-facebook/container_registry) - the [mautrix-facebook](https://github.com/tulir/mautrix-facebook) bridge to [Facebook](https://facebook.com/) (optional)
 
-- [tulir/mautrix-hangouts](https://hub.docker.com/r/tulir/mautrix-hangouts/) - the [mautrix-hangouts](https://github.com/tulir/mautrix-hangouts) bridge to [Google Hangouts](https://en.wikipedia.org/wiki/Google_Hangouts) (optional)
+- [tulir/mautrix-hangouts](https://mau.dev/tulir/mautrix-hangouts/container_registry) - the [mautrix-hangouts](https://github.com/tulir/mautrix-hangouts) bridge to [Google Hangouts](https://en.wikipedia.org/wiki/Google_Hangouts) (optional)
 
 - [tulir/mautrix-signal](https://mau.dev/tulir/mautrix-signal/container_registry) - the [mautrix-signal](https://github.com/tulir/mautrix-signal) bridge to [Signal](https://www.signal.org/) (optional)
 

From 6b1e25d84327cedeca7de1e4d962779f9ac4db9c Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 3 Jan 2021 10:21:59 +0200
Subject: [PATCH 175/179] Do not create matrix_bridge_sms Postgres database

That bridge only supports its own file-based database,
so preparing a Postgres database for it is pointless.
---
 group_vars/matrix_servers | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 45260688..dabe0d75 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -1088,12 +1088,6 @@ matrix_postgres_additional_databases: |
       'password': matrix_mautrix_whatsapp_database_password,
     }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres') else [])
     +
-    ([{
-      'name': 'matrix_bridge_sms',
-      'username': 'matrix_bridge_sms',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge.sms.db') | to_uuid,
-    }] if matrix_sms_bridge_enabled else [])
-    +
     ([{
       'name': matrix_mx_puppet_skype_database_name,
       'username': matrix_mx_puppet_skype_database_username,

From 24d73a554d35fc009f2b6f304c194db9b2431fcd Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 4 Jan 2021 23:46:34 +0200
Subject: [PATCH 176/179] Fix incorrect federation port in Traefik example

---
 docs/configuring-playbook-own-webserver.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuring-playbook-own-webserver.md b/docs/configuring-playbook-own-webserver.md
index c930da40..915c2c37 100644
--- a/docs/configuring-playbook-own-webserver.md
+++ b/docs/configuring-playbook-own-webserver.md
@@ -219,7 +219,7 @@ services:
       - "--certificatesresolvers.default.acme.storage=/letsencrypt/acme.json"
     ports:
       - "443:443"
-      - "8080:8080"
+      - "8448:8448"
     volumes:
       - "./letsencrypt:/letsencrypt"
       - "/var/run/docker.sock:/var/run/docker.sock:ro"

From 0a977ce113d07e9446821664eaec989943f32f88 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 5 Jan 2021 22:59:24 +0200
Subject: [PATCH 177/179] Fix incorrect path in docs

---
 docs/maintenance-postgres.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/maintenance-postgres.md b/docs/maintenance-postgres.md
index 14d5cdb9..93d0d0bf 100644
--- a/docs/maintenance-postgres.md
+++ b/docs/maintenance-postgres.md
@@ -69,7 +69,7 @@ This playbook can upgrade your existing Postgres setup with the following comman
 
 	ansible-playbook -i inventory/hosts setup.yml --tags=upgrade-postgres
 
-**The old Postgres data directory is backed up** automatically, by renaming it to `/matrix/postgres-auto-upgrade-backup`.
+**The old Postgres data directory is backed up** automatically, by renaming it to `/matrix/postgres/data-auto-upgrade-backup`.
 To rename to a different path, pass some extra flags to the command above, like this: `--extra-vars="postgres_auto_upgrade_backup_data_path=/another/disk/matrix-postgres-before-upgrade"`
 
 The auto-upgrade-backup directory stays around forever, until you **manually decide to delete it**.

From a2f6adbeaf1f064d1c67dbf6958f31044754309b Mon Sep 17 00:00:00 2001
From: teutat3s <10206665+teutat3s@users.noreply.github.com>
Date: Wed, 6 Jan 2021 22:00:49 +0100
Subject: [PATCH 178/179] Fix appservice-slack default db: nedb

---
 group_vars/matrix_servers | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index dabe0d75..6111600e 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -129,7 +129,7 @@ matrix_appservice_slack_systemd_required_services_list: |
   }}
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
-matrix_appservice_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_appservice_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'nedb' }}"
 matrix_appservice_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.slack.db') | to_uuid }}"
 
 ######################################################################

From d7dba0a2ffdd939a01eda0c2640c9c8acf0396da Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Fri, 8 Jan 2021 16:37:30 +0200
Subject: [PATCH 179/179] Update some Docker image references

---
 docs/ansible.md              | 2 +-
 docs/maintenance-postgres.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/ansible.md b/docs/ansible.md
index 3c062a75..ff513bbf 100644
--- a/docs/ansible.md
+++ b/docs/ansible.md
@@ -51,7 +51,7 @@ docker run -it --rm \
 -v `pwd`:/work \
 -v $HOME/.ssh/id_rsa:/root/.ssh/id_rsa:ro \
 --entrypoint=/bin/sh \
-devture/ansible:2.9.13-r0
+docker.io/devture/ansible:2.9.14-r0
 ```
 
 The above command tries to mount an SSH key (`$HOME/.ssh/id_rsa`) into the container (at `/root/.ssh/id_rsa`).
diff --git a/docs/maintenance-postgres.md b/docs/maintenance-postgres.md
index 93d0d0bf..7c936479 100644
--- a/docs/maintenance-postgres.md
+++ b/docs/maintenance-postgres.md
@@ -45,7 +45,7 @@ docker run \
 --log-driver=none \
 --network=matrix \
 --env-file=/matrix/postgres/env-postgres-psql \
-postgres:13.0-alpine \
+docker.io/postgres:13.1-alpine \
 pg_dumpall -h matrix-postgres \
 | gzip -c \
 > /postgres.sql.gz