From ab1a9fd87e4f9a424abe381bfd30bf09243eb266 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 8 Sep 2017 16:53:24 +0300 Subject: [PATCH] Add support for using an external PostgreSQL server --- README.md | 29 +++++++++- roles/matrix-server/defaults/main.yml | 6 +- .../matrix-server/tasks/import_sqlite_db.yml | 2 +- roles/matrix-server/tasks/setup_postgres.yml | 57 ++++++++++++++++--- roles/matrix-server/tasks/setup_synapse.yml | 9 ++- .../systemd/matrix-synapse.service.j2 | 6 +- .../usr-local-bin/matrix-postgres-cli.j2 | 6 +- 7 files changed, 95 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 16fea42c..620c4e1f 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Using this playbook, you can get the following services configured on your serve - (optional) [Amazon S3](https://aws.amazon.com/s3/) storage for your Matrix Synapse's content repository (`media_store`) files using [s3fs-fuse](https://github.com/s3fs-fuse/s3fs-fuse) -- a [PostgreSQL](https://www.postgresql.org/) database for Matrix Synapse - providing better performance than the default [SQLite](https://sqlite.org/) database +- (optional default) [PostgreSQL](https://www.postgresql.org/) database for Matrix Synapse - providing better performance than the default [SQLite](https://sqlite.org/) database. Using an external PostgreSQL server [is possible](#using-an-external-postgresql-server-optional) as well - a [STUN/TURN server](https://github.com/coturn/coturn) for WebRTC audio/video calls @@ -35,7 +35,9 @@ This is similar to the [EMnify/matrix-synapse-auto-deploy](https://github.com/EM - this one retrieves and automatically renews free [Let's Encrypt](https://letsencrypt.org/) **SSL certificates** for you -- this one optionally can store the `media_store` content repository files on [Amazon S3](https://aws.amazon.com/s3/) +- this one optionally can store the `media_store` content repository files on [Amazon S3](https://aws.amazon.com/s3/) (but defaults to storing files on the server's filesystem) + +- this one optionally allows you to use an external PostgreSQL server for Matrix Synapse's database (but defaults to running one in a container) Special thanks goes to: @@ -97,6 +99,9 @@ You can follow these steps: ## Amazon S3 configuration (optional) +By default, this playbook configures your server to store Matrix Synapse's content repository (`media_store`) files on the local filesystem. +If that's alright, you can skip ahead. + If you'd like to store Matrix Synapse's content repository (`media_store`) files on Amazon S3, you can let this playbook configure [s3fs-fuse](https://github.com/s3fs-fuse/s3fs-fuse) for you. @@ -131,6 +136,26 @@ matrix_s3_media_store_aws_secret_key: "secret-key-goes-here" ``` +## Using an external PostgreSQL server (optional) + +By default, this playbook would set up a PostgreSQL database server on your machine, running in a Docker container. +If that's alright, you can skip ahead. + +If you'd like to use an external PostgreSQL server that you manage, you can edit your configuration file (`inventory/matrix./vars.yml`). +It should be something like this: + +``` +matrix_postgres_use_external: true +matrix_postgres_connection_hostname: "your-postgres-server-hostname" +matrix_postgres_connection_username: "your-postgres-server-username" +matrix_postgres_connection_password: "your-postgres-server-password" +matrix_postgres_db_name: "your-postgres-server-database-name" +``` + +The database (as specified in `matrix_postgres_db_name`) must exist and be accessible with the given credentials. +It must be empty or contain a valid Matrix Synapse database. If empty, Matrix Synapse would populate it the first time it runs. + + ## Installing Once you have your server and you have [configured your DNS records](#configuring-dns), you can proceed with installing. diff --git a/roles/matrix-server/defaults/main.yml b/roles/matrix-server/defaults/main.yml index 47910501..8ae0ff85 100644 --- a/roles/matrix-server/defaults/main.yml +++ b/roles/matrix-server/defaults/main.yml @@ -18,6 +18,10 @@ matrix_user_username: "matrix" matrix_user_uid: 991 matrix_user_gid: 991 +# The defaults below cause a postgres server to be configured (running within a container). +# Using an external server is possible by tweaking all of the parameters below. +matrix_postgres_use_external: false +matrix_postgres_connection_hostname: "postgres" matrix_postgres_connection_username: "synapse" matrix_postgres_connection_password: "synapse-password" matrix_postgres_db_name: "homeserver" @@ -57,4 +61,4 @@ matrix_max_log_files_count: 10 matrix_s3_media_store_enabled: false matrix_s3_media_store_bucket_name: "your-bucket-name" matrix_s3_media_store_aws_access_key: "your-aws-access-key" -matrix_s3_media_store_aws_secret_key: "your-aws-secret-key" \ No newline at end of file +matrix_s3_media_store_aws_secret_key: "your-aws-secret-key" diff --git a/roles/matrix-server/tasks/import_sqlite_db.yml b/roles/matrix-server/tasks/import_sqlite_db.yml index 0390e195..4bd860d5 100644 --- a/roles/matrix-server/tasks/import_sqlite_db.yml +++ b/roles/matrix-server/tasks/import_sqlite_db.yml @@ -72,7 +72,7 @@ - "{{ matrix_scratchpad_dir }}:/scratchpad" - "{{ matrix_scratchpad_dir }}/synapse_port_db_with_patch:/usr/local/bin/synapse_port_db_with_patch" links: - - "matrix-postgres:postgres" + - "matrix-postgres:{{ matrix_postgres_connection_hostname }}" - name: Ensure scratchpad directory is deleted file: diff --git a/roles/matrix-server/tasks/setup_postgres.yml b/roles/matrix-server/tasks/setup_postgres.yml index 94fad7b8..04b186fa 100644 --- a/roles/matrix-server/tasks/setup_postgres.yml +++ b/roles/matrix-server/tasks/setup_postgres.yml @@ -1,13 +1,10 @@ --- -- name: Ensure postgres data path exists - file: - path: "{{ matrix_postgres_data_path }}" - state: directory - mode: 0700 - owner: "{{ matrix_user_username }}" - group: "{{ matrix_user_username }}" +# +# Generic tasks, no matter what kind of server we're using (internal/external) +# +# Even if we don't run the internal server, we still need this for running the CLI - name: Ensure postgres Docker image is pulled docker_image: name: "{{ docker_postgres_image }}" @@ -27,8 +24,52 @@ dest: "/usr/local/bin/matrix-postgres-cli" mode: 0750 +# +# Tasks related to setting up an internal postgres server +# + +- name: Ensure postgres data path exists + file: + path: "{{ matrix_postgres_data_path }}" + state: directory + mode: 0700 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_username }}" + when: "not matrix_postgres_use_external" + - name: Ensure matrix-postgres.service installed template: src: "{{ role_path }}/templates/systemd/matrix-postgres.service.j2" dest: "/etc/systemd/system/matrix-postgres.service" - mode: 0644 \ No newline at end of file + mode: 0644 + when: "not matrix_postgres_use_external" + +# +# Tasks related to getting rid of the internal postgres server (if it was previously enabled) +# + +- name: Check existance of matrix-postgres service + stat: path="/etc/systemd/system/matrix-postgres.service" + register: matrix_postgres_service_stat + when: matrix_postgres_use_external + +- name: Ensure matrix-postgres is stopped + service: name=matrix-postgres state=stopped daemon_reload=yes + when: "matrix_postgres_use_external and matrix_postgres_service_stat.stat.exists" + +- name: Ensure matrix-postgres.service doesn't exist + file: + path: "/etc/systemd/system/matrix-postgres.service" + state: absent + when: "matrix_postgres_use_external and matrix_postgres_service_stat.stat.exists" + +- name: Check existance of matrix-postgres local data path + stat: path="{{ matrix_postgres_data_path }}" + register: matrix_postgres_data_path_stat + when: matrix_postgres_use_external + +# We just want to notify the user. Deleting data is too destructive. +- name: Notify if matrix-postgres local data remains + debug: + msg: "Note: You are not using a local PostgreSQL database, but some old data remains from before in {{ matrix_postgres_data_path }}. Feel free to delete that." + when: "matrix_postgres_use_external and matrix_postgres_data_path_stat.stat.exists" \ No newline at end of file diff --git a/roles/matrix-server/tasks/setup_synapse.yml b/roles/matrix-server/tasks/setup_synapse.yml index e53dbc50..64cae5fb 100644 --- a/roles/matrix-server/tasks/setup_synapse.yml +++ b/roles/matrix-server/tasks/setup_synapse.yml @@ -109,12 +109,11 @@ line: '\1name: "psycopg2"' backrefs: yes -- name: Augment Matrix config (add the Postgres connection parameters) - lineinfile: +- name: Augment Matrix config (set the Postgres connection parameters) + replace: dest: "{{ matrix_synapse_config_dir_path }}/homeserver.yaml" - regexp: '(.*)database: "(.*)homeserver.db"' - line: '\1user: "{{ matrix_postgres_connection_username }}"\n\1password: "{{ matrix_postgres_connection_password }}"\n\1database: "homeserver"\n\1host: "postgres"\n\1cp_min: 5\n\1cp_max: 10' - backrefs: yes + regexp: '(.*)name: "psycopg2"((?:.|\n)*?)\n\n' + replace: '\1name: "psycopg2"\n\1args:\n\1\1user: "{{ matrix_postgres_connection_username }}"\n\1\1password: "{{ matrix_postgres_connection_password }}"\n\1\1database: "{{ matrix_postgres_db_name }}"\n\1\1host: "{{ matrix_postgres_connection_hostname }}"\n\1\1cp_min: 5\n\1\1cp_max: 10\n\n' - name: Augment Matrix config (configure Coturn) lineinfile: "dest={{ matrix_synapse_config_dir_path }}/turnserver.conf" diff --git a/roles/matrix-server/templates/systemd/matrix-synapse.service.j2 b/roles/matrix-server/templates/systemd/matrix-synapse.service.j2 index 8752f1c3..6690a1fe 100644 --- a/roles/matrix-server/templates/systemd/matrix-synapse.service.j2 +++ b/roles/matrix-server/templates/systemd/matrix-synapse.service.j2 @@ -2,8 +2,10 @@ Description=Matrix Synapse server After=docker.service Requires=docker.service +{% if not matrix_postgres_use_external %} Requires=matrix-postgres.service After=matrix-postgres.service +{% endif %} {% if matrix_s3_media_store_enabled %} After=matrix-s3fs.service Requires=matrix-s3fs.service @@ -15,7 +17,9 @@ ExecStartPre=-/usr/bin/docker kill matrix-synapse ExecStartPre=-/usr/bin/docker rm matrix-synapse ExecStartPre=-/usr/bin/chown {{ matrix_user_username }}:{{ matrix_user_username }} {{ ssl_certs_path }} -R ExecStart=/usr/bin/docker run --rm --name matrix-synapse \ - --link matrix-postgres:postgres \ + {% if not matrix_postgres_use_external %} + --link matrix-postgres:{{ matrix_postgres_connection_hostname }} \ + {% endif %} -p 8448:8448 \ -p 3478:3478 \ -p 3478:3478/udp \ diff --git a/roles/matrix-server/templates/usr-local-bin/matrix-postgres-cli.j2 b/roles/matrix-server/templates/usr-local-bin/matrix-postgres-cli.j2 index 383a78fc..2e6a15ce 100644 --- a/roles/matrix-server/templates/usr-local-bin/matrix-postgres-cli.j2 +++ b/roles/matrix-server/templates/usr-local-bin/matrix-postgres-cli.j2 @@ -4,6 +4,8 @@ docker run \ -it \ --rm \ --env-file={{ matrix_environment_variables_data_path }}/env-postgres-pgsql-docker \ - --link=matrix-postgres:postgres \ + {% if not matrix_postgres_use_external %} + --link=matrix-postgres:{{ matrix_postgres_connection_hostname }} \ + {% endif %} {{ docker_postgres_image }} \ - psql -h postgres + psql -h {{ matrix_postgres_connection_hostname }} \ No newline at end of file