From 715bdf2c6487c56be5ad17a4e7b80d94efa46ce2 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 22 Dec 2020 19:32:43 +0200 Subject: [PATCH] Add support for automatic (nedb -> Postgres) migration to mx-appservice-irc --- group_vars/matrix_servers | 3 +- roles/matrix-base/defaults/main.yml | 1 + .../defaults/main.yml | 17 +++-- .../tasks/migrate_nedb_to_postgres.yml | 53 ++++++++++++++++ .../tasks/setup_install.yml | 63 ++++++++++++------- .../tasks/validate_config.yml | 20 ------ .../templates/config.yaml.j2 | 4 +- roles/matrix-postgres/tasks/import_nedb.yml | 21 +------ roles/matrix-postgres/tasks/main.yml | 8 --- 9 files changed, 109 insertions(+), 81 deletions(-) create mode 100644 roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 318ebac7..4ddadb43 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -181,8 +181,7 @@ matrix_appservice_irc_appservice_token: "{{ matrix_synapse_macaroon_secret_key | matrix_appservice_irc_homeserver_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'irc.hs.token') | to_uuid }}" -# Postgres is the default, except if not using `matrix_postgres` (internal postgres) -matrix_appservice_irc_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_appservice_irc_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'nedb' }}" matrix_appservice_irc_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.irc.db') | to_uuid }}" diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index e0522ba8..d0fdcdf8 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -106,6 +106,7 @@ matrix_docker_package_name: docker-ce run_postgres_import: true run_postgres_upgrade: true run_postgres_import_sqlite_db: true +run_postgres_import_nedb: true run_postgres_vacuum: true run_synapse_register_user: true run_synapse_update_user_password: true diff --git a/roles/matrix-bridge-appservice-irc/defaults/main.yml b/roles/matrix-bridge-appservice-irc/defaults/main.yml index 5110fd85..0b671e76 100644 --- a/roles/matrix-bridge-appservice-irc/defaults/main.yml +++ b/roles/matrix-bridge-appservice-irc/defaults/main.yml @@ -23,12 +23,17 @@ matrix_appservice_irc_database_hostname: 'matrix-postgres' matrix_appservice_irc_database_port: 5432 matrix_appservice_irc_database_name: matrix_appservice_irc -matrix_appservice_irc_database_connString: >-2 - {%- if matrix_appservice_irc_database_engine == 'postgres' -%} - postgresql://{{ matrix_appservice_irc_database_username }}:{{ matrix_appservice_irc_database_password }}@{{ matrix_appservice_irc_database_hostname }}:{{ matrix_appservice_irc_database_port }}/{{ matrix_appservice_irc_database_name }}?sslmode=disable - {%- elif matrix_appservice_irc_database_engine == 'nedb' -%} - {{ matrix_appservice_irc_database_engine }}://{{ matrix_appservice_irc_database_file }} - {%- endif -%} +# This is just the Postgres connection string, if Postgres is used. +# Naming clashes with `matrix_appservice_irc_database_connectionString` somewhat. +matrix_appservice_irc_database_connection_string: 'postgresql://{{ matrix_appservice_irc_database_username }}:{{ matrix_appservice_irc_database_password }}@{{ matrix_appservice_irc_database_hostname }}:{{ matrix_appservice_irc_database_port }}/{{ matrix_appservice_irc_database_name }}?sslmode=disable' + +# This is what actually goes into `database.connectionString` for the bridge. +matrix_appservice_irc_database_connectionString: "{{ + { + 'nedb': 'nedb:///data', + 'postgres': matrix_appservice_irc_database_connection_string, + }[matrix_appservice_irc_database_engine] +}}" matrix_appservice_irc_ircService_servers: [] diff --git a/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml b/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml new file mode 100644 index 00000000..bc6525ec --- /dev/null +++ b/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml @@ -0,0 +1,53 @@ +- name: Fail if Postgres not enabled + fail: + msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot migrate." + when: "not matrix_postgres_enabled|bool" + +# Defaults + +- name: Set postgres_start_wait_time, if not provided + set_fact: + postgres_start_wait_time: 15 + when: "postgres_start_wait_time|default('') == ''" + +# Actual import work + +- name: Ensure matrix-postgres is started + service: + name: matrix-postgres + state: started + daemon_reload: yes + register: matrix_postgres_service_start_result + +- name: Wait a bit, so that Postgres can start + wait_for: + timeout: "{{ postgres_start_wait_time }}" + delegate_to: 127.0.0.1 + become: false + when: "matrix_postgres_service_start_result.changed|bool" + +- name: Ensure matrix-appservice-irc is stopped + service: + name: matrix-appservice-irc + state: stopped + +- name: Import appservice-irc NeDB database into Postgres + command: + cmd: >- + {{ matrix_host_command_docker }} run + --rm + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --network={{ matrix_docker_network }} + --mount type=bind,src={{ matrix_appservice_irc_data_path }},dst=/data + --entrypoint=/bin/sh + {{ matrix_appservice_irc_docker_image }} + -c + '/usr/local/bin/node /app/lib/scripts/migrate-db-to-pgres.js --dbdir /data --privateKey /data/passkey.pem --connectionString {{ matrix_appservice_irc_database_connection_string }}' + +- name: Archive NeDB database files + command: + cmd: "mv {{ matrix_appservice_irc_data_path }}/{{ item }} {{ matrix_appservice_irc_data_path }}/{{ item }}.backup" + with_items: + - rooms.db + - users.db diff --git a/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml b/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml index 5e313347..a748df96 100644 --- a/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml @@ -1,12 +1,5 @@ --- -- name: Ensure Appservice IRC image is pulled - docker_image: - name: "{{ matrix_appservice_irc_docker_image }}" - source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" - force_source: "{{ matrix_appservice_irc_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_irc_docker_image_force_pull }}" - - name: Ensure Appservice IRC paths exist file: path: "{{ item }}" @@ -24,25 +17,49 @@ path: "{{ matrix_appservice_irc_base_path }}/passkey.pem" register: matrix_appservice_irc_stat_passkey -- name: (Data relocation) Ensure matrix-appservice-irc.service is stopped - service: - name: matrix-appservice-irc - state: stopped - daemon_reload: yes - failed_when: false +- block: + - name: (Data relocation) Ensure matrix-appservice-irc.service is stopped + service: + name: matrix-appservice-irc + state: stopped + daemon_reload: yes + failed_when: false + + - name: (Data relocation) Move AppService IRC passkey.pem file to ./data directory + command: "mv {{ matrix_appservice_irc_base_path }}/passkey.pem {{ matrix_appservice_irc_data_path }}/passkey.pem" + + - name: (Data relocation) Move AppService IRC database files to ./data directory + command: "mv {{ matrix_appservice_irc_base_path }}/{{ item }} {{ matrix_appservice_irc_data_path }}/{{ item }}" + with_items: + - rooms.db + - users.db + failed_when: false when: "matrix_appservice_irc_stat_passkey.stat.exists" -- name: (Data relocation) Move AppService IRC passkey.pem file to ./data directory - command: "mv {{ matrix_appservice_irc_base_path }}/passkey.pem {{ matrix_appservice_irc_data_path }}/passkey.pem" - when: "matrix_appservice_irc_stat_passkey.stat.exists" -- name: (Data relocation) Move AppService IRC database files to ./data directory - command: "mv {{ matrix_appservice_irc_base_path }}/{{ item }} {{ matrix_appservice_irc_data_path }}/{{ item }}" - with_items: - - rooms.db - - users.db - failed_when: false - when: "matrix_appservice_irc_stat_passkey.stat.exists" +- set_fact: + matrix_appservice_irc_requires_restart: false + +- block: + - name: Check if a nedb database already exists + stat: + path: "{{ matrix_appservice_irc_data_path }}/users.db" + register: matrix_appservice_irc_nedb_database_path_local_stat_result + + - block: + - import_tasks: "{{ role_path }}/tasks/migrate_nedb_to_postgres.yml" + + - set_fact: + matrix_appservice_irc_requires_restart: true + when: "matrix_appservice_irc_nedb_database_path_local_stat_result.stat.exists|bool" + when: "matrix_appservice_irc_database_engine == 'postgres'" + +- name: Ensure Appservice IRC image is pulled + docker_image: + name: "{{ matrix_appservice_irc_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_appservice_irc_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_irc_docker_image_force_pull }}" - name: Ensure Matrix Appservice IRC config installed copy: diff --git a/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml b/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml index 9b89a340..bd08427c 100644 --- a/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml +++ b/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml @@ -33,23 +33,3 @@ when: "item.old in vars" with_items: - {'old': 'matrix_appservice_irc_container_expose_client_server_api_port', 'new': ''} - -- block: - - name: Check if a neDB database already exists - stat: - path: "{{ matrix_appservice_irc_data_path }}/" - register: matrix_appservice_irc_nedb_stat_result - - - name: Fail if an neDB database already exists when using Postgres - fail: - msg: >-2 - matrix_appservice_irc_database_engine has been set to `postgres` (which is our new default now). - However, we've discovered an existing neDB database in {{ matrix_appservice_irc_data_path }}/. - It appears that you've been using this bridge with the neDB engine until now. - To continue using neDB, opt into it explicitly: add `matrix_appservice_irc_database_engine: nedb` to your vars.yml file and re-run this same command. - Alternatively, to migrate your existing neDB database to Postgres: - 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-irc-nedb --extra-vars='nedb_database_path={{ matrix_appservice_irc_data_path }} postgres_connection_string_variable_name=matrix_appservice_irc_database_connString'`) - 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) - when: "matrix_appservice_irc_nedb_stat_result.stat.exists" - when: "matrix_appservice_irc_database_engine == 'postgres'" diff --git a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 index 0da28403..94bbda7b 100644 --- a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 +++ b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 @@ -127,8 +127,8 @@ advanced: # Use an external database to store bridge state. database: # database engine (must be 'postgres' or 'nedb'). Default: nedb - engine: {{ matrix_appservice_irc_database_engine }} + engine: {{ matrix_appservice_irc_database_engine|to_json }} # Either a PostgreSQL connection string, or a path to the NeDB storage directory. # For postgres, it must start with postgres:// # For NeDB, it must start with nedb://. The path is relative to the project directory. - connectionString: {{ matrix_appservice_irc_database_connString | to_json }} + connectionString: {{ matrix_appservice_irc_database_connectionString|to_json }} diff --git a/roles/matrix-postgres/tasks/import_nedb.yml b/roles/matrix-postgres/tasks/import_nedb.yml index cc1f9d78..2a3dd587 100644 --- a/roles/matrix-postgres/tasks/import_nedb.yml +++ b/roles/matrix-postgres/tasks/import_nedb.yml @@ -67,22 +67,6 @@ become: false when: "matrix_postgres_service_start_result.changed|bool" -# See https://github.com/matrix-org/matrix-appservice-irc/wiki/Migrating-from-NEdB-to-PostgreSQL -- name: Import appservice_irc NeDB database from {{ sqlite_database_path }} into Postgres - when: database == 'appservice_irc' - command: - cmd: >- - {{ matrix_host_command_docker }} run - --rm - --user={{ matrix_user_uid }}:{{ matrix_user_gid }} - --cap-drop=ALL - --network={{ matrix_docker_network }} - --mount type=bind,src={{ matrix_appservice_irc_data_path }}:/data:ro - --entrypoint=/bin/sh - {{ matrix_appservice_irc_docker_image }} - -c - './scripts/migrate-db-to-pgres.sh -d /data -p passkey.pem -c {{ postgres_db_connection_string }}' - # No migration.sh available, but found this: # https://github.com/matrix-org/matrix-appservice-slack/blob/develop/src/scripts/migrateToPostgres.ts # Usage should be similar to appservice_irc @@ -95,15 +79,12 @@ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} --cap-drop=ALL --network={{ matrix_docker_network }} - --mount type=bind,src={{ matrix_appservice_irc_data_path }}:/data:ro + --mount type=bind,src={{ matrix_appservice_irc_data_path }},dst=/data,ro --entrypoint=/bin/sh {{ matrix_appservice_slack_docker_image }} -c 'node /lib/scripts/migrate-db-to-pgres.js -d /data -p passkey.pem -c {{ postgres_db_connection_string }}' -- name: Archive NeDB database ({{ sqlite_database_path }} -> {{ sqlite_database_path }}.backup) - command: - cmd: "mv {{ sqlite_database_path }} {{ sqlite_database_path }}.backup" - name: Inject result set_fact: diff --git a/roles/matrix-postgres/tasks/main.yml b/roles/matrix-postgres/tasks/main.yml index 51801314..86b8f872 100644 --- a/roles/matrix-postgres/tasks/main.yml +++ b/roles/matrix-postgres/tasks/main.yml @@ -32,14 +32,6 @@ tags: - import-generic-sqlite-db -# Imports appservice-irc NeDB into postgres -- import_tasks: "{{ role_path }}/tasks/import_nedb.yml" - vars: - database: appservice_irc - when: run_postgres_import_nedb|bool - tags: - - import-irc-nedb - # Imports slacks neDB to postgres. - import_tasks: "{{ role_path }}/tasks/import_nedb.yml" vars: