From ad1425eee4f8d9eb1f92f5f0acecef8da3447280 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 23 Dec 2020 09:08:54 +0200 Subject: [PATCH] Add pgloader self-building support (for ARM) --- group_vars/matrix_servers | 2 + roles/matrix-postgres/defaults/main.yml | 13 ++++- .../tasks/util/migrate_db_to_postgres.yml | 55 +++++++++++++++---- 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 4ddadb43..c8af3cf2 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -995,6 +995,8 @@ matrix_postgres_connection_username: "synapse" matrix_postgres_connection_password: "synapse-password" matrix_postgres_db_name: "homeserver" +matrix_postgres_pgloader_container_image_self_build: "{{ matrix_architecture != 'amd64' }}" + matrix_postgres_additional_databases: | {{ ([{ diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index 0c516281..8f1d0d78 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -65,4 +65,15 @@ matrix_postgres_import_databases_ignore_regex: "^CREATE DATABASE ({{ matrix_post # For most (subsequent) runs, Postgres would already be running, so no waiting will be happening at all. matrix_postgres_additional_databases_postgres_start_wait_timeout_seconds: 15 -matrix_postgres_pgloader_docker_image: "docker.io/illagrenan/pgloader:3.6.2" + +matrix_postgres_pgloader_container_image_self_build: false +matrix_postgres_pgloader_container_image_self_build_repo: "https://github.com/illagrenan/pgloader-docker.git" +matrix_postgres_pgloader_container_image_self_build_repo_branch: "v{{ matrix_postgres_pgloader_docker_image_tag }}" +matrix_postgres_pgloader_container_image_self_build_src_path: "{{ matrix_postgres_base_path }}/pgloader-container-src" + +# We use illagrenan/pgloader, instead of the more official dimitri/pgloader image, +# because the official one only provides a `latest` tag. +matrix_postgres_pgloader_docker_image: "{{ matrix_postgres_pgloader_docker_image_name_prefix }}illagrenan/pgloader:{{ matrix_postgres_pgloader_docker_image_tag }}" +matrix_postgres_pgloader_docker_image_name_prefix: "{{ 'localhost/' if matrix_postgres_pgloader_container_image_self_build else 'docker.io/' }}" +matrix_postgres_pgloader_docker_image_tag: "3.6.2" +matrix_postgres_pgloader_docker_image_force_pull: "{{ matrix_postgres_pgloader_docker_image.endswith(':latest') }}" diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml index d01611ef..0da48c64 100644 --- a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml +++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml @@ -31,17 +31,50 @@ msg: "File cannot be found on the server at {{ matrix_postgres_db_migration_request.src }}" when: "not matrix_postgres_db_migration_request_src_stat_result.stat.exists" -- name: Fail if we cannot migrate on the current architecture ({{ matrix_architecture }}) - fail: - msg: >- - {{ matrix_postgres_db_migration_request.engine_variable_name }} (part of {{ matrix_postgres_db_migration_request.caller }}) has been set to `postgres` (which is our new default now). - However, we've discovered an existing file-based database ({{ matrix_postgres_db_migration_request.engine_old }}) in {{ matrix_postgres_db_migration_request.src }}. - It appears that you've been using this bridge with a file-based database engine until now. - To continue using {{ matrix_postgres_db_migration_request.engine_old }}, opt into it explicitly: add `{{ matrix_postgres_db_migration_request.engine_variable_name }}: {{ matrix_postgres_db_migration_request.engine_old }}` to your vars.yml file and re-run this same command. - We'd normally auto-migrate you to Postgres, but we can't do it on the {{ matrix_architecture }} architecture. Our pgloader container image only supports amd64 (for now). - Learn more here: https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/740 - when: "matrix_architecture != 'amd64'" - +- block: + - name: Ensure pgloader repository is present on self-build + git: + repo: "{{ matrix_postgres_pgloader_container_image_self_build_repo }}" + dest: "{{ matrix_postgres_pgloader_container_image_self_build_src_path }}" + version: "{{ matrix_postgres_pgloader_container_image_self_build_repo_branch }}" + force: "yes" + register: matrix_postgres_pgloader_git_pull_results + + # If `stable` is used, we hit an error when processing /opt/src/pgloader/build/quicklisp/dists/quicklisp/software/uax-15-20201220-git/data/CompositionExclusions.txt: + # > the octet sequence #(194) cannot be decoded + # + # The issue is described here and is not getting fixed for months: https://github.com/dimitri/pgloader/pull/1179 + # + # Although we're not using the dimitri/pgloader image, the one we're using suffers from the same problem. + - name: Switch pgloader base image from Debian stable (likely 10.x/Buster) to Bullseye + lineinfile: + path: "{{ matrix_postgres_pgloader_container_image_self_build_src_path }}/Dockerfile" + regexp: "{{ item.match }}" + line: "{{ item.replace }}" + with_items: + - match: '^FROM debian:stable-slim as builder$' + replace: 'FROM debian:bullseye-slim as builder' + - match: '^FROM debian:stable-slim$' + replace: 'FROM debian:bullseye-slim' + + - name: Ensure pgloader Docker image is built + docker_image: + name: "{{ matrix_postgres_pgloader_docker_image }}" + source: build + force_source: "{{ matrix_postgres_pgloader_git_pull_results.changed }}" + build: + dockerfile: Dockerfile + path: "{{ matrix_postgres_pgloader_container_image_self_build_src_path }}" + pull: yes + when: "matrix_postgres_pgloader_container_image_self_build|bool" + +- name: Ensure pgloader Docker image is pulled + docker_image: + name: "{{ matrix_postgres_pgloader_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_postgres_pgloader_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_postgres_pgloader_docker_image_force_pull }}" + when: "not matrix_postgres_pgloader_container_image_self_build" # Defaults