Add pgloader self-building support (for ARM)

group_vars/matrix_servers

@@ -995,6 +995,8 @@ matrix_postgres_connection_username: "synapse"
 matrix_postgres_connection_password: "synapse-password"
 matrix_postgres_db_name: "homeserver"
 
+matrix_postgres_pgloader_container_image_self_build: "{{ matrix_architecture != 'amd64' }}"
+
 matrix_postgres_additional_databases: |
   {{
     ([{

roles/matrix-postgres/defaults/main.yml

@@ -65,4 +65,15 @@ matrix_postgres_import_databases_ignore_regex: "^CREATE DATABASE ({{ matrix_post
 # For most (subsequent) runs, Postgres would already be running, so no waiting will be happening at all.
 matrix_postgres_additional_databases_postgres_start_wait_timeout_seconds: 15
 
-matrix_postgres_pgloader_docker_image: "docker.io/illagrenan/pgloader:3.6.2"
+
+matrix_postgres_pgloader_container_image_self_build: false
+matrix_postgres_pgloader_container_image_self_build_repo: "https://github.com/illagrenan/pgloader-docker.git"
+matrix_postgres_pgloader_container_image_self_build_repo_branch: "v{{ matrix_postgres_pgloader_docker_image_tag }}"
+matrix_postgres_pgloader_container_image_self_build_src_path: "{{ matrix_postgres_base_path }}/pgloader-container-src"
+
+# We use illagrenan/pgloader, instead of the more official dimitri/pgloader image,
+# because the official one only provides a `latest` tag.
+matrix_postgres_pgloader_docker_image: "{{ matrix_postgres_pgloader_docker_image_name_prefix }}illagrenan/pgloader:{{ matrix_postgres_pgloader_docker_image_tag }}"
+matrix_postgres_pgloader_docker_image_name_prefix: "{{ 'localhost/' if matrix_postgres_pgloader_container_image_self_build else 'docker.io/' }}"
+matrix_postgres_pgloader_docker_image_tag: "3.6.2"
+matrix_postgres_pgloader_docker_image_force_pull: "{{ matrix_postgres_pgloader_docker_image.endswith(':latest') }}"

roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml

@@ -31,17 +31,50 @@
     msg: "File cannot be found on the server at {{ matrix_postgres_db_migration_request.src }}"
   when: "not matrix_postgres_db_migration_request_src_stat_result.stat.exists"
 
-- name: Fail if we cannot migrate on the current architecture ({{ matrix_architecture }})
+- block:
-  fail:
+    - name: Ensure pgloader repository is present on self-build
-    msg: >-
+      git:
-      {{ matrix_postgres_db_migration_request.engine_variable_name }} (part of {{ matrix_postgres_db_migration_request.caller }}) has been set to `postgres` (which is our new default now).
+        repo: "{{ matrix_postgres_pgloader_container_image_self_build_repo }}"
-      However, we've discovered an existing file-based database ({{ matrix_postgres_db_migration_request.engine_old }}) in {{ matrix_postgres_db_migration_request.src }}.
+        dest: "{{ matrix_postgres_pgloader_container_image_self_build_src_path }}"
-      It appears that you've been using this bridge with a file-based database engine until now.
+        version: "{{ matrix_postgres_pgloader_container_image_self_build_repo_branch }}"
-      To continue using {{ matrix_postgres_db_migration_request.engine_old }}, opt into it explicitly: add `{{ matrix_postgres_db_migration_request.engine_variable_name }}: {{ matrix_postgres_db_migration_request.engine_old }}` to your vars.yml file and re-run this same command.
+        force: "yes"
-      We'd normally auto-migrate you to Postgres, but we can't do it on the {{ matrix_architecture }} architecture. Our pgloader container image only supports amd64 (for now).
+      register: matrix_postgres_pgloader_git_pull_results
-      Learn more here: https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/740
+
-  when: "matrix_architecture != 'amd64'"
+    # If `stable` is used, we hit an error when processing /opt/src/pgloader/build/quicklisp/dists/quicklisp/software/uax-15-20201220-git/data/CompositionExclusions.txt:
-
+    # > the octet sequence #(194) cannot be decoded
+    #
+    # The issue is described here and is not getting fixed for months: https://github.com/dimitri/pgloader/pull/1179
+    #
+    # Although we're not using the dimitri/pgloader image, the one we're using suffers from the same problem.
+    - name: Switch pgloader base image from Debian stable (likely 10.x/Buster) to Bullseye
+      lineinfile:
+        path: "{{ matrix_postgres_pgloader_container_image_self_build_src_path }}/Dockerfile"
+        regexp: "{{ item.match }}"
+        line: "{{ item.replace }}"
+      with_items:
+        - match: '^FROM debian:stable-slim as builder$'
+          replace: 'FROM debian:bullseye-slim as builder'
+        - match: '^FROM debian:stable-slim$'
+          replace: 'FROM debian:bullseye-slim'
+
+    - name: Ensure pgloader Docker image is built
+      docker_image:
+        name: "{{ matrix_postgres_pgloader_docker_image }}"
+        source: build
+        force_source: "{{ matrix_postgres_pgloader_git_pull_results.changed }}"
+        build:
+          dockerfile: Dockerfile
+          path: "{{ matrix_postgres_pgloader_container_image_self_build_src_path }}"
+          pull: yes
+  when: "matrix_postgres_pgloader_container_image_self_build|bool"
+
+- name: Ensure pgloader Docker image is pulled
+  docker_image:
+    name: "{{ matrix_postgres_pgloader_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_postgres_pgloader_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_postgres_pgloader_docker_image_force_pull }}"
+  when: "not matrix_postgres_pgloader_container_image_self_build"
 
 # Defaults