From 7c5c3aedc4416209a0a1c361620e1159ce6384b0 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 27 Nov 2022 09:34:46 +0200 Subject: [PATCH] Put s3-storage-provider scripts under /matrix/synapse/ext/s3-storage-provider/bin, not /usr/local/bin --- ...ng-playbook-synapse-s3-storage-provider.md | 8 ++--- roles/custom/matrix-synapse/defaults/main.yml | 1 + .../ext/s3-storage-provider/setup_install.yml | 32 +++++++++---------- .../s3-storage-provider/setup_uninstall.yml | 6 +--- .../matrix-synapse/tasks/setup_install.yml | 3 ++ .../migrate.j2} | 0 .../shell.j2} | 0 ...pse-s3-storage-provider-migrate.service.j2 | 2 +- 8 files changed, 25 insertions(+), 27 deletions(-) rename roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/{usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 => bin/migrate.j2} (100%) rename roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/{usr-local-bin/matrix-synapse-s3-storage-provider-shell.j2 => bin/shell.j2} (100%) diff --git a/docs/configuring-playbook-synapse-s3-storage-provider.md b/docs/configuring-playbook-synapse-s3-storage-provider.md index 2dba264b..a06d658f 100644 --- a/docs/configuring-playbook-synapse-s3-storage-provider.md +++ b/docs/configuring-playbook-synapse-s3-storage-provider.md @@ -62,7 +62,7 @@ Migrating your existing data can happen in multiple ways: Instead of using `s3_media_upload` directly, which is very slow and painful for an initial data migration, we recommend [using another tool in combination with `s3_media_upload`](#using-another-tool-in-combination-with-s3_media_upload). -To copy your existing files, SSH into the server and run `/usr/local/bin/matrix-synapse-s3-storage-provider-shell`. +To copy your existing files, SSH into the server and run `/matrix/synapse/ext/s3-storage-provider/bin/shell`. This launches a Synapse container, which has access to the local media store, Postgres database, S3 store and has some convenient environment variables configured for you to use (`MEDIA_PATH`, `BUCKET`, `ENDPOINT`, `UPDATE_DB_DAYS`, etc). @@ -76,12 +76,12 @@ Then use the following commands (`$` values come from environment variables - th The `s3_media_upload upload` command may take a lot of time to complete. -Instead of running the above commands manually in the shell, you can also run the `/usr/local/bin/matrix-synapse-s3-storage-provider-migrate` script which will run the same commands automatically. We demonstrate how to do it manually, because: +Instead of running the above commands manually in the shell, you can also run the `/matrix/synapse/ext/s3-storage-provider/bin/migrate` script which will run the same commands automatically. We demonstrate how to do it manually, because: - it's what the upstream project demonstrates and it teaches you how to use the `s3_media_upload` tool - allows you to check and verify the output of each command, to catch mistakes - includes progress bars and detailed output for each command -- allows you to easily interrupt slow-running commands, etc. (the `/usr/local/bin/matrix-synapse-s3-storage-provider-migrate` starts a container without interactive TTY support, so `Ctrl+C` may not work and you and require killing via `docker kill ..`) +- allows you to easily interrupt slow-running commands, etc. (the `/matrix/synapse/ext/s3-storage-provider/bin/migrate` starts a container without interactive TTY support, so `Ctrl+C` may not work and you and require killing via `docker kill ..`) ### Using another tool in combination with `s3_media_upload` @@ -119,7 +119,7 @@ As described in [How it works?](#how-it-works) above, when new media is uploaded By default, we periodically ensure that all local files are uploaded to S3 and are then removed from the local filesystem. This is done automatically using: -- the `/usr/local/bin/matrix-synapse-s3-storage-provider-migrate` script +- the `/matrix/synapse/ext/s3-storage-provider/bin/migrate` script - .. invoked via the `matrix-synapse-s3-storage-provider-migrate.service` service - .. triggered by the `matrix-synapse-s3-storage-provider-migrate.timer` timer, every day at 05:00 diff --git a/roles/custom/matrix-synapse/defaults/main.yml b/roles/custom/matrix-synapse/defaults/main.yml index c1f6b99a..1a9b18a7 100644 --- a/roles/custom/matrix-synapse/defaults/main.yml +++ b/roles/custom/matrix-synapse/defaults/main.yml @@ -60,6 +60,7 @@ matrix_synapse_bin_path: "{{ matrix_synapse_base_path }}/bin" matrix_synapse_ext_path: "{{ matrix_synapse_base_path }}/ext" matrix_synapse_ext_s3_storage_provider_base_path: "{{ matrix_synapse_base_path }}/ext/s3-storage-provider" +matrix_synapse_ext_s3_storage_provider_bin_path: "{{ matrix_synapse_ext_s3_storage_provider_base_path }}/bin" matrix_synapse_container_client_api_port: 8008 diff --git a/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/setup_install.yml b/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/setup_install.yml index b64bf6d2..11958f45 100644 --- a/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/setup_install.yml +++ b/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/setup_install.yml @@ -7,19 +7,22 @@ # # Below are additional tasks for setting up various helper scripts, etc. -- name: Ensure s3-storage-provider env file installed - ansible.builtin.template: - src: "{{ role_path }}/templates/synapse/ext/s3-storage-provider/env.j2" - dest: "{{ matrix_synapse_ext_s3_storage_provider_base_path }}/env" - mode: 0640 - -- name: Ensure s3-storage-provider data path exists +- name: Ensure s3-storage-provider paths exists ansible.builtin.file: - path: "{{ matrix_synapse_ext_s3_storage_provider_base_path }}/data" + path: "{{ item }}" state: directory mode: 0750 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" + with_items: + - "{{ matrix_synapse_ext_s3_storage_provider_base_path }}" + - "{{ matrix_synapse_ext_s3_storage_provider_bin_path }}" + +- name: Ensure s3-storage-provider env file installed + ansible.builtin.template: + src: "{{ role_path }}/templates/synapse/ext/s3-storage-provider/env.j2" + dest: "{{ matrix_synapse_ext_s3_storage_provider_base_path }}/env" + mode: 0640 - name: Ensure s3-storage-provider database.yaml file installed ansible.builtin.template: @@ -29,12 +32,12 @@ - name: Ensure s3-storage-provider scripts installed ansible.builtin.template: - src: "{{ role_path }}/templates/synapse/ext/s3-storage-provider/usr-local-bin/{{ item }}.j2" - dest: "{{ matrix_local_bin_path }}/{{ item }}" + src: "{{ role_path }}/templates/synapse/ext/s3-storage-provider/bin/{{ item }}.j2" + dest: "{{ matrix_synapse_ext_s3_storage_provider_bin_path }}/{{ item }}" mode: 0750 with_items: - - matrix-synapse-s3-storage-provider-shell - - matrix-synapse-s3-storage-provider-migrate + - shell + - migrate - name: Ensure matrix-synapse-s3-storage-provider-migrate.service and timer are installed ansible.builtin.template: @@ -45,8 +48,3 @@ - matrix-synapse-s3-storage-provider-migrate.service - matrix-synapse-s3-storage-provider-migrate.timer register: matrix_synapse_s3_storage_provider_systemd_service_result - -- name: Ensure systemd reloaded after matrix-synapse-s3-storage-provider-migrate.service installation - ansible.builtin.service: - daemon_reload: true - when: matrix_synapse_s3_storage_provider_systemd_service_result.changed | bool diff --git a/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/setup_uninstall.yml b/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/setup_uninstall.yml index a112a37c..83e8a0f7 100644 --- a/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/setup_uninstall.yml +++ b/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/setup_uninstall.yml @@ -11,9 +11,5 @@ - name: Ensure s3-storage-provider files don't exist ansible.builtin.file: - path: "{{ item }}" + path: "{{ matrix_synapse_ext_s3_storage_provider_base_path }}" state: absent - with_items: - - "{{ matrix_local_bin_path }}/matrix-synapse-s3-storage-provider-shell" - - "{{ matrix_local_bin_path }}/matrix-synapse-s3-storage-provider-migrate" - - "{{ matrix_synapse_ext_s3_storage_provider_base_path }}" diff --git a/roles/custom/matrix-synapse/tasks/setup_install.yml b/roles/custom/matrix-synapse/tasks/setup_install.yml index 28fbc41d..d1ef7ed4 100644 --- a/roles/custom/matrix-synapse/tasks/setup_install.yml +++ b/roles/custom/matrix-synapse/tasks/setup_install.yml @@ -13,7 +13,10 @@ - {path: "{{ matrix_synapse_bin_path }}", when: true} - {path: "{{ matrix_synapse_docker_src_files_path }}", when: "{{ matrix_synapse_container_image_self_build }}"} - {path: "{{ matrix_synapse_customized_docker_src_files_path }}", when: "{{ matrix_synapse_container_image_customizations_enabled }}"} + - {path: "{{ matrix_synapse_ext_s3_storage_provider_base_path }}", when: "{{ matrix_synapse_ext_synapse_s3_storage_provider_enabled }}"} + - {path: "{{ matrix_synapse_ext_s3_storage_provider_bin_path }}", when: "{{ matrix_synapse_ext_synapse_s3_storage_provider_enabled }}"} + # We handle matrix_synapse_media_store_path elsewhere (in ./synapse/setup_install.yml), # because if it's using Goofys and it's already mounted (from before), # trying to chown/chmod it here will cause trouble. diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/bin/migrate.j2 similarity index 100% rename from roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 rename to roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/bin/migrate.j2 diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-shell.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/bin/shell.j2 similarity index 100% rename from roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-shell.j2 rename to roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/bin/shell.j2 diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.service.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.service.j2 index 159681a0..626c8385 100644 --- a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.service.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.service.j2 @@ -4,4 +4,4 @@ Description=Migrates locally-stored Synapse media store files to S3 [Service] Type=oneshot Environment="HOME={{ devture_systemd_docker_base_systemd_unit_home_path }}" -ExecStart={{ matrix_local_bin_path }}/matrix-synapse-s3-storage-provider-migrate +ExecStart={{ matrix_synapse_ext_s3_storage_provider_bin_path }}/migrate