From 30745db6d5f098fa68ac228b900f6efa82fd5ed1 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 18:29:30 +0300 Subject: [PATCH] Add storage-class support to the s3_media_upload command and mention matrix-synapse-s3-storage-provider-migrate script --- ...configuring-playbook-synapse-s3-storage-provider.md | 10 ++++++++-- .../templates/synapse/ext/s3-storage-provider/env.j2 | 1 + .../matrix-synapse-s3-storage-provider-migrate.j2 | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/configuring-playbook-synapse-s3-storage-provider.md b/docs/configuring-playbook-synapse-s3-storage-provider.md index 6ab2820a..fce5b4ed 100644 --- a/docs/configuring-playbook-synapse-s3-storage-provider.md +++ b/docs/configuring-playbook-synapse-s3-storage-provider.md @@ -70,10 +70,16 @@ Then use the following commands (`$` values come from environment variables - th - `$UPDATE_DB_DURATION` is influenced by the `matrix_synapse_ext_synapse_s3_storage_provider_update_db_day_count` variable (defaults to `0`) - `$UPDATE_DB_DURATION` defaults to `0d` (0 days), which means **include files which haven't been accessed for more than 0 days** (that is, **all files will be included**). - `s3_media_upload check-deleted $MEDIA_PATH` - check whether files in the local cache still exist in the local media repository directory -- `s3_media_upload upload $MEDIA_PATH $BUCKET --delete --endpoint-url $ENDPOINT` - uploads locally-stored files to S3 and deletes them from the local media repository directory +- `s3_media_upload upload $MEDIA_PATH $BUCKET --delete --storage-class $STORAGE_CLASS --endpoint-url $ENDPOINT` - uploads locally-stored files to S3 and deletes them from the local media repository directory -The `upload` command may take a lot of time to complete. +The `s3_media_upload upload` command may take a lot of time to complete. +Instead of running the above commands manually in the shell, you can also run the `/usr/local/bin/matrix-synapse-s3-storage-provider-migrate` script which will run the same commands automatically. We demonstrate how to do it manually, because: + +- it's what the upstream project demonstrates and it teaches you how to use the `s3_media_upload` tool +- allows you to check and verify the output of each command, to catch mistakes +- includes progress bars and detailed output for each command +- allows you to easily interrupt slow-running commands, etc. (the `/usr/local/bin/matrix-synapse-s3-storage-provider-migrate` starts a container without interactive TTY support, so `Ctrl+C` may not work and you and require killing via `docker kill ..`) ### Using another tool in combination with `s3_media_upload` diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 index 3788499b..6dfcbe41 100644 --- a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 @@ -4,6 +4,7 @@ AWS_DEFAULT_REGION={{ matrix_synapse_ext_synapse_s3_storage_provider_config_regi ENDPOINT={{ matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url }} BUCKET={{ matrix_synapse_ext_synapse_s3_storage_provider_config_bucket }} +STORAGE_CLASS={{ matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class }} MEDIA_PATH=/matrix-media-store-parent/{{ matrix_synapse_media_store_directory_name }} diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 index 0893f5d6..e6684e69 100644 --- a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 @@ -10,4 +10,4 @@ --network={{ matrix_docker_network }} \ --entrypoint=/bin/bash \ {{ matrix_synapse_docker_image_final }} \ - -c 's3_media_upload update-db $UPDATE_DB_DURATION && s3_media_upload --no-progress check-deleted $MEDIA_PATH && s3_media_upload --no-progress upload $MEDIA_PATH $BUCKET --delete --endpoint-url $ENDPOINT' + -c 's3_media_upload update-db $UPDATE_DB_DURATION && s3_media_upload --no-progress check-deleted $MEDIA_PATH && s3_media_upload --no-progress upload $MEDIA_PATH $BUCKET --delete --storage-class $STORAGE_CLASS --endpoint-url $ENDPOINT'