From 384da4f34f5257b761013878c1836f383358f170 Mon Sep 17 00:00:00 2001 From: Cody Wyatt Neiman Date: Sun, 30 Oct 2022 19:01:49 -0400 Subject: [PATCH 01/12] Add S3 SSE-C support to synapse-s3-storage-provider --- roles/custom/matrix-synapse/defaults/main.yml | 3 +++ .../templates/synapse/ext/s3-storage-provider/env.j2 | 6 ++++++ .../ext/s3-storage-provider/media_storage_provider.yaml.j2 | 6 ++++++ .../matrix-synapse-s3-storage-provider-migrate.j2 | 6 +++++- 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/roles/custom/matrix-synapse/defaults/main.yml b/roles/custom/matrix-synapse/defaults/main.yml index 54351256..372ed1cf 100644 --- a/roles/custom/matrix-synapse/defaults/main.yml +++ b/roles/custom/matrix-synapse/defaults/main.yml @@ -810,6 +810,9 @@ matrix_synapse_ext_synapse_s3_storage_provider_config_region_name: '' matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url: '' matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id: '' matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key: '' +matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled: false +matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key: '' +matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_algo: 'AES256' matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class: STANDARD matrix_synapse_ext_synapse_s3_storage_provider_config_threadpool_size: 40 # matrix_synapse_ext_synapse_s3_storage_provider_update_db_day_count is a day value (number) for the `s3_media_upload update-db` command. diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 index 6dfcbe41..58d26255 100644 --- a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 @@ -4,6 +4,12 @@ AWS_DEFAULT_REGION={{ matrix_synapse_ext_synapse_s3_storage_provider_config_regi ENDPOINT={{ matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url }} BUCKET={{ matrix_synapse_ext_synapse_s3_storage_provider_config_bucket }} + +{% if matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled %} +SSE_CUSTOMER_KEY={{ matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key }} +SSE_CUSTOMER_ALGO={{ matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_algo }} +{% endif %} + STORAGE_CLASS={{ matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class }} MEDIA_PATH=/matrix-media-store-parent/{{ matrix_synapse_media_store_directory_name }} diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 index 97b0f5f2..a602b6f9 100644 --- a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 @@ -9,6 +9,12 @@ config: access_key_id: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id | to_json }} secret_access_key: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key | to_json }} + + {% if matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled %} + sse_customer_key: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key | to_json }} + sse_customer_algo: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_algo | to_json }} + {% endif %} + storage_class: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class | to_json }} threadpool_size: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_threadpool_size | to_json }} diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 index d48ae122..031c0ea0 100644 --- a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 @@ -10,4 +10,8 @@ --network={{ matrix_docker_network }} \ --entrypoint=/bin/bash \ {{ matrix_synapse_docker_image_final }} \ - -c 's3_media_upload update-db $UPDATE_DB_DURATION && s3_media_upload --no-progress check-deleted $MEDIA_PATH && s3_media_upload --no-progress upload $MEDIA_PATH $BUCKET --delete --storage-class $STORAGE_CLASS --endpoint-url $ENDPOINT' + {% if matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled %} + -c 's3_media_upload update-db $UPDATE_DB_DURATION && s3_media_upload --no-progress check-deleted $MEDIA_PATH && s3_media_upload --no-progress upload $MEDIA_PATH $BUCKET --delete --storage-class $STORAGE_CLASS --endpoint-url $ENDPOINT --sse-customer-algo $SSE_CUSTOMER_ALGO --sse-customer-key $SSE_CUSTOMER_KEY' + {% else %} + -c 's3_media_upload update-db $UPDATE_DB_DURATION && s3_media_upload --no-progress check-deleted $MEDIA_PATH && s3_media_upload --no-progress upload $MEDIA_PATH $BUCKET --delete --storage-class $STORAGE_CLASS --endpoint-url $ENDPOINT + {% endif %} From a7320e02ff0e208d9e31f11a2dedfae619abcf13 Mon Sep 17 00:00:00 2001 From: Cody Wyatt Neiman Date: Sun, 13 Nov 2022 03:18:53 -0500 Subject: [PATCH 02/12] Adjust sse-c template formatting --- .../ext/s3-storage-provider/media_storage_provider.yaml.j2 | 5 ++--- .../matrix-synapse-s3-storage-provider-migrate.j2 | 5 +---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 index a602b6f9..e888e3c5 100644 --- a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 @@ -9,12 +9,11 @@ config: access_key_id: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id | to_json }} secret_access_key: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key | to_json }} - - {% if matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled %} +{% if matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled %} sse_customer_key: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key | to_json }} sse_customer_algo: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_algo | to_json }} - {% endif %} +{% endif %} storage_class: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class | to_json }} threadpool_size: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_threadpool_size | to_json }} diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 index 031c0ea0..4b2386b1 100644 --- a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 @@ -11,7 +11,4 @@ --entrypoint=/bin/bash \ {{ matrix_synapse_docker_image_final }} \ {% if matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled %} - -c 's3_media_upload update-db $UPDATE_DB_DURATION && s3_media_upload --no-progress check-deleted $MEDIA_PATH && s3_media_upload --no-progress upload $MEDIA_PATH $BUCKET --delete --storage-class $STORAGE_CLASS --endpoint-url $ENDPOINT --sse-customer-algo $SSE_CUSTOMER_ALGO --sse-customer-key $SSE_CUSTOMER_KEY' - {% else %} - -c 's3_media_upload update-db $UPDATE_DB_DURATION && s3_media_upload --no-progress check-deleted $MEDIA_PATH && s3_media_upload --no-progress upload $MEDIA_PATH $BUCKET --delete --storage-class $STORAGE_CLASS --endpoint-url $ENDPOINT - {% endif %} + -c 's3_media_upload update-db $UPDATE_DB_DURATION && s3_media_upload --no-progress check-deleted $MEDIA_PATH && s3_media_upload --no-progress upload $MEDIA_PATH $BUCKET --delete --storage-class $STORAGE_CLASS --endpoint-url $ENDPOINT {% if matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled %}--sse-customer-algo $SSE_CUSTOMER_ALGO --sse-customer-key $SSE_CUSTOMER_KEY{% endif %}' From b6bb5731cd2f2da466ea066ca0d6101d8f23d119 Mon Sep 17 00:00:00 2001 From: Cody Wyatt Neiman Date: Sun, 13 Nov 2022 03:20:30 -0500 Subject: [PATCH 03/12] Remove leftover sse-c enabled cmd check --- .../usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 index 4b2386b1..2f0cd0e5 100644 --- a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 @@ -10,5 +10,4 @@ --network={{ matrix_docker_network }} \ --entrypoint=/bin/bash \ {{ matrix_synapse_docker_image_final }} \ - {% if matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled %} -c 's3_media_upload update-db $UPDATE_DB_DURATION && s3_media_upload --no-progress check-deleted $MEDIA_PATH && s3_media_upload --no-progress upload $MEDIA_PATH $BUCKET --delete --storage-class $STORAGE_CLASS --endpoint-url $ENDPOINT {% if matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled %}--sse-customer-algo $SSE_CUSTOMER_ALGO --sse-customer-key $SSE_CUSTOMER_KEY{% endif %}' From 533e47e9b98f94fc9dc9a0ca3166fe28cc924520 Mon Sep 17 00:00:00 2001 From: Cody Wyatt Neiman Date: Tue, 3 Jan 2023 15:16:55 -0500 Subject: [PATCH 04/12] Add documentation on using s3 sse-c --- docs/configuring-playbook-synapse-s3-storage-provider.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/configuring-playbook-synapse-s3-storage-provider.md b/docs/configuring-playbook-synapse-s3-storage-provider.md index 6022eaad..f029b3f3 100644 --- a/docs/configuring-playbook-synapse-s3-storage-provider.md +++ b/docs/configuring-playbook-synapse-s3-storage-provider.md @@ -39,6 +39,13 @@ matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id: access-key- matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key: secret-key-goes-here matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class: STANDARD # or STANDARD_IA, etc. +# S3 Server Side Encryption with a Customer provided key (SSE-C) can also be configured as follows +# This is not recommended unless you understand what you are doing, and may make restoring from backups additionally challenging +# You can read more about SSE-C here: https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html +matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled: true +matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key: ssec-key-goes-here # Generate with: cat /dev/urandom | head -c 32 | base64 - +matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_algo: AES256 + # For additional advanced settings, take a look at `roles/custom/matrix-synapse/defaults/main.yml` ``` From 4d44f7b49ed750d97e413f2b4b156048e8608bbf Mon Sep 17 00:00:00 2001 From: Cody Wyatt Neiman Date: Tue, 3 Jan 2023 15:18:09 -0500 Subject: [PATCH 05/12] Use base64 encoded string for sse-c key --- .../templates/synapse/ext/s3-storage-provider/env.j2 | 2 +- .../ext/s3-storage-provider/media_storage_provider.yaml.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 index 58d26255..6cc7753f 100644 --- a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 @@ -6,7 +6,7 @@ ENDPOINT={{ matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url } BUCKET={{ matrix_synapse_ext_synapse_s3_storage_provider_config_bucket }} {% if matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled %} -SSE_CUSTOMER_KEY={{ matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key }} +SSE_CUSTOMER_KEY={{ matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key | b64decode }} SSE_CUSTOMER_ALGO={{ matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_algo }} {% endif %} diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 index e888e3c5..988ad002 100644 --- a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 @@ -10,7 +10,7 @@ config: secret_access_key: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key | to_json }} {% if matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled %} - sse_customer_key: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key | to_json }} + sse_customer_key: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key | b64decode | to_json }} sse_customer_algo: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_algo | to_json }} {% endif %} From 6d96bcee1da1a306aaeab101026a7a08702f417a Mon Sep 17 00:00:00 2001 From: Cody Wyatt Neiman Date: Tue, 3 Jan 2023 15:18:49 -0500 Subject: [PATCH 06/12] Allow 'git' as a version for s3 storage provider --- .../templates/synapse/customizations/Dockerfile.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/custom/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 b/roles/custom/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 index 3919e955..b77dbc23 100644 --- a/roles/custom/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 @@ -1,7 +1,11 @@ FROM {{ matrix_synapse_docker_image }} {% if matrix_synapse_container_image_customizations_s3_storage_provider_installation_enabled %} + {% if matrix_synapse_ext_synapse_s3_storage_provider_version == 'git' %} +RUN apt-get update -yq && apt-get install -yq git && pip install git+https://github.com/matrix-org/synapse-s3-storage-provider.git + {% else %} RUN pip install synapse-s3-storage-provider=={{ matrix_synapse_ext_synapse_s3_storage_provider_version }} + {% endif %} {% endif %} {{ matrix_synapse_container_image_customizations_dockerfile_body_custom }} From b9604da9d9b861b5d156581ce46b16b7a757e119 Mon Sep 17 00:00:00 2001 From: Cody Wyatt Neiman Date: Tue, 3 Jan 2023 15:26:06 -0500 Subject: [PATCH 07/12] Add note on using synapse s3 provider git version for sse-c support --- docs/configuring-playbook-synapse-s3-storage-provider.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuring-playbook-synapse-s3-storage-provider.md b/docs/configuring-playbook-synapse-s3-storage-provider.md index f029b3f3..a4ee7063 100644 --- a/docs/configuring-playbook-synapse-s3-storage-provider.md +++ b/docs/configuring-playbook-synapse-s3-storage-provider.md @@ -45,6 +45,8 @@ matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class: STANDARD # matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled: true matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key: ssec-key-goes-here # Generate with: cat /dev/urandom | head -c 32 | base64 - matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_algo: AES256 +# Using the git version is also required until > v1.1.2 is released +matrix_synapse_ext_synapse_s3_storage_provider_version: git # For additional advanced settings, take a look at `roles/custom/matrix-synapse/defaults/main.yml` ``` From f5390562ed5a9df7cf254e783d41fe260c643931 Mon Sep 17 00:00:00 2001 From: Cody Wyatt Neiman Date: Tue, 3 Jan 2023 16:13:09 -0500 Subject: [PATCH 08/12] Fix synapse s3 storage provider container indentation --- .../templates/synapse/customizations/Dockerfile.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/custom/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 b/roles/custom/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 index b77dbc23..65375aee 100644 --- a/roles/custom/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 @@ -1,11 +1,11 @@ FROM {{ matrix_synapse_docker_image }} {% if matrix_synapse_container_image_customizations_s3_storage_provider_installation_enabled %} - {% if matrix_synapse_ext_synapse_s3_storage_provider_version == 'git' %} +{% if matrix_synapse_ext_synapse_s3_storage_provider_version == 'git' %} RUN apt-get update -yq && apt-get install -yq git && pip install git+https://github.com/matrix-org/synapse-s3-storage-provider.git - {% else %} +{% else %} RUN pip install synapse-s3-storage-provider=={{ matrix_synapse_ext_synapse_s3_storage_provider_version }} - {% endif %} +{% endif %} {% endif %} {{ matrix_synapse_container_image_customizations_dockerfile_body_custom }} From 7e5e1712f5e15f59d3598367f4128ed5d6135382 Mon Sep 17 00:00:00 2001 From: Cody Wyatt Neiman Date: Tue, 3 Jan 2023 17:53:33 -0500 Subject: [PATCH 09/12] Encode s3 sse-c key for utf-8 --- docs/configuring-playbook-synapse-s3-storage-provider.md | 2 +- .../templates/synapse/ext/s3-storage-provider/env.j2 | 2 +- .../ext/s3-storage-provider/media_storage_provider.yaml.j2 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/configuring-playbook-synapse-s3-storage-provider.md b/docs/configuring-playbook-synapse-s3-storage-provider.md index a4ee7063..25d9a54a 100644 --- a/docs/configuring-playbook-synapse-s3-storage-provider.md +++ b/docs/configuring-playbook-synapse-s3-storage-provider.md @@ -43,7 +43,7 @@ matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class: STANDARD # # This is not recommended unless you understand what you are doing, and may make restoring from backups additionally challenging # You can read more about SSE-C here: https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled: true -matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key: ssec-key-goes-here # Generate with: cat /dev/urandom | head -c 32 | base64 - +matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key: ssec-key-goes-here # Generate with: cat /dev/urandom | base64 | head -c 32 matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_algo: AES256 # Using the git version is also required until > v1.1.2 is released matrix_synapse_ext_synapse_s3_storage_provider_version: git diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 index 6cc7753f..58d26255 100644 --- a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 @@ -6,7 +6,7 @@ ENDPOINT={{ matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url } BUCKET={{ matrix_synapse_ext_synapse_s3_storage_provider_config_bucket }} {% if matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled %} -SSE_CUSTOMER_KEY={{ matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key | b64decode }} +SSE_CUSTOMER_KEY={{ matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key }} SSE_CUSTOMER_ALGO={{ matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_algo }} {% endif %} diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 index 988ad002..e888e3c5 100644 --- a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 @@ -10,7 +10,7 @@ config: secret_access_key: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key | to_json }} {% if matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled %} - sse_customer_key: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key | b64decode | to_json }} + sse_customer_key: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key | to_json }} sse_customer_algo: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_algo | to_json }} {% endif %} From 3a28b7e332e06188b9533d1521888d2f7e0af513 Mon Sep 17 00:00:00 2001 From: Cody Wyatt Neiman Date: Tue, 3 Jan 2023 18:01:15 -0500 Subject: [PATCH 10/12] Specify version requirement for sse-c in s3 storage provider --- docs/configuring-playbook-synapse-s3-storage-provider.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuring-playbook-synapse-s3-storage-provider.md b/docs/configuring-playbook-synapse-s3-storage-provider.md index 25d9a54a..4b077027 100644 --- a/docs/configuring-playbook-synapse-s3-storage-provider.md +++ b/docs/configuring-playbook-synapse-s3-storage-provider.md @@ -45,7 +45,7 @@ matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class: STANDARD # matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled: true matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key: ssec-key-goes-here # Generate with: cat /dev/urandom | base64 | head -c 32 matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_algo: AES256 -# Using the git version is also required until > v1.1.2 is released +# Using the git version is also required for SSE-C until > v1.1.2 is released matrix_synapse_ext_synapse_s3_storage_provider_version: git # For additional advanced settings, take a look at `roles/custom/matrix-synapse/defaults/main.yml` From fc9eaa6ec5ebb87d00d6224a686c0dcf25266dda Mon Sep 17 00:00:00 2001 From: Cody Wyatt Neiman Date: Mon, 9 Jan 2023 15:52:38 -0500 Subject: [PATCH 11/12] Remove git version for s3 storage provider --- docs/configuring-playbook-synapse-s3-storage-provider.md | 2 -- roles/custom/matrix-synapse/defaults/main.yml | 2 +- .../templates/synapse/customizations/Dockerfile.j2 | 4 ---- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/docs/configuring-playbook-synapse-s3-storage-provider.md b/docs/configuring-playbook-synapse-s3-storage-provider.md index 4b077027..4f6314e4 100644 --- a/docs/configuring-playbook-synapse-s3-storage-provider.md +++ b/docs/configuring-playbook-synapse-s3-storage-provider.md @@ -45,8 +45,6 @@ matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class: STANDARD # matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled: true matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key: ssec-key-goes-here # Generate with: cat /dev/urandom | base64 | head -c 32 matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_algo: AES256 -# Using the git version is also required for SSE-C until > v1.1.2 is released -matrix_synapse_ext_synapse_s3_storage_provider_version: git # For additional advanced settings, take a look at `roles/custom/matrix-synapse/defaults/main.yml` ``` diff --git a/roles/custom/matrix-synapse/defaults/main.yml b/roles/custom/matrix-synapse/defaults/main.yml index 4da590a5..51927919 100644 --- a/roles/custom/matrix-synapse/defaults/main.yml +++ b/roles/custom/matrix-synapse/defaults/main.yml @@ -791,7 +791,7 @@ matrix_synapse_ext_encryption_config_yaml: | # Installing it requires building a customized Docker image for Synapse (see `matrix_synapse_container_image_customizations_enabled`). # Enabling this will enable customizations and inject the appropriate Dockerfile clauses for installing synapse-s3-storage-provider. matrix_synapse_ext_synapse_s3_storage_provider_enabled: false -matrix_synapse_ext_synapse_s3_storage_provider_version: 1.1.2 +matrix_synapse_ext_synapse_s3_storage_provider_version: 1.2.0 # Controls whether media from this (local) server is stored in s3-storage-provider matrix_synapse_ext_synapse_s3_storage_provider_store_local: true # Controls whether media from remote servers is stored in s3-storage-provider diff --git a/roles/custom/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 b/roles/custom/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 index 65375aee..3919e955 100644 --- a/roles/custom/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 @@ -1,11 +1,7 @@ FROM {{ matrix_synapse_docker_image }} {% if matrix_synapse_container_image_customizations_s3_storage_provider_installation_enabled %} -{% if matrix_synapse_ext_synapse_s3_storage_provider_version == 'git' %} -RUN apt-get update -yq && apt-get install -yq git && pip install git+https://github.com/matrix-org/synapse-s3-storage-provider.git -{% else %} RUN pip install synapse-s3-storage-provider=={{ matrix_synapse_ext_synapse_s3_storage_provider_version }} {% endif %} -{% endif %} {{ matrix_synapse_container_image_customizations_dockerfile_body_custom }} From f0d1e23c9d3891524e89bee72a8202c500661850 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 10 Jan 2023 09:22:55 +0200 Subject: [PATCH 12/12] Move around whitelines --- .../ext/s3-storage-provider/media_storage_provider.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 index e888e3c5..ac2b58db 100644 --- a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 @@ -12,8 +12,8 @@ config: {% if matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled %} sse_customer_key: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key | to_json }} sse_customer_algo: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_algo | to_json }} - {% endif %} + storage_class: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class | to_json }} threadpool_size: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_threadpool_size | to_json }}