From 13ef9e85cf664a03942e3d38280238988a247a2f Mon Sep 17 00:00:00 2001 From: Peetz0r Date: Fri, 29 Jan 2021 05:29:25 +0100 Subject: [PATCH 01/24] Prometheus Initial attempt. Seems to work fine. Only tested on debian amd64 so far --- group_vars/matrix_servers | 21 +++++ roles/matrix-prometheus/defaults/main.yml | 51 ++++++++++ roles/matrix-prometheus/tasks/init.yml | 5 + roles/matrix-prometheus/tasks/main.yml | 14 +++ roles/matrix-prometheus/tasks/setup.yml | 93 +++++++++++++++++++ .../tasks/validate_config.yml | 7 ++ .../templates/prometheus.yml.j2 | 35 +++++++ .../systemd/matrix-prometheus.service.j2 | 42 +++++++++ setup.yml | 1 + 9 files changed, 269 insertions(+) create mode 100644 roles/matrix-prometheus/defaults/main.yml create mode 100644 roles/matrix-prometheus/tasks/init.yml create mode 100644 roles/matrix-prometheus/tasks/main.yml create mode 100644 roles/matrix-prometheus/tasks/setup.yml create mode 100644 roles/matrix-prometheus/tasks/validate_config.yml create mode 100644 roles/matrix-prometheus/templates/prometheus.yml.j2 create mode 100644 roles/matrix-prometheus/templates/systemd/matrix-prometheus.service.j2 diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 5d76a60c..833089a2 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1368,6 +1368,27 @@ matrix_synapse_admin_container_self_build: "{{ matrix_architecture != 'amd64' }} +###################################################################### +# +# matrix-prometheus +# +###################################################################### + +matrix_prometheus_enabled: false + +# Normally, matrix-nginx-proxy is enabled and nginx can reach Prometheus over the container network. +# If matrix-nginx-proxy is not enabled, or you otherwise have a need for it, you can expose +# Prometheus' HTTP port to the local host. +matrix_prometheus_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else '127.0.0.1:9090' }}" + +###################################################################### +# +# /matrix-prometheus +# +###################################################################### + + + ###################################################################### # # matrix-registration diff --git a/roles/matrix-prometheus/defaults/main.yml b/roles/matrix-prometheus/defaults/main.yml new file mode 100644 index 00000000..10424424 --- /dev/null +++ b/roles/matrix-prometheus/defaults/main.yml @@ -0,0 +1,51 @@ +# matrix-prometheus is an open-source systems monitoring and alerting toolkit +# See: https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md + +matrix_prometheus_enabled: false + +matrix_prometheus_docker_image: "docker.io/prom/prometheus:v2.24.1" +matrix_prometheus_docker_image_force_pull: "{{ matrix_prometheus_docker_image.endswith(':latest') }}" + +matrix_synapse_prometheus_rules_download_url: "https://raw.githubusercontent.com/matrix-org/synapse/{{ matrix_synapse_docker_image_tag }}/contrib/prometheus/synapse-v2.rules" + +matrix_prometheus_base_path: "{{ matrix_base_data_path }}/prometheus" +matrix_prometheus_config_path: "{{ matrix_prometheus_base_path }}/config" +matrix_prometheus_data_path: "{{ matrix_prometheus_base_path }}/data" + +# A list of extra arguments to pass to the container +matrix_prometheus_container_extra_arguments: [] + +# List of systemd services that matrix-prometheus.service depends on +matrix_prometheus_systemd_required_services_list: ['docker.service'] + +# List of systemd services that matrix-prometheus.service wants +matrix_prometheus_systemd_wanted_services_list: [] + +# Controls whether the matrix-prometheus container exposes its HTTP port (tcp/9090 in the container). +# +# Takes an ":" or "" value (e.g. "127.0.0.1:9090"), or empty string to not expose. +matrix_prometheus_container_http_host_bind_port: '' + + +# Default prometheus configuration template which covers the generic use case. +# You can customize it by controlling the various variables inside it. +# +# For a more advanced customization, you can extend the default (see `matrix_prometheus_configuration_extension_yaml`) +# or completely replace this variable with your own template. +matrix_prometheus_configuration_yaml: "{{ lookup('template', 'templates/prometheus.yml.j2') }}" + +matrix_prometheus_configuration_extension_yaml: | + # Your custom YAML configuration goes here. + # This configuration extends the default starting configuration (`matrix_prometheus_configuration_yaml`). + # + # You can override individual variables from the default configuration, or introduce new ones. + # + # If you need something more special, you can take full control by + # completely redefining `matrix_prometheus_configuration_yaml`. + +matrix_prometheus_configuration_extension: "{{ matrix_prometheus_configuration_extension_yaml|from_yaml if matrix_prometheus_configuration_extension_yaml|from_yaml is mapping else {} }}" + +# Holds the final configuration (a combination of the default and its extension). +# You most likely don't need to touch this variable. Instead, see `matrix_prometheus_configuration_yaml`. +matrix_prometheus_configuration: "{{ matrix_prometheus_configuration_yaml|from_yaml|combine(matrix_prometheus_configuration_extension, recursive=True) }}" + diff --git a/roles/matrix-prometheus/tasks/init.yml b/roles/matrix-prometheus/tasks/init.yml new file mode 100644 index 00000000..12fae831 --- /dev/null +++ b/roles/matrix-prometheus/tasks/init.yml @@ -0,0 +1,5 @@ +- set_fact: + matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-prometheus.service'] }}" + when: matrix_prometheus_enabled|bool + + diff --git a/roles/matrix-prometheus/tasks/main.yml b/roles/matrix-prometheus/tasks/main.yml new file mode 100644 index 00000000..2290048f --- /dev/null +++ b/roles/matrix-prometheus/tasks/main.yml @@ -0,0 +1,14 @@ +- import_tasks: "{{ role_path }}/tasks/init.yml" + tags: + - always + +- import_tasks: "{{ role_path }}/tasks/validate_config.yml" + when: run_setup|bool + tags: + - setup-all + - setup-prometheus + +- import_tasks: "{{ role_path }}/tasks/setup.yml" + tags: + - setup-all + - setup-prometheus diff --git a/roles/matrix-prometheus/tasks/setup.yml b/roles/matrix-prometheus/tasks/setup.yml new file mode 100644 index 00000000..1746b961 --- /dev/null +++ b/roles/matrix-prometheus/tasks/setup.yml @@ -0,0 +1,93 @@ +--- + +# +# Tasks related to setting up matrix-prometheus +# + +- name: Ensure matrix-prometheus image is pulled + docker_image: + name: "{{ matrix_prometheus_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_prometheus_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_prometheus_docker_image_force_pull }}" + when: "matrix_prometheus_enabled|bool" + +- name: Ensure Prometheus paths exists + file: + path: "{{ item }}" + state: directory + mode: 0750 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + with_items: + - "{{ matrix_prometheus_base_path }}" + - "{{ matrix_prometheus_config_path }}" + - "{{ matrix_prometheus_data_path }}" + when: matrix_prometheus_enabled|bool + +- name: Ensure prometheus.yml installed + copy: + content: "{{ matrix_prometheus_configuration|to_nice_yaml }}" + dest: "{{ matrix_prometheus_config_path }}/prometheus.yml" + mode: 0644 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + when: matrix_prometheus_enabled|bool + +- name: Download synapse-v2.rules + get_url: + url: "{{ matrix_synapse_prometheus_rules_download_url }}" + dest: "{{ matrix_prometheus_config_path }}/synapse-v2.rules" + force: true + mode: 0440 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + when: matrix_prometheus_enabled|bool + + +- name: Ensure matrix-prometheus.service installed + template: + src: "{{ role_path }}/templates/systemd/matrix-prometheus.service.j2" + dest: "{{ matrix_systemd_path }}/matrix-prometheus.service" + mode: 0644 + register: matrix_prometheus_systemd_service_result + when: matrix_prometheus_enabled|bool + +- name: Ensure systemd reloaded after matrix-prometheus.service installation + service: + daemon_reload: yes + when: "matrix_prometheus_enabled|bool and matrix_prometheus_systemd_service_result.changed" + +# +# Tasks related to getting rid of matrix-prometheus (if it was previously enabled) +# + +- name: Check existence of matrix-prometheus service + stat: + path: "{{ matrix_systemd_path }}/matrix-prometheus.service" + register: matrix_prometheus_service_stat + +- name: Ensure matrix-prometheus is stopped + service: + name: matrix-prometheus + state: stopped + daemon_reload: yes + register: stopping_result + when: "not matrix_prometheus_enabled|bool and matrix_prometheus_service_stat.stat.exists" + +- name: Ensure matrix-prometheus.service doesn't exist + file: + path: "{{ matrix_systemd_path }}/matrix-prometheus.service" + state: absent + when: "not matrix_prometheus_enabled|bool and matrix_prometheus_service_stat.stat.exists" + +- name: Ensure systemd reloaded after matrix-prometheus.service removal + service: + daemon_reload: yes + when: "not matrix_prometheus_enabled|bool and matrix_prometheus_service_stat.stat.exists" + +- name: Ensure matrix-prometheus Docker image doesn't exist + docker_image: + name: "{{ matrix_prometheus_docker_image }}" + state: absent + when: "not matrix_prometheus_enabled|bool" diff --git a/roles/matrix-prometheus/tasks/validate_config.yml b/roles/matrix-prometheus/tasks/validate_config.yml new file mode 100644 index 00000000..b614b438 --- /dev/null +++ b/roles/matrix-prometheus/tasks/validate_config.yml @@ -0,0 +1,7 @@ +--- + +- name: Fail if Synapse metrics not enabled + fail: + msg: > + You need to enable `matrix_synapse_metrics_enabled` for Prometheus grab metrics. + when: "not matrix_synapse_metrics_enabled" diff --git a/roles/matrix-prometheus/templates/prometheus.yml.j2 b/roles/matrix-prometheus/templates/prometheus.yml.j2 new file mode 100644 index 00000000..7b90baeb --- /dev/null +++ b/roles/matrix-prometheus/templates/prometheus.yml.j2 @@ -0,0 +1,35 @@ +#jinja2: lstrip_blocks: "True" +global: + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: + {% if matrix_synapse_metrics_enabled %} + - 'synapse-v2.rules' + {% endif %} + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + - job_name: 'prometheus' + + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 5s + scrape_timeout: 5s + + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + + static_configs: + - targets: ['localhost:9090'] + + {% if matrix_synapse_metrics_enabled %} + - job_name: 'synapse' + metrics_path: '/_synapse/metrics' + static_configs: + - targets: ['matrix-synapse:{{ matrix_synapse_metrics_port }}'] + {% endif %} + diff --git a/roles/matrix-prometheus/templates/systemd/matrix-prometheus.service.j2 b/roles/matrix-prometheus/templates/systemd/matrix-prometheus.service.j2 new file mode 100644 index 00000000..dd3ac72c --- /dev/null +++ b/roles/matrix-prometheus/templates/systemd/matrix-prometheus.service.j2 @@ -0,0 +1,42 @@ +#jinja2: lstrip_blocks: "True" +[Unit] +Description=matrix-prometheus +{% for service in matrix_prometheus_systemd_required_services_list %} +Requires={{ service }} +After={{ service }} +{% endfor %} +{% for service in matrix_prometheus_systemd_wanted_services_list %} +Wants={{ service }} +{% endfor %} +DefaultDependencies=no + +[Service] +Type=simple +Environment="HOME={{ matrix_systemd_unit_home_path }}" +ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus 2>/dev/null' +ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus 2>/dev/null' + + +ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-prometheus \ + --log-driver=none \ + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + --cap-drop=ALL \ + --network={{ matrix_docker_network }} \ + {% if matrix_prometheus_container_http_host_bind_port %} + -p {{ matrix_prometheus_container_http_host_bind_port }}:9090 \ + {% endif %} + -v {{ matrix_prometheus_config_path }}:/etc/prometheus:z \ + -v {{ matrix_prometheus_data_path }}:/prometheus:z \ + {% for arg in matrix_prometheus_container_extra_arguments %} + {{ arg }} \ + {% endfor %} + {{ matrix_prometheus_docker_image }} + +ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus 2>/dev/null' +ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus 2>/dev/null' +Restart=always +RestartSec=30 +SyslogIdentifier=matrix-prometheus + +[Install] +WantedBy=multi-user.target diff --git a/setup.yml b/setup.yml index 9bb1788f..7965185c 100755 --- a/setup.yml +++ b/setup.yml @@ -28,6 +28,7 @@ - matrix-bot-matrix-reminder-bot - matrix-synapse - matrix-synapse-admin + - matrix-prometheus - matrix-registration - matrix-client-element - matrix-jitsi From e525970b393350f9280bdf8ddcb78f9d55c1da24 Mon Sep 17 00:00:00 2001 From: Peetz0r Date: Fri, 29 Jan 2021 06:17:45 +0100 Subject: [PATCH 02/24] Prometheus Node Exporter Basic system stats, to show stuff the synapse metrics can't show such as resource usage by bridges, etc Seems to work fine as well. This too has only been tested on debian amd64 so far --- group_vars/matrix_servers | 21 +++++++ .../defaults/main.yml | 21 +++++++ .../tasks/init.yml | 5 ++ .../tasks/main.yml | 8 +++ .../tasks/setup.yml | 60 +++++++++++++++++++ .../tasks/validate_config.yml | 7 +++ ...matrix-prometheus-node-exporter.service.j2 | 40 +++++++++++++ .../tasks/validate_config.yml | 6 +- .../templates/prometheus.yml.j2 | 5 ++ setup.yml | 1 + 10 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 roles/matrix-prometheus-node-exporter/defaults/main.yml create mode 100644 roles/matrix-prometheus-node-exporter/tasks/init.yml create mode 100644 roles/matrix-prometheus-node-exporter/tasks/main.yml create mode 100644 roles/matrix-prometheus-node-exporter/tasks/setup.yml create mode 100644 roles/matrix-prometheus-node-exporter/tasks/validate_config.yml create mode 100644 roles/matrix-prometheus-node-exporter/templates/systemd/matrix-prometheus-node-exporter.service.j2 diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 833089a2..4f2cfa6a 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1368,6 +1368,27 @@ matrix_synapse_admin_container_self_build: "{{ matrix_architecture != 'amd64' }} +###################################################################### +# +# matrix-prometheus-node-exporter +# +###################################################################### + +matrix_prometheus_node_exporter_enabled: false + +# Normally, matrix-nginx-proxy is enabled and nginx can reach Prometheus Node Exporter over the container network. +# If matrix-nginx-proxy is not enabled, or you otherwise have a need for it, you can expose +# Prometheus' HTTP port to the local host. +matrix_prometheus_node_exporter_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else '127.0.0.1:9100' }}" + +###################################################################### +# +# /matrix-prometheus-node-exporter +# +###################################################################### + + + ###################################################################### # # matrix-prometheus diff --git a/roles/matrix-prometheus-node-exporter/defaults/main.yml b/roles/matrix-prometheus-node-exporter/defaults/main.yml new file mode 100644 index 00000000..a5664b83 --- /dev/null +++ b/roles/matrix-prometheus-node-exporter/defaults/main.yml @@ -0,0 +1,21 @@ +# matrix-prometheus-node-exporter is an Prometheus exporter for machine metrics +# See: https://prometheus.io/docs/guides/node-exporter/ + +matrix_prometheus_node_exporter_enabled: false + +matrix_prometheus_node_exporter_docker_image: "docker.io/prom/node-exporter:v1.0.1" +matrix_prometheus_node_exporter_docker_image_force_pull: "{{ matrix_prometheus_node_exporter_docker_image.endswith(':latest') }}" + +# A list of extra arguments to pass to the container +matrix_prometheus_node_exporter_container_extra_arguments: [] + +# List of systemd services that matrix-prometheus.service depends on +matrix_prometheus_node_exporter_systemd_required_services_list: ['docker.service'] + +# List of systemd services that matrix-prometheus.service wants +matrix_prometheus_node_exporter_systemd_wanted_services_list: [] + +# Controls whether the matrix-prometheus container exposes its HTTP port (tcp/9100 in the container). +# +# Takes an ":" or "" value (e.g. "127.0.0.1:9100"), or empty string to not expose. +matrix_prometheus_node_exporter_container_http_host_bind_port: '' diff --git a/roles/matrix-prometheus-node-exporter/tasks/init.yml b/roles/matrix-prometheus-node-exporter/tasks/init.yml new file mode 100644 index 00000000..2894b717 --- /dev/null +++ b/roles/matrix-prometheus-node-exporter/tasks/init.yml @@ -0,0 +1,5 @@ +- set_fact: + matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-prometheus-node-exporter.service'] }}" + when: matrix_prometheus_node_exporter_enabled|bool + + diff --git a/roles/matrix-prometheus-node-exporter/tasks/main.yml b/roles/matrix-prometheus-node-exporter/tasks/main.yml new file mode 100644 index 00000000..172b5721 --- /dev/null +++ b/roles/matrix-prometheus-node-exporter/tasks/main.yml @@ -0,0 +1,8 @@ +- import_tasks: "{{ role_path }}/tasks/init.yml" + tags: + - always + +- import_tasks: "{{ role_path }}/tasks/setup.yml" + tags: + - setup-all + - setup-prometheus-node-exporter diff --git a/roles/matrix-prometheus-node-exporter/tasks/setup.yml b/roles/matrix-prometheus-node-exporter/tasks/setup.yml new file mode 100644 index 00000000..6f03fbaa --- /dev/null +++ b/roles/matrix-prometheus-node-exporter/tasks/setup.yml @@ -0,0 +1,60 @@ +--- + +# +# Tasks related to setting up matrix-prometheus-node-exporter +# + +- name: Ensure matrix-prometheus-node-exporter image is pulled + docker_image: + name: "{{ matrix_prometheus_node_exporter_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_prometheus_node_exporter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_prometheus_node_exporter_docker_image_force_pull }}" + when: "matrix_prometheus_node_exporter_enabled|bool" + +- name: Ensure matrix-prometheus-node-exporter.service installed + template: + src: "{{ role_path }}/templates/systemd/matrix-prometheus-node-exporter.service.j2" + dest: "{{ matrix_systemd_path }}/matrix-prometheus-node-exporter.service" + mode: 0644 + register: matrix_prometheus_node_exporter_systemd_service_result + when: matrix_prometheus_node_exporter_enabled|bool + +- name: Ensure systemd reloaded after matrix-prometheus.service installation + service: + daemon_reload: yes + when: "matrix_prometheus_node_exporter_enabled|bool and matrix_prometheus_node_exporter_systemd_service_result.changed" + +# +# Tasks related to getting rid of matrix-prometheus-node-exporter (if it was previously enabled) +# + +- name: Check existence of matrix-prometheus-node-exporter service + stat: + path: "{{ matrix_systemd_path }}/matrix-prometheus-node-exporter.service" + register: matrix_prometheus_node_exporter_service_stat + +- name: Ensure matrix-prometheus-node-exporter is stopped + service: + name: matrix-prometheus-node-exporter + state: stopped + daemon_reload: yes + register: stopping_result + when: "not matrix_prometheus_node_exporter_enabled|bool and matrix_prometheus_node_exporter_service_stat.stat.exists" + +- name: Ensure matrix-prometheus-node-exporter.service doesn't exist + file: + path: "{{ matrix_systemd_path }}/matrix-prometheus-node-exporter.service" + state: absent + when: "not matrix_prometheus_node_exporter_enabled|bool and matrix_prometheus_node_exporter_service_stat.stat.exists" + +- name: Ensure systemd reloaded after matrix-prometheus-node-exporter.service removal + service: + daemon_reload: yes + when: "not matrix_prometheus_node_exporter_enabled|bool and matrix_prometheus_node_exporter_service_stat.stat.exists" + +- name: Ensure matrix-prometheus-node-exporter Docker image doesn't exist + docker_image: + name: "{{ matrix_prometheus_node_exporter_docker_image }}" + state: absent + when: "not matrix_prometheus_node_exporter_enabled|bool" diff --git a/roles/matrix-prometheus-node-exporter/tasks/validate_config.yml b/roles/matrix-prometheus-node-exporter/tasks/validate_config.yml new file mode 100644 index 00000000..713646ae --- /dev/null +++ b/roles/matrix-prometheus-node-exporter/tasks/validate_config.yml @@ -0,0 +1,7 @@ +--- + +- name: Fail if Synapse metrics or Prometheus Node Exporter not enabled + fail: + msg: > + You need to enable `matrix_synapse_metrics_enabled` and/or `matrix_prometheus_node_exporter_enabled` for Prometheus grab metrics. + when: "not matrix_synapse_metrics_enabled and not matrix_prometheus_node_exporter_enabled" diff --git a/roles/matrix-prometheus-node-exporter/templates/systemd/matrix-prometheus-node-exporter.service.j2 b/roles/matrix-prometheus-node-exporter/templates/systemd/matrix-prometheus-node-exporter.service.j2 new file mode 100644 index 00000000..622947d0 --- /dev/null +++ b/roles/matrix-prometheus-node-exporter/templates/systemd/matrix-prometheus-node-exporter.service.j2 @@ -0,0 +1,40 @@ +#jinja2: lstrip_blocks: "True" +[Unit] +Description=matrix-prometheus-node-exporter +{% for service in matrix_prometheus_node_exporter_systemd_required_services_list %} +Requires={{ service }} +After={{ service }} +{% endfor %} +{% for service in matrix_prometheus_node_exporter_systemd_wanted_services_list %} +Wants={{ service }} +{% endfor %} +DefaultDependencies=no + +[Service] +Type=simple +Environment="HOME={{ matrix_systemd_unit_home_path }}" +ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus-node-exporter 2>/dev/null' +ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus-node-exporter 2>/dev/null' + + +ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-prometheus-node-exporter \ + --log-driver=none \ + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + --cap-drop=ALL \ + --network={{ matrix_docker_network }} \ + {% if matrix_prometheus_node_exporter_container_http_host_bind_port %} + -p {{ matrix_prometheus_node_exporter_container_http_host_bind_port }}:9100 \ + {% endif %} + {% for arg in matrix_prometheus_node_exporter_container_extra_arguments %} + {{ arg }} \ + {% endfor %} + {{ matrix_prometheus_node_exporter_docker_image }} + +ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus-node-exporter 2>/dev/null' +ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus-node-exporter 2>/dev/null' +Restart=always +RestartSec=30 +SyslogIdentifier=matrix-prometheus-node-exporter + +[Install] +WantedBy=multi-user.target diff --git a/roles/matrix-prometheus/tasks/validate_config.yml b/roles/matrix-prometheus/tasks/validate_config.yml index b614b438..713646ae 100644 --- a/roles/matrix-prometheus/tasks/validate_config.yml +++ b/roles/matrix-prometheus/tasks/validate_config.yml @@ -1,7 +1,7 @@ --- -- name: Fail if Synapse metrics not enabled +- name: Fail if Synapse metrics or Prometheus Node Exporter not enabled fail: msg: > - You need to enable `matrix_synapse_metrics_enabled` for Prometheus grab metrics. - when: "not matrix_synapse_metrics_enabled" + You need to enable `matrix_synapse_metrics_enabled` and/or `matrix_prometheus_node_exporter_enabled` for Prometheus grab metrics. + when: "not matrix_synapse_metrics_enabled and not matrix_prometheus_node_exporter_enabled" diff --git a/roles/matrix-prometheus/templates/prometheus.yml.j2 b/roles/matrix-prometheus/templates/prometheus.yml.j2 index 7b90baeb..317dcd16 100644 --- a/roles/matrix-prometheus/templates/prometheus.yml.j2 +++ b/roles/matrix-prometheus/templates/prometheus.yml.j2 @@ -33,3 +33,8 @@ scrape_configs: - targets: ['matrix-synapse:{{ matrix_synapse_metrics_port }}'] {% endif %} + {% if matrix_prometheus_node_exporter_enabled %} + - job_name: node + static_configs: + - targets: ['matrix-prometheus-node-exporter:9100'] + {% endif %} diff --git a/setup.yml b/setup.yml index 7965185c..838e08c8 100755 --- a/setup.yml +++ b/setup.yml @@ -28,6 +28,7 @@ - matrix-bot-matrix-reminder-bot - matrix-synapse - matrix-synapse-admin + - matrix-prometheus-node-exporter - matrix-prometheus - matrix-registration - matrix-client-element From eb5aa93e8a5b83593de4030982e583eae8aa8ee8 Mon Sep 17 00:00:00 2001 From: Peetz0r Date: Fri, 29 Jan 2021 09:10:49 +0100 Subject: [PATCH 03/24] Grafana Also includes the dashboards for Synapse and for Node Exporter. Again has only been tested on debian amd64 so far, but the grafana docker image is available for arm64 and arm32. Nice. --- group_vars/matrix_servers | 21 ++++ roles/matrix-grafana/defaults/main.yml | 47 +++++++ roles/matrix-grafana/tasks/init.yml | 5 + roles/matrix-grafana/tasks/main.yml | 14 +++ roles/matrix-grafana/tasks/setup.yml | 115 ++++++++++++++++++ .../matrix-grafana/tasks/validate_config.yml | 7 ++ .../templates/dashboards.yaml.j2 | 9 ++ .../templates/datasources.yaml.j2 | 8 ++ roles/matrix-grafana/templates/grafana.ini.j2 | 20 +++ .../systemd/matrix-grafana.service.j2 | 42 +++++++ setup.yml | 1 + 11 files changed, 289 insertions(+) create mode 100644 roles/matrix-grafana/defaults/main.yml create mode 100644 roles/matrix-grafana/tasks/init.yml create mode 100644 roles/matrix-grafana/tasks/main.yml create mode 100644 roles/matrix-grafana/tasks/setup.yml create mode 100644 roles/matrix-grafana/tasks/validate_config.yml create mode 100644 roles/matrix-grafana/templates/dashboards.yaml.j2 create mode 100644 roles/matrix-grafana/templates/datasources.yaml.j2 create mode 100644 roles/matrix-grafana/templates/grafana.ini.j2 create mode 100644 roles/matrix-grafana/templates/systemd/matrix-grafana.service.j2 diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 4f2cfa6a..976a0de1 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1410,6 +1410,27 @@ matrix_prometheus_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_en +###################################################################### +# +# matrix-grafana +# +###################################################################### + +matrix_grafana_enabled: false + +# Normally, matrix-nginx-proxy is enabled and nginx can reach Grafana over the container network. +# If matrix-nginx-proxy is not enabled, or you otherwise have a need for it, you can expose +# Grafana's HTTP port to the local host. +matrix_grafana_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else '127.0.0.1:3000' }}" + +###################################################################### +# +# /matrix-grafana +# +###################################################################### + + + ###################################################################### # # matrix-registration diff --git a/roles/matrix-grafana/defaults/main.yml b/roles/matrix-grafana/defaults/main.yml new file mode 100644 index 00000000..2257d794 --- /dev/null +++ b/roles/matrix-grafana/defaults/main.yml @@ -0,0 +1,47 @@ +# matrix-grafana is open source visualization and analytics software +# See: https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md + +matrix_grafana_enabled: false + +matrix_grafana_docker_image: "docker.io/grafana/grafana:7.3.7" +matrix_grafana_docker_image_force_pull: "{{ matrix_grafana_docker_image.endswith(':latest') }}" + +# Not conditional, because when someone disables metrics +# they might still want to look at the old existing data. +# So it would be silly to delete the dashboard in such case. +matrix_grafana_dashboard_download_urls: +- "https://raw.githubusercontent.com/matrix-org/synapse/master/contrib/grafana/synapse.json" +- "https://raw.githubusercontent.com/rfrail3/grafana-dashboards/master/prometheus/node-exporter-full.json" + +matrix_grafana_base_path: "{{ matrix_base_data_path }}/grafana" +matrix_grafana_config_path: "{{ matrix_grafana_base_path }}/config" +matrix_grafana_data_path: "{{ matrix_grafana_base_path }}/data" + +# Allow viewing Grafana without logging in +matrix_grafana_anonymous_access: false + +# specify organization name that should be used for unauthenticated users +# if you change this in the Grafana admin panel, this needs to be updated +# to match to keep anonymous logins working +matrix_grafana_anonymous_access_org_name: 'Main Org.' + + +# default admin credentials, you are asked to change these on first login +matrix_grafana_default_admin_user: admin +matrix_grafana_default_admin_password: admin + +# A list of extra arguments to pass to the container +matrix_grafana_container_extra_arguments: [] + +# List of systemd services that matrix-grafana.service depends on +matrix_grafana_systemd_required_services_list: ['docker.service'] + +# List of systemd services that matrix-grafana.service wants +matrix_grafana_systemd_wanted_services_list: [] + +# Controls whether the matrix-grafana container exposes its HTTP port (tcp/3000 in the container). +# +# Takes an ":" or "" value (e.g. "127.0.0.1:3000"), or empty string to not expose. +matrix_grafana_container_http_host_bind_port: '' + + diff --git a/roles/matrix-grafana/tasks/init.yml b/roles/matrix-grafana/tasks/init.yml new file mode 100644 index 00000000..8a22e301 --- /dev/null +++ b/roles/matrix-grafana/tasks/init.yml @@ -0,0 +1,5 @@ +- set_fact: + matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-grafana.service'] }}" + when: matrix_grafana_enabled|bool + + diff --git a/roles/matrix-grafana/tasks/main.yml b/roles/matrix-grafana/tasks/main.yml new file mode 100644 index 00000000..122ec65e --- /dev/null +++ b/roles/matrix-grafana/tasks/main.yml @@ -0,0 +1,14 @@ +- import_tasks: "{{ role_path }}/tasks/init.yml" + tags: + - always + +- import_tasks: "{{ role_path }}/tasks/validate_config.yml" + when: run_setup|bool + tags: + - setup-all + - setup-grafana + +- import_tasks: "{{ role_path }}/tasks/setup.yml" + tags: + - setup-all + - setup-grafana diff --git a/roles/matrix-grafana/tasks/setup.yml b/roles/matrix-grafana/tasks/setup.yml new file mode 100644 index 00000000..581e6617 --- /dev/null +++ b/roles/matrix-grafana/tasks/setup.yml @@ -0,0 +1,115 @@ +--- + +# +# Tasks related to setting up matrix-grafana +# + +- name: Ensure matrix-grafana image is pulled + docker_image: + name: "{{ matrix_grafana_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_grafana_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_grafana_docker_image_force_pull }}" + when: "matrix_grafana_enabled|bool" + +- name: Ensure grafana paths exists + file: + path: "{{ item }}" + state: directory + mode: 0750 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + with_items: + - "{{ matrix_grafana_base_path }}" + - "{{ matrix_grafana_config_path }}" + - "{{ matrix_grafana_config_path }}/provisioning" + - "{{ matrix_grafana_config_path }}/provisioning/datasources" + - "{{ matrix_grafana_config_path }}/provisioning/dashboards" + - "{{ matrix_grafana_config_path }}/dashboards" + - "{{ matrix_grafana_data_path }}" + when: matrix_grafana_enabled|bool + +- name: Ensure grafana.ini present + template: + src: "{{ role_path }}/templates/grafana.ini.j2" + dest: "{{ matrix_grafana_config_path }}/grafana.ini" + mode: 0440 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + when: matrix_grafana_enabled|bool + +- name: Ensure provisioning/datasources/default.yaml present + template: + src: "{{ role_path }}/templates/datasources.yaml.j2" + dest: "{{ matrix_grafana_config_path }}/provisioning/datasources/default.yaml" + mode: 0440 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + when: matrix_grafana_enabled|bool + +- name: Ensure provisioning/dashboards/default.yaml present + template: + src: "{{ role_path }}/templates/dashboards.yaml.j2" + dest: "{{ matrix_grafana_config_path }}/provisioning/dashboards/default.yaml" + mode: 0440 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + when: matrix_grafana_enabled|bool + +- name: Ensure dashboard(s) downloaded + get_url: + url: "{{ item }}" + dest: "{{ matrix_grafana_config_path }}/dashboards/" + force: true + mode: 0440 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + with_items: "{{ matrix_grafana_dashboard_download_urls }}" + when: matrix_grafana_enabled|bool + +- name: Ensure matrix-grafana.service installed + template: + src: "{{ role_path }}/templates/systemd/matrix-grafana.service.j2" + dest: "{{ matrix_systemd_path }}/matrix-grafana.service" + mode: 0644 + register: matrix_grafana_systemd_service_result + when: matrix_grafana_enabled|bool + +- name: Ensure systemd reloaded after matrix-grafana.service installation + service: + daemon_reload: yes + when: "matrix_grafana_enabled|bool and matrix_grafana_systemd_service_result.changed" + +# +# Tasks related to getting rid of matrix-grafana (if it was previously enabled) +# + +- name: Check existence of matrix-grafana service + stat: + path: "{{ matrix_systemd_path }}/matrix-grafana.service" + register: matrix_grafana_service_stat + +- name: Ensure matrix-grafana is stopped + service: + name: matrix-grafana + state: stopped + daemon_reload: yes + register: stopping_result + when: "not matrix_grafana_enabled|bool and matrix_grafana_service_stat.stat.exists" + +- name: Ensure matrix-grafana.service doesn't exist + file: + path: "{{ matrix_systemd_path }}/matrix-grafana.service" + state: absent + when: "not matrix_grafana_enabled|bool and matrix_grafana_service_stat.stat.exists" + +- name: Ensure systemd reloaded after matrix-grafana.service removal + service: + daemon_reload: yes + when: "not matrix_grafana_enabled|bool and matrix_grafana_service_stat.stat.exists" + +- name: Ensure matrix-grafana Docker image doesn't exist + docker_image: + name: "{{ matrix_grafana_docker_image }}" + state: absent + when: "not matrix_grafana_enabled|bool" diff --git a/roles/matrix-grafana/tasks/validate_config.yml b/roles/matrix-grafana/tasks/validate_config.yml new file mode 100644 index 00000000..63d4919a --- /dev/null +++ b/roles/matrix-grafana/tasks/validate_config.yml @@ -0,0 +1,7 @@ +--- + +- name: Fail if Prometheus not enabled + fail: + msg: > + You need to enable `matrix_prometheus_enabled` to use Prometheus as data source for Grafana. + when: "not matrix_prometheus_enabled" diff --git a/roles/matrix-grafana/templates/dashboards.yaml.j2 b/roles/matrix-grafana/templates/dashboards.yaml.j2 new file mode 100644 index 00000000..b6662e59 --- /dev/null +++ b/roles/matrix-grafana/templates/dashboards.yaml.j2 @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: + - name: {{ matrix_domain }} - Dashboards + folder: '' # The folder where to place the dashboards + type: file + allowUiUpdates: true + options: + path: /etc/grafana/dashboards diff --git a/roles/matrix-grafana/templates/datasources.yaml.j2 b/roles/matrix-grafana/templates/datasources.yaml.j2 new file mode 100644 index 00000000..ffa6046b --- /dev/null +++ b/roles/matrix-grafana/templates/datasources.yaml.j2 @@ -0,0 +1,8 @@ +apiVersion: 1 + +datasources: + - name: {{ matrix_domain }} - Prometheus + type: prometheus + # Access mode - proxy (server in the UI) or direct (browser in the UI). + access: proxy + url: http://matrix-prometheus:9090 diff --git a/roles/matrix-grafana/templates/grafana.ini.j2 b/roles/matrix-grafana/templates/grafana.ini.j2 new file mode 100644 index 00000000..694bf7d7 --- /dev/null +++ b/roles/matrix-grafana/templates/grafana.ini.j2 @@ -0,0 +1,20 @@ +[security] +# default admin user, created on startup +admin_user = {{ matrix_grafana_default_admin_user }} + +# default admin password, can be changed before first start of grafana, or in profile settings +admin_password = {{ matrix_grafana_default_admin_password }} + +[auth.anonymous] +# enable anonymous access +enabled = {{ matrix_grafana_anonymous_access }} + +# specify organization name that should be used for unauthenticated users +org_name = {{ matrix_grafana_anonymous_access_org_name }} + +[dashboards] +{% if matrix_synapse_metrics_enabled %} +default_home_dashboard_path = /etc/grafana/dashboards/synapse.json +{% else %} +default_home_dashboard_path = /etc/grafana/dashboards/node-exporter-full.json +{% endif %} diff --git a/roles/matrix-grafana/templates/systemd/matrix-grafana.service.j2 b/roles/matrix-grafana/templates/systemd/matrix-grafana.service.j2 new file mode 100644 index 00000000..f2ab6642 --- /dev/null +++ b/roles/matrix-grafana/templates/systemd/matrix-grafana.service.j2 @@ -0,0 +1,42 @@ +#jinja2: lstrip_blocks: "True" +[Unit] +Description=matrix-grafana +{% for service in matrix_grafana_systemd_required_services_list %} +Requires={{ service }} +After={{ service }} +{% endfor %} +{% for service in matrix_grafana_systemd_wanted_services_list %} +Wants={{ service }} +{% endfor %} +DefaultDependencies=no + +[Service] +Type=simple +Environment="HOME={{ matrix_systemd_unit_home_path }}" +ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-grafana 2>/dev/null' +ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-grafana 2>/dev/null' + + +ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-grafana \ + --log-driver=none \ + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + --cap-drop=ALL \ + --network={{ matrix_docker_network }} \ + {% if matrix_grafana_container_http_host_bind_port %} + -p {{ matrix_grafana_container_http_host_bind_port }}:3000 \ + {% endif %} + -v {{ matrix_grafana_config_path }}:/etc/grafana:z \ + -v {{ matrix_grafana_data_path }}:/var/lib/grafana:z \ + {% for arg in matrix_grafana_container_extra_arguments %} + {{ arg }} \ + {% endfor %} + {{ matrix_grafana_docker_image }} + +ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-grafana 2>/dev/null' +ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-grafana 2>/dev/null' +Restart=always +RestartSec=30 +SyslogIdentifier=matrix-grafana + +[Install] +WantedBy=multi-user.target diff --git a/setup.yml b/setup.yml index 838e08c8..e7fdae19 100755 --- a/setup.yml +++ b/setup.yml @@ -30,6 +30,7 @@ - matrix-synapse-admin - matrix-prometheus-node-exporter - matrix-prometheus + - matrix-grafana - matrix-registration - matrix-client-element - matrix-jitsi From 989100b1c14a126051d7ad68c323d954061714b8 Mon Sep 17 00:00:00 2001 From: Peetz0r Date: Fri, 29 Jan 2021 10:30:04 +0100 Subject: [PATCH 04/24] Grafana nginx proxy config --- group_vars/matrix_servers | 3 + roles/matrix-base/defaults/main.yml | 3 + roles/matrix-nginx-proxy/defaults/main.yml | 7 ++ .../tasks/setup_nginx_proxy.yml | 13 +++ .../nginx/conf.d/matrix-grafana.conf.j2 | 79 +++++++++++++++++++ 5 files changed, 105 insertions(+) create mode 100644 roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-grafana.conf.j2 diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 976a0de1..b314dc99 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -974,6 +974,7 @@ matrix_nginx_proxy_proxy_matrix_enabled: true matrix_nginx_proxy_proxy_element_enabled: "{{ matrix_client_element_enabled }}" matrix_nginx_proxy_proxy_dimension_enabled: "{{ matrix_dimension_enabled }}" matrix_nginx_proxy_proxy_jitsi_enabled: "{{ matrix_jitsi_enabled }}" +matrix_nginx_proxy_proxy_grafana_enabled: "{{ matrix_grafana_enabled }}" matrix_nginx_proxy_proxy_matrix_corporal_api_enabled: "{{ matrix_corporal_enabled and matrix_corporal_http_api_enabled }}" matrix_nginx_proxy_proxy_matrix_corporal_api_addr_with_container: "matrix-corporal:41081" @@ -1024,6 +1025,8 @@ matrix_ssl_domains_to_obtain_certificates_for: | + ([matrix_server_fqn_jitsi] if matrix_jitsi_enabled else []) + + ([matrix_server_fqn_grafana] if matrix_grafana_enabled else []) + + ([matrix_domain] if matrix_nginx_proxy_base_domain_serving_enabled else []) }} diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index a238e503..2aa99a32 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -21,6 +21,9 @@ matrix_server_fqn_dimension: "dimension.{{ matrix_domain }}" # This is where you access Jitsi. matrix_server_fqn_jitsi: "jitsi.{{ matrix_domain }}" +# This is where you access Grafana. +matrix_server_fqn_grafana: "stats.{{ matrix_domain }}" + matrix_federation_public_port: 8448 # The architecture that your server runs. diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 5eedb4ce..d0ff6d95 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -115,6 +115,10 @@ matrix_nginx_proxy_proxy_dimension_hostname: "{{ matrix_server_fqn_dimension }}" matrix_nginx_proxy_proxy_jitsi_enabled: false matrix_nginx_proxy_proxy_jitsi_hostname: "{{ matrix_server_fqn_jitsi }}" +# Controls whether proxying the grafana domain should be done. +matrix_nginx_proxy_proxy_grafana_enabled: false +matrix_nginx_proxy_proxy_grafana_hostname: "{{ matrix_server_fqn_grafana }}" + # Controls whether proxying for the matrix-corporal API (`/_matrix/corporal`) should be done (on the matrix domain) matrix_nginx_proxy_proxy_matrix_corporal_api_enabled: false matrix_nginx_proxy_proxy_matrix_corporal_api_addr_with_container: "matrix-corporal:41081" @@ -212,6 +216,9 @@ matrix_nginx_proxy_proxy_dimension_additional_server_configuration_blocks: [] # A list of strings containing additional configuration blocks to add to Jitsi's server configuration. matrix_nginx_proxy_proxy_jitsi_additional_server_configuration_blocks: [] +# A list of strings containing additional configuration blocks to add to Grafana's server configuration. +matrix_nginx_proxy_proxy_grafana_additional_server_configuration_blocks: [] + # A list of strings containing additional configuration blocks to add to the base domain server configuration. matrix_nginx_proxy_proxy_domain_additional_server_configuration_blocks: [] diff --git a/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml b/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml index 90f0da73..9d7ea515 100644 --- a/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml +++ b/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml @@ -80,6 +80,13 @@ mode: 0644 when: matrix_nginx_proxy_proxy_jitsi_enabled|bool +- name: Ensure Matrix nginx-proxy configuration for grafana domain exists + template: + src: "{{ role_path }}/templates/nginx/conf.d/matrix-grafana.conf.j2" + dest: "{{ matrix_nginx_proxy_confd_path }}/matrix-grafana.conf" + mode: 0644 + when: matrix_nginx_proxy_proxy_grafana_enabled|bool + - name: Ensure Matrix nginx-proxy data directory for base domain exists file: path: "{{ matrix_nginx_proxy_data_path }}/matrix-domain" @@ -183,6 +190,12 @@ state: absent when: "not matrix_nginx_proxy_proxy_jitsi_enabled|bool" +- name: Ensure Matrix nginx-proxy configuration for grafana domain deleted + file: + path: "{{ matrix_nginx_proxy_confd_path }}/matrix-grafana.conf" + state: absent + when: "not matrix_nginx_proxy_proxy_grafana_enabled|bool" + - name: Ensure Matrix nginx-proxy homepage for base domain deleted file: path: "{{ matrix_nginx_proxy_data_path }}/matrix-domain/index.html" diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-grafana.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-grafana.conf.j2 new file mode 100644 index 00000000..0e1f1c2d --- /dev/null +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-grafana.conf.j2 @@ -0,0 +1,79 @@ +#jinja2: lstrip_blocks: "True" + +{% macro render_vhost_directives() %} + gzip on; + gzip_types text/plain application/json application/javascript text/css image/x-icon font/ttf image/gif; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Content-Type-Options nosniff; + add_header X-Frame-Options SAMEORIGIN; + {% for configuration_block in matrix_nginx_proxy_proxy_grafana_additional_server_configuration_blocks %} + {{- configuration_block }} + {% endfor %} + + location / { + {% if matrix_nginx_proxy_enabled %} + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-grafana:3000"; + proxy_pass http://$backend; + {% else %} + {# Generic configuration for use outside of our container setup #} + proxy_pass http://127.0.0.1:3000; + {% endif %} + + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } +{% endmacro %} + +server { + listen {{ 8080 if matrix_nginx_proxy_enabled else 80 }}; + + server_name {{ matrix_nginx_proxy_proxy_grafana_hostname }}; + + server_tokens off; + root /dev/null; + + {% if matrix_nginx_proxy_https_enabled %} + location /.well-known/acme-challenge { + {% if matrix_nginx_proxy_enabled %} + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-certbot:8080"; + proxy_pass http://$backend; + {% else %} + {# Generic configuration for use outside of our container setup #} + proxy_pass http://127.0.0.1:{{ matrix_ssl_lets_encrypt_certbot_standalone_http_port }}; + {% endif %} + } + + location / { + return 301 https://$http_host$request_uri; + } + {% else %} + {{ render_vhost_directives() }} + {% endif %} +} + +{% if matrix_nginx_proxy_https_enabled %} +server { + listen {{ 8443 if matrix_nginx_proxy_enabled else 443 }} ssl http2; + listen [::]:{{ 8443 if matrix_nginx_proxy_enabled else 443 }} ssl http2; + + server_name {{ matrix_nginx_proxy_proxy_grafana_hostname }}; + + server_tokens off; + root /dev/null; + + ssl_certificate {{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_proxy_grafana_hostname }}/fullchain.pem; + ssl_certificate_key {{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_proxy_grafana_hostname }}/privkey.pem; + + ssl_protocols {{ matrix_nginx_proxy_ssl_protocols }}; + {% if matrix_nginx_proxy_ssl_ciphers != "" %} + ssl_ciphers {{ matrix_nginx_proxy_ssl_ciphers }}; + {% endif %} + ssl_prefer_server_ciphers {{ matrix_nginx_proxy_ssl_prefer_server_ciphers }}; + + {{ render_vhost_directives() }} +} +{% endif %} From a10e3244d914cd42ef88688949994d245b852556 Mon Sep 17 00:00:00 2001 From: Peetz0r Date: Fri, 29 Jan 2021 10:59:27 +0100 Subject: [PATCH 05/24] Documentation for graphs --- ...configuring-playbook-prometheus-grafana.md | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 docs/configuring-playbook-prometheus-grafana.md diff --git a/docs/configuring-playbook-prometheus-grafana.md b/docs/configuring-playbook-prometheus-grafana.md new file mode 100644 index 00000000..5ad1449b --- /dev/null +++ b/docs/configuring-playbook-prometheus-grafana.md @@ -0,0 +1,36 @@ +# Enabling metrics and graphs for your Matrix server (optional) + +It can be useful to have some (visual) insight in the performance of your homeserver. + +You can enable this with the following settings in your configuration file (`inventory/host_vars/matrix./vars.yml`): + +```yaml +matrix_prometheus_enabled: true + +matrix_synapse_metrics_enabled: true +matrix_prometheus_node_exporter_enabled: true + +matrix_grafana_enabled: true +matrix_grafana_anonymous_access: true +matrix_grafana_default_admin_user: yourname +matrix_grafana_default_admin_password: securelongpassword +``` + +## What does it do? + +Name | Description +-----|---------- +`matrix_prometheus_enabled`|Prometheus is a time series database. It holds all the data we're going to talk about. +`matrix_synapse_metrics_enabled`|Enables metrics specific to Synapse +`matrix_prometheus_node_exporter_enabled`|Node Exporter is an addon of sorts to Prometheus that collects generic system information such as CPU, memory, filesystem, and even system temperatures +`matrix_grafana_enabled`|Grafana is the visual component. It shows the dashboards with the graphs that we're interested in +`matrix_grafana_anonymous_access`|By default you need to login to see graphs. If you want to publicly share your graphs (e.g. when asking for help in [`#synapse:matrix.org`](https://matrix.to/#/#synapse:matrix.org?via=matrix.org&via=privacytools.io&via=mozilla.org)) you'll want to enable this option. +`matrix_grafana_default_admin_user`
`matrix_grafana_default_admin_password`|By default Grafana creates a user with `admin` as the username and password. If you feel this is insecure and you want to change it beforehand, you can do that here + +## More inforation + +- [Understanding Synapse Performance Issues Through Grafana Graphs](https://github.com/matrix-org/synapse/wiki/Understanding-Synapse-Performance-Issues-Through-Grafana-Graphs) at the Synapse Github Wiki +- [The Prometheus scraping rules](https://github.com/matrix-org/synapse/tree/master/contrib/prometheus) (we use v2) +- [The Synapse Grafana dashboard](https://github.com/matrix-org/synapse/tree/master/contrib/grafana) +- [The Node Exporter dashboard](https://github.com/rfrail3/grafana-dashboards) (for generic non-synapse performance graphs) + From 76d7e84be533884ac8148ad3084f1e89cb3e9550 Mon Sep 17 00:00:00 2001 From: Peetz0r Date: Fri, 29 Jan 2021 12:23:59 +0100 Subject: [PATCH 06/24] Make prometheus-node-exporter a bit more capable By running it in a more privileged container with access to the host network stack and such --- .../systemd/matrix-prometheus-node-exporter.service.j2 | 7 +++++-- roles/matrix-prometheus/templates/prometheus.yml.j2 | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/roles/matrix-prometheus-node-exporter/templates/systemd/matrix-prometheus-node-exporter.service.j2 b/roles/matrix-prometheus-node-exporter/templates/systemd/matrix-prometheus-node-exporter.service.j2 index 622947d0..b7f410f1 100644 --- a/roles/matrix-prometheus-node-exporter/templates/systemd/matrix-prometheus-node-exporter.service.j2 +++ b/roles/matrix-prometheus-node-exporter/templates/systemd/matrix-prometheus-node-exporter.service.j2 @@ -21,14 +21,17 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-prometheus-nod --log-driver=none \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ --cap-drop=ALL \ - --network={{ matrix_docker_network }} \ {% if matrix_prometheus_node_exporter_container_http_host_bind_port %} -p {{ matrix_prometheus_node_exporter_container_http_host_bind_port }}:9100 \ {% endif %} {% for arg in matrix_prometheus_node_exporter_container_extra_arguments %} {{ arg }} \ {% endfor %} - {{ matrix_prometheus_node_exporter_docker_image }} + --net="host" \ + --pid="host" \ + -v "/:/host:ro,rslave" \ + {{ matrix_prometheus_node_exporter_docker_image }} \ + --path.rootfs=/host ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus-node-exporter 2>/dev/null' ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus-node-exporter 2>/dev/null' diff --git a/roles/matrix-prometheus/templates/prometheus.yml.j2 b/roles/matrix-prometheus/templates/prometheus.yml.j2 index 317dcd16..6e91ace2 100644 --- a/roles/matrix-prometheus/templates/prometheus.yml.j2 +++ b/roles/matrix-prometheus/templates/prometheus.yml.j2 @@ -36,5 +36,5 @@ scrape_configs: {% if matrix_prometheus_node_exporter_enabled %} - job_name: node static_configs: - - targets: ['matrix-prometheus-node-exporter:9100'] + - targets: ['172.18.0.1:9100'] {% endif %} From 1079272563ea95b132bc0fda55358a04c51abbd1 Mon Sep 17 00:00:00 2001 From: Peetz0r Date: Sat, 30 Jan 2021 06:11:44 +0100 Subject: [PATCH 07/24] Mention stats subdomain in docs --- docs/configuring-dns.md | 3 +++ docs/configuring-playbook-prometheus-grafana.md | 2 ++ 2 files changed, 5 insertions(+) diff --git a/docs/configuring-dns.md b/docs/configuring-dns.md index cef4cd50..84e2cd0b 100644 --- a/docs/configuring-dns.md +++ b/docs/configuring-dns.md @@ -24,6 +24,7 @@ If you decide to go with the alternative method ([Server Delegation via a DNS SR | CNAME | `element` | - | - | - | `matrix.` | | CNAME | `dimension` (*) | - | - | - | `matrix.` | | CNAME | `jitsi` (*) | - | - | - | `matrix.` | +| CNAME | `stats` (*) | - | - | - | `matrix.` | | SRV | `_matrix-identity._tcp` | 10 | 0 | 443 | `matrix.` | @@ -42,6 +43,8 @@ The `dimension.` subdomain may be necessary, because this playbook The `jitsi.` subdomain may be necessary, because this playbook could install the [Jitsi video-conferencing platform](https://jitsi.org/) for you. Jitsi installation is disabled by default, because it may be heavy and is not a core required component. To learn how to install it, see our [Jitsi](configuring-playbook-jitsi.md) guide. If you do not wish to set up Jitsi, feel free to skip the `jitsi.` DNS record. +The `stats.` subdomain may be necessary, because this playbook could install [Grafana](https://grafana.com/) and setup performance metrics for you. Grafana installation is disabled by default, it is not a core required component. To learn how to install it, see our [metrics and graphs guide](configuring-playbook-prometheus-grafana.md). If you do not wish to set up Grafana, feel free to skip the `stats.` DNS record. It is possible to install Prometheus without installing Grafana, this would also not require the `stats.` subdomain. + ## `_matrix-identity._tcp` SRV record setup diff --git a/docs/configuring-playbook-prometheus-grafana.md b/docs/configuring-playbook-prometheus-grafana.md index 5ad1449b..b714dc2c 100644 --- a/docs/configuring-playbook-prometheus-grafana.md +++ b/docs/configuring-playbook-prometheus-grafana.md @@ -16,6 +16,8 @@ matrix_grafana_default_admin_user: yourname matrix_grafana_default_admin_password: securelongpassword ``` +The dashboards will by default be available on the `stats.` subdomain, proxied via Nginx. + ## What does it do? Name | Description From 8aafb44cb86c19d2bfaa8cea296edc954c1428f0 Mon Sep 17 00:00:00 2001 From: Peetz0r Date: Sat, 30 Jan 2021 06:38:15 +0100 Subject: [PATCH 08/24] Mention new images in docks --- docs/container-images.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/container-images.md b/docs/container-images.md index aee24b04..28fce950 100644 --- a/docs/container-images.md +++ b/docs/container-images.md @@ -85,3 +85,9 @@ These services are not part of our default installation, but can be enabled by [ - [anoa/matrix-reminder-bot](https://hub.docker.com/r/anoa/matrix-reminder-bot) - the [matrix-reminder-bot](https://github.com/anoadragon453/matrix-reminder-bot) bot for one-off & recurring reminders and alarms (optional) - [awesometechnologies/synapse-admin](https://hub.docker.com/r/awesometechnologies/synapse-admin) - the [synapse-admin](https://github.com/Awesome-Technologies/synapse-admin) web UI tool for administrating users and rooms on your Matrix server (optional) + +- [prom/prometheus](https://hub.docker.com/r/prom/prometheus/) - [Prometheus](https://github.com/prometheus/prometheus/) is a systems and service monitoring system + +- [prom/node-exporter](https://hub.docker.com/r/prom/node-exporter/) - [Prometheus Node Exporter](https://github.com/prometheus/node_exporter/) is an addon for Prometheus that gathers standard system metrics + +- [grafana/grafana](https://hub.docker.com/r/grafana/grafana/) - [Grafana](https://github.com/grafana/grafana/) is a graphing tool that works well with the above two images. Our playbook also adds two dashboards for [Synapse](https://github.com/matrix-org/synapse/tree/master/contrib/grafana) and [Node Exporter](https://github.com/rfrail3/grafana-dashboards) From da82d670af40140923fee9703d3c5487df1b6805 Mon Sep 17 00:00:00 2001 From: Peetz0r Date: Sat, 30 Jan 2021 07:43:26 +0100 Subject: [PATCH 09/24] Document security and privacy considerations for stats. --- docs/configuring-playbook-prometheus-grafana.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/configuring-playbook-prometheus-grafana.md b/docs/configuring-playbook-prometheus-grafana.md index b714dc2c..9e2c5dd4 100644 --- a/docs/configuring-playbook-prometheus-grafana.md +++ b/docs/configuring-playbook-prometheus-grafana.md @@ -29,6 +29,12 @@ Name | Description `matrix_grafana_anonymous_access`|By default you need to login to see graphs. If you want to publicly share your graphs (e.g. when asking for help in [`#synapse:matrix.org`](https://matrix.to/#/#synapse:matrix.org?via=matrix.org&via=privacytools.io&via=mozilla.org)) you'll want to enable this option. `matrix_grafana_default_admin_user`
`matrix_grafana_default_admin_password`|By default Grafana creates a user with `admin` as the username and password. If you feel this is insecure and you want to change it beforehand, you can do that here +## Security and privacy + +Metrics and resulting graphs can contain a lot if information. This includes system specs but also usage patterns. This applies especially to small personal/family scale homeservers. Someone might be able to figure out when you wake up and go to sleep by looking at the graphs over time. Think about this before enabling anonymous access. And you should really not forget to change your Grafana password. + +Most of our docker containers run with limited system access, but the `prometheus-node-exporter` has access to the host network stack and (readonly) root filesystem. This is required to report on them. If you don't like that, you can set `matrix_prometheus_node_exporter_enabled: false` (which is actually the default). You will still get Synapse metrics with this container disabled. Both of the dashboards will always be enabled, so you can still look at historical data after disabling either source. + ## More inforation - [Understanding Synapse Performance Issues Through Grafana Graphs](https://github.com/matrix-org/synapse/wiki/Understanding-Synapse-Performance-Issues-Through-Grafana-Graphs) at the Synapse Github Wiki From 144a5e61983e77045247c5b7aaade0aba43be84d Mon Sep 17 00:00:00 2001 From: Peetz0r Date: Sun, 31 Jan 2021 02:09:12 +0100 Subject: [PATCH 10/24] Register docker network info and use it for prometheus-node-exporter Using the hardcoded IP did break while I was messing with IPv6 stuff on the other branch --- roles/matrix-base/tasks/setup_matrix_base.yml | 1 + roles/matrix-prometheus/templates/prometheus.yml.j2 | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index 0fad2b3d..b74b0316 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -23,6 +23,7 @@ docker_network: name: "{{ matrix_docker_network }}" driver: bridge + register: matrix_docker_network_info - name: Ensure matrix-remove-all script created template: diff --git a/roles/matrix-prometheus/templates/prometheus.yml.j2 b/roles/matrix-prometheus/templates/prometheus.yml.j2 index 6e91ace2..08e6fcf1 100644 --- a/roles/matrix-prometheus/templates/prometheus.yml.j2 +++ b/roles/matrix-prometheus/templates/prometheus.yml.j2 @@ -36,5 +36,5 @@ scrape_configs: {% if matrix_prometheus_node_exporter_enabled %} - job_name: node static_configs: - - targets: ['172.18.0.1:9100'] + - targets: ['{{ matrix_docker_network_info.network.IPAM.Config[0].Gateway }}:9100'] {% endif %} From 3a77261dc60ef00f51f598ef38d00895e5e931e0 Mon Sep 17 00:00:00 2001 From: Peetz0r Date: Wed, 10 Feb 2021 23:11:02 +0100 Subject: [PATCH 11/24] Update Grafana 7.3.7 => 7.4.0 --- roles/matrix-grafana/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/matrix-grafana/defaults/main.yml b/roles/matrix-grafana/defaults/main.yml index 2257d794..00ed947e 100644 --- a/roles/matrix-grafana/defaults/main.yml +++ b/roles/matrix-grafana/defaults/main.yml @@ -3,7 +3,7 @@ matrix_grafana_enabled: false -matrix_grafana_docker_image: "docker.io/grafana/grafana:7.3.7" +matrix_grafana_docker_image: "docker.io/grafana/grafana:7.4.0" matrix_grafana_docker_image_force_pull: "{{ matrix_grafana_docker_image.endswith(':latest') }}" # Not conditional, because when someone disables metrics @@ -12,7 +12,7 @@ matrix_grafana_docker_image_force_pull: "{{ matrix_grafana_docker_image.endswith matrix_grafana_dashboard_download_urls: - "https://raw.githubusercontent.com/matrix-org/synapse/master/contrib/grafana/synapse.json" - "https://raw.githubusercontent.com/rfrail3/grafana-dashboards/master/prometheus/node-exporter-full.json" - + matrix_grafana_base_path: "{{ matrix_base_data_path }}/grafana" matrix_grafana_config_path: "{{ matrix_grafana_base_path }}/config" matrix_grafana_data_path: "{{ matrix_grafana_base_path }}/data" From fde222a0417a66220784a4c59778f82850719321 Mon Sep 17 00:00:00 2001 From: Peetz0r Date: Wed, 10 Feb 2021 23:11:17 +0100 Subject: [PATCH 12/24] Update Prometheus Node Exporter 1.0.1 => 1.1.0 --- roles/matrix-prometheus-node-exporter/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-prometheus-node-exporter/defaults/main.yml b/roles/matrix-prometheus-node-exporter/defaults/main.yml index a5664b83..29dce364 100644 --- a/roles/matrix-prometheus-node-exporter/defaults/main.yml +++ b/roles/matrix-prometheus-node-exporter/defaults/main.yml @@ -3,7 +3,7 @@ matrix_prometheus_node_exporter_enabled: false -matrix_prometheus_node_exporter_docker_image: "docker.io/prom/node-exporter:v1.0.1" +matrix_prometheus_node_exporter_docker_image: "docker.io/prom/node-exporter:v1.1.0" matrix_prometheus_node_exporter_docker_image_force_pull: "{{ matrix_prometheus_node_exporter_docker_image.endswith(':latest') }}" # A list of extra arguments to pass to the container From 9531d137869af24ea14f8583997c599f576bd7fd Mon Sep 17 00:00:00 2001 From: Peetz0r Date: Sat, 30 Jan 2021 08:05:52 +0100 Subject: [PATCH 13/24] Split DNS table in default and optional parts --- docs/configuring-dns.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/configuring-dns.md b/docs/configuring-dns.md index 84e2cd0b..c25b079e 100644 --- a/docs/configuring-dns.md +++ b/docs/configuring-dns.md @@ -15,22 +15,25 @@ As we discuss in [Server Delegation](howto-server-delegation.md), there are 2 di This playbook mostly discusses the well-known file method, because it's easier to manage with regard to certificates. If you decide to go with the alternative method ([Server Delegation via a DNS SRV record (advanced)](howto-server-delegation.md#server-delegation-via-a-dns-srv-record-advanced)), please be aware that the general flow that this playbook guides you through may not match what you need to do. - -## General outline of DNS settings you need to do +## Required DNS settings for services enabled by default | Type | Host | Priority | Weight | Port | Target | | ----- | ---------------------------- | -------- | ------ | ---- | ---------------------- | | A | `matrix` | - | - | - | `matrix-server-IP` | | CNAME | `element` | - | - | - | `matrix.` | -| CNAME | `dimension` (*) | - | - | - | `matrix.` | -| CNAME | `jitsi` (*) | - | - | - | `matrix.` | -| CNAME | `stats` (*) | - | - | - | `matrix.` | | SRV | `_matrix-identity._tcp` | 10 | 0 | 443 | `matrix.` | +Be mindful as to how long it will take for the DNS records to propagate. + +If you are using Cloudflare DNS, make sure to disable the proxy and set all records to `DNS only`. Otherwise, fetching certificates will fail. -DNS records marked with `(*)` above are optional. They refer to services that will not be installed by default (see the section below). If you won't be installing these services, feel free to skip creating these DNS records. Also be mindful as to how long it will take for the DNS records to propagate. +## Required DNS settings for optional services -> If you are using Cloudflare DNS, make sure to disable the proxy and set all records to `DNS only`. Otherwise, fetching certificates will fail. +| Type | Host | Priority | Weight | Port | Target | +| ----- | ---------------------------- | -------- | ------ | ---- | ---------------------- | +| CNAME | `dimension` (*) | - | - | - | `matrix.` | +| CNAME | `jitsi` (*) | - | - | - | `matrix.` | +| CNAME | `stats` (*) | - | - | - | `matrix.` | ## Subdomains setup From f0cd29462845c70ba8b1e25bb8c5a927b4a7a207 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 31 Jan 2021 11:41:22 +0200 Subject: [PATCH 14/24] Fix matrix-prometheus-node-exporter failure to start The quotes around "host" for both `--pid` and `--net` were causing trouble for me: > docker: --pid: invalid PID mode. and: > docker: Error response from daemon: network "host" not found. I've also changed the `-v` call to `--mount` for consistency with the rest of the playbook. --- .../systemd/matrix-prometheus-node-exporter.service.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/matrix-prometheus-node-exporter/templates/systemd/matrix-prometheus-node-exporter.service.j2 b/roles/matrix-prometheus-node-exporter/templates/systemd/matrix-prometheus-node-exporter.service.j2 index b7f410f1..58349444 100644 --- a/roles/matrix-prometheus-node-exporter/templates/systemd/matrix-prometheus-node-exporter.service.j2 +++ b/roles/matrix-prometheus-node-exporter/templates/systemd/matrix-prometheus-node-exporter.service.j2 @@ -27,9 +27,9 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-prometheus-nod {% for arg in matrix_prometheus_node_exporter_container_extra_arguments %} {{ arg }} \ {% endfor %} - --net="host" \ - --pid="host" \ - -v "/:/host:ro,rslave" \ + --net=host \ + --pid=host \ + --mount type=bind,src=/,dst=/host,ro,bind-propagation=rslave \ {{ matrix_prometheus_node_exporter_docker_image }} \ --path.rootfs=/host From 3ce97123888473cd29fab16a847196fd1538d724 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 31 Jan 2021 12:01:56 +0200 Subject: [PATCH 15/24] Fix Grafana dashboard/datasource label --- roles/matrix-grafana/templates/dashboards.yaml.j2 | 2 +- roles/matrix-grafana/templates/datasources.yaml.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/matrix-grafana/templates/dashboards.yaml.j2 b/roles/matrix-grafana/templates/dashboards.yaml.j2 index b6662e59..aae42ba2 100644 --- a/roles/matrix-grafana/templates/dashboards.yaml.j2 +++ b/roles/matrix-grafana/templates/dashboards.yaml.j2 @@ -1,7 +1,7 @@ apiVersion: 1 providers: - - name: {{ matrix_domain }} - Dashboards + - name: {{ matrix_server_fqn_matrix }} - Dashboards folder: '' # The folder where to place the dashboards type: file allowUiUpdates: true diff --git a/roles/matrix-grafana/templates/datasources.yaml.j2 b/roles/matrix-grafana/templates/datasources.yaml.j2 index ffa6046b..6ccbe374 100644 --- a/roles/matrix-grafana/templates/datasources.yaml.j2 +++ b/roles/matrix-grafana/templates/datasources.yaml.j2 @@ -1,7 +1,7 @@ apiVersion: 1 datasources: - - name: {{ matrix_domain }} - Prometheus + - name: {{ matrix_server_fqn_matrix }} - Prometheus type: prometheus # Access mode - proxy (server in the UI) or direct (browser in the UI). access: proxy From 1d7d99c5cd94c961d7fc78c6fb9e6b67ce7ec99c Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 31 Jan 2021 12:02:14 +0200 Subject: [PATCH 16/24] Improve metrics docs page a bit We mainly switch the anonymous metrics viewing variable to false, along with other wording changes. --- docs/configuring-playbook-prometheus-grafana.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/configuring-playbook-prometheus-grafana.md b/docs/configuring-playbook-prometheus-grafana.md index 9e2c5dd4..0c759692 100644 --- a/docs/configuring-playbook-prometheus-grafana.md +++ b/docs/configuring-playbook-prometheus-grafana.md @@ -1,6 +1,6 @@ # Enabling metrics and graphs for your Matrix server (optional) -It can be useful to have some (visual) insight in the performance of your homeserver. +It can be useful to have some (visual) insight into the performance of your homeserver. You can enable this with the following settings in your configuration file (`inventory/host_vars/matrix./vars.yml`): @@ -11,8 +11,13 @@ matrix_synapse_metrics_enabled: true matrix_prometheus_node_exporter_enabled: true matrix_grafana_enabled: true -matrix_grafana_anonymous_access: true + +matrix_grafana_anonymous_access: false + matrix_grafana_default_admin_user: yourname + +# Passwords containing special characters may be troublesome. +# Changing the password subsequently won't work. matrix_grafana_default_admin_password: securelongpassword ``` @@ -26,7 +31,7 @@ Name | Description `matrix_synapse_metrics_enabled`|Enables metrics specific to Synapse `matrix_prometheus_node_exporter_enabled`|Node Exporter is an addon of sorts to Prometheus that collects generic system information such as CPU, memory, filesystem, and even system temperatures `matrix_grafana_enabled`|Grafana is the visual component. It shows the dashboards with the graphs that we're interested in -`matrix_grafana_anonymous_access`|By default you need to login to see graphs. If you want to publicly share your graphs (e.g. when asking for help in [`#synapse:matrix.org`](https://matrix.to/#/#synapse:matrix.org?via=matrix.org&via=privacytools.io&via=mozilla.org)) you'll want to enable this option. +`matrix_grafana_anonymous_access`|By default you need to log in to see graphs. If you want to publicly share your graphs (e.g. when asking for help in [`#synapse:matrix.org`](https://matrix.to/#/#synapse:matrix.org?via=matrix.org&via=privacytools.io&via=mozilla.org)) you'll want to enable this option. `matrix_grafana_default_admin_user`
`matrix_grafana_default_admin_password`|By default Grafana creates a user with `admin` as the username and password. If you feel this is insecure and you want to change it beforehand, you can do that here ## Security and privacy From df3dd1c82459b61b4ed549797580de4c37ebad4b Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 31 Jan 2021 17:34:32 +0200 Subject: [PATCH 17/24] Use --read-only FS for metrics-related containers It seems like it doesn't cause any issues for any of these services. --- roles/matrix-grafana/templates/systemd/matrix-grafana.service.j2 | 1 + .../templates/systemd/matrix-prometheus-node-exporter.service.j2 | 1 + .../templates/systemd/matrix-prometheus.service.j2 | 1 + 3 files changed, 3 insertions(+) diff --git a/roles/matrix-grafana/templates/systemd/matrix-grafana.service.j2 b/roles/matrix-grafana/templates/systemd/matrix-grafana.service.j2 index f2ab6642..a4f81e35 100644 --- a/roles/matrix-grafana/templates/systemd/matrix-grafana.service.j2 +++ b/roles/matrix-grafana/templates/systemd/matrix-grafana.service.j2 @@ -21,6 +21,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-grafana \ --log-driver=none \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ --cap-drop=ALL \ + --read-only \ --network={{ matrix_docker_network }} \ {% if matrix_grafana_container_http_host_bind_port %} -p {{ matrix_grafana_container_http_host_bind_port }}:3000 \ diff --git a/roles/matrix-prometheus-node-exporter/templates/systemd/matrix-prometheus-node-exporter.service.j2 b/roles/matrix-prometheus-node-exporter/templates/systemd/matrix-prometheus-node-exporter.service.j2 index 58349444..93638c19 100644 --- a/roles/matrix-prometheus-node-exporter/templates/systemd/matrix-prometheus-node-exporter.service.j2 +++ b/roles/matrix-prometheus-node-exporter/templates/systemd/matrix-prometheus-node-exporter.service.j2 @@ -21,6 +21,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-prometheus-nod --log-driver=none \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ --cap-drop=ALL \ + --read-only \ {% if matrix_prometheus_node_exporter_container_http_host_bind_port %} -p {{ matrix_prometheus_node_exporter_container_http_host_bind_port }}:9100 \ {% endif %} diff --git a/roles/matrix-prometheus/templates/systemd/matrix-prometheus.service.j2 b/roles/matrix-prometheus/templates/systemd/matrix-prometheus.service.j2 index dd3ac72c..ad75d664 100644 --- a/roles/matrix-prometheus/templates/systemd/matrix-prometheus.service.j2 +++ b/roles/matrix-prometheus/templates/systemd/matrix-prometheus.service.j2 @@ -21,6 +21,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-prometheus \ --log-driver=none \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ --cap-drop=ALL \ + --read-only \ --network={{ matrix_docker_network }} \ {% if matrix_prometheus_container_http_host_bind_port %} -p {{ matrix_prometheus_container_http_host_bind_port }}:9090 \ From 85a260daaf5d04795f2be4a8de09fac26be57ecd Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 31 Jan 2021 18:17:57 +0200 Subject: [PATCH 18/24] Make --tags=setup-prometheus not break, relying on matrix-base facts --- roles/matrix-base/tasks/setup_matrix_base.yml | 1 - roles/matrix-prometheus/defaults/main.yml | 3 +++ roles/matrix-prometheus/tasks/setup.yml | 17 ++++++++++++++++- .../templates/prometheus.yml.j2 | 2 +- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index b74b0316..0fad2b3d 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -23,7 +23,6 @@ docker_network: name: "{{ matrix_docker_network }}" driver: bridge - register: matrix_docker_network_info - name: Ensure matrix-remove-all script created template: diff --git a/roles/matrix-prometheus/defaults/main.yml b/roles/matrix-prometheus/defaults/main.yml index 10424424..a0e79acc 100644 --- a/roles/matrix-prometheus/defaults/main.yml +++ b/roles/matrix-prometheus/defaults/main.yml @@ -26,6 +26,9 @@ matrix_prometheus_systemd_wanted_services_list: [] # Takes an ":" or "" value (e.g. "127.0.0.1:9090"), or empty string to not expose. matrix_prometheus_container_http_host_bind_port: '' +# Target addresses for the "node" scraper configuration. +# Unless you define this as a non-empty list, it gets populated at runtime with the IP address of `matrix-prometheus-node-exporter` and port 9100. +matrix_prometheus_endpoint_node_targets: [] # Default prometheus configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. diff --git a/roles/matrix-prometheus/tasks/setup.yml b/roles/matrix-prometheus/tasks/setup.yml index 1746b961..c9a207ec 100644 --- a/roles/matrix-prometheus/tasks/setup.yml +++ b/roles/matrix-prometheus/tasks/setup.yml @@ -24,7 +24,22 @@ - "{{ matrix_prometheus_config_path }}" - "{{ matrix_prometheus_data_path }}" when: matrix_prometheus_enabled|bool - + +- block: + # Well, this actually creates the network if it doesn't exist, but.. + # The network should have been created by `matrix-base` already. + # We don't rely on that other call and its result, because it runs + # on `--tags=setup-all`, but will get skipped during `--tags=setup-prometheus`. + - name: Fetch Matrix Docker network details + docker_network: + name: "{{ matrix_docker_network }}" + driver: bridge + register: matrix_docker_network_info + + - set_fact: + matrix_prometheus_endpoint_node_targets: ["{{ matrix_docker_network_info.network.IPAM.Config[0].Gateway }}:9100"] + when: "matrix_prometheus_enabled|bool and matrix_prometheus_node_exporter_enabled|bool and matrix_prometheus_endpoint_node_targets|length == 0" + - name: Ensure prometheus.yml installed copy: content: "{{ matrix_prometheus_configuration|to_nice_yaml }}" diff --git a/roles/matrix-prometheus/templates/prometheus.yml.j2 b/roles/matrix-prometheus/templates/prometheus.yml.j2 index 08e6fcf1..4fdf9905 100644 --- a/roles/matrix-prometheus/templates/prometheus.yml.j2 +++ b/roles/matrix-prometheus/templates/prometheus.yml.j2 @@ -36,5 +36,5 @@ scrape_configs: {% if matrix_prometheus_node_exporter_enabled %} - job_name: node static_configs: - - targets: ['{{ matrix_docker_network_info.network.IPAM.Config[0].Gateway }}:9100'] + - targets: {{ matrix_prometheus_endpoint_node_targets|to_json }} {% endif %} From 18e31526a885cd9590b8d639e75da507db29fa35 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 31 Jan 2021 18:26:08 +0200 Subject: [PATCH 19/24] Rename some variables --- group_vars/matrix_servers | 2 ++ roles/matrix-prometheus/defaults/main.yml | 6 +++++- roles/matrix-prometheus/tasks/setup.yml | 4 ++-- roles/matrix-prometheus/templates/prometheus.yml.j2 | 4 ++-- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index b314dc99..ad700597 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1405,6 +1405,8 @@ matrix_prometheus_enabled: false # Prometheus' HTTP port to the local host. matrix_prometheus_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else '127.0.0.1:9090' }}" +matrix_prometheus_scraper_node_enabled: "{{ matrix_prometheus_node_exporter_enabled }}" + ###################################################################### # # /matrix-prometheus diff --git a/roles/matrix-prometheus/defaults/main.yml b/roles/matrix-prometheus/defaults/main.yml index a0e79acc..c07c3801 100644 --- a/roles/matrix-prometheus/defaults/main.yml +++ b/roles/matrix-prometheus/defaults/main.yml @@ -26,9 +26,13 @@ matrix_prometheus_systemd_wanted_services_list: [] # Takes an ":" or "" value (e.g. "127.0.0.1:9090"), or empty string to not expose. matrix_prometheus_container_http_host_bind_port: '' +# Tells whether the "node" scraper configuration is enabled. +# This configuration aims to scrape the current node (this server). +matrix_prometheus_scraper_node_enabled: false + # Target addresses for the "node" scraper configuration. # Unless you define this as a non-empty list, it gets populated at runtime with the IP address of `matrix-prometheus-node-exporter` and port 9100. -matrix_prometheus_endpoint_node_targets: [] +matrix_prometheus_scraper_node_targets: [] # Default prometheus configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. diff --git a/roles/matrix-prometheus/tasks/setup.yml b/roles/matrix-prometheus/tasks/setup.yml index c9a207ec..7b98b76a 100644 --- a/roles/matrix-prometheus/tasks/setup.yml +++ b/roles/matrix-prometheus/tasks/setup.yml @@ -37,8 +37,8 @@ register: matrix_docker_network_info - set_fact: - matrix_prometheus_endpoint_node_targets: ["{{ matrix_docker_network_info.network.IPAM.Config[0].Gateway }}:9100"] - when: "matrix_prometheus_enabled|bool and matrix_prometheus_node_exporter_enabled|bool and matrix_prometheus_endpoint_node_targets|length == 0" + matrix_prometheus_scraper_node_targets: ["{{ matrix_docker_network_info.network.IPAM.Config[0].Gateway }}:9100"] + when: "matrix_prometheus_enabled|bool and matrix_prometheus_scraper_node_enabled|bool and matrix_prometheus_scraper_node_targets|length == 0" - name: Ensure prometheus.yml installed copy: diff --git a/roles/matrix-prometheus/templates/prometheus.yml.j2 b/roles/matrix-prometheus/templates/prometheus.yml.j2 index 4fdf9905..4fe8394d 100644 --- a/roles/matrix-prometheus/templates/prometheus.yml.j2 +++ b/roles/matrix-prometheus/templates/prometheus.yml.j2 @@ -33,8 +33,8 @@ scrape_configs: - targets: ['matrix-synapse:{{ matrix_synapse_metrics_port }}'] {% endif %} - {% if matrix_prometheus_node_exporter_enabled %} + {% if matrix_prometheus_scraper_node_enabled %} - job_name: node static_configs: - - targets: {{ matrix_prometheus_endpoint_node_targets|to_json }} + - targets: {{ matrix_prometheus_scraper_node_targets|to_json }} {% endif %} From 6842102e008a0682bc676793556f32c2593ff723 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 31 Jan 2021 18:30:02 +0200 Subject: [PATCH 20/24] Split install/uninstall tasks in matrix-prometheus --- roles/matrix-prometheus/tasks/main.yml | 9 +++- .../tasks/{setup.yml => setup_install.yml} | 48 +------------------ .../tasks/setup_uninstall.yml | 31 ++++++++++++ 3 files changed, 41 insertions(+), 47 deletions(-) rename roles/matrix-prometheus/tasks/{setup.yml => setup_install.yml} (59%) create mode 100644 roles/matrix-prometheus/tasks/setup_uninstall.yml diff --git a/roles/matrix-prometheus/tasks/main.yml b/roles/matrix-prometheus/tasks/main.yml index 2290048f..edb01988 100644 --- a/roles/matrix-prometheus/tasks/main.yml +++ b/roles/matrix-prometheus/tasks/main.yml @@ -8,7 +8,14 @@ - setup-all - setup-prometheus -- import_tasks: "{{ role_path }}/tasks/setup.yml" +- import_tasks: "{{ role_path }}/tasks/setup_install.yml" + when: "run_setup|bool and matrix_prometheus_enabled|bool" + tags: + - setup-all + - setup-prometheus + +- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml" + when: "run_setup|bool and not matrix_prometheus_enabled|bool" tags: - setup-all - setup-prometheus diff --git a/roles/matrix-prometheus/tasks/setup.yml b/roles/matrix-prometheus/tasks/setup_install.yml similarity index 59% rename from roles/matrix-prometheus/tasks/setup.yml rename to roles/matrix-prometheus/tasks/setup_install.yml index 7b98b76a..b69e349d 100644 --- a/roles/matrix-prometheus/tasks/setup.yml +++ b/roles/matrix-prometheus/tasks/setup_install.yml @@ -1,16 +1,11 @@ --- -# -# Tasks related to setting up matrix-prometheus -# - - name: Ensure matrix-prometheus image is pulled docker_image: name: "{{ matrix_prometheus_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_prometheus_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_prometheus_docker_image_force_pull }}" - when: "matrix_prometheus_enabled|bool" - name: Ensure Prometheus paths exists file: @@ -23,7 +18,6 @@ - "{{ matrix_prometheus_base_path }}" - "{{ matrix_prometheus_config_path }}" - "{{ matrix_prometheus_data_path }}" - when: matrix_prometheus_enabled|bool - block: # Well, this actually creates the network if it doesn't exist, but.. @@ -38,7 +32,7 @@ - set_fact: matrix_prometheus_scraper_node_targets: ["{{ matrix_docker_network_info.network.IPAM.Config[0].Gateway }}:9100"] - when: "matrix_prometheus_enabled|bool and matrix_prometheus_scraper_node_enabled|bool and matrix_prometheus_scraper_node_targets|length == 0" + when: "matrix_prometheus_scraper_node_enabled|bool and matrix_prometheus_scraper_node_targets|length == 0" - name: Ensure prometheus.yml installed copy: @@ -47,7 +41,6 @@ mode: 0644 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" - when: matrix_prometheus_enabled|bool - name: Download synapse-v2.rules get_url: @@ -57,8 +50,6 @@ mode: 0440 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" - when: matrix_prometheus_enabled|bool - - name: Ensure matrix-prometheus.service installed template: @@ -66,43 +57,8 @@ dest: "{{ matrix_systemd_path }}/matrix-prometheus.service" mode: 0644 register: matrix_prometheus_systemd_service_result - when: matrix_prometheus_enabled|bool - name: Ensure systemd reloaded after matrix-prometheus.service installation service: daemon_reload: yes - when: "matrix_prometheus_enabled|bool and matrix_prometheus_systemd_service_result.changed" - -# -# Tasks related to getting rid of matrix-prometheus (if it was previously enabled) -# - -- name: Check existence of matrix-prometheus service - stat: - path: "{{ matrix_systemd_path }}/matrix-prometheus.service" - register: matrix_prometheus_service_stat - -- name: Ensure matrix-prometheus is stopped - service: - name: matrix-prometheus - state: stopped - daemon_reload: yes - register: stopping_result - when: "not matrix_prometheus_enabled|bool and matrix_prometheus_service_stat.stat.exists" - -- name: Ensure matrix-prometheus.service doesn't exist - file: - path: "{{ matrix_systemd_path }}/matrix-prometheus.service" - state: absent - when: "not matrix_prometheus_enabled|bool and matrix_prometheus_service_stat.stat.exists" - -- name: Ensure systemd reloaded after matrix-prometheus.service removal - service: - daemon_reload: yes - when: "not matrix_prometheus_enabled|bool and matrix_prometheus_service_stat.stat.exists" - -- name: Ensure matrix-prometheus Docker image doesn't exist - docker_image: - name: "{{ matrix_prometheus_docker_image }}" - state: absent - when: "not matrix_prometheus_enabled|bool" + when: "matrix_prometheus_systemd_service_result.changed|bool" diff --git a/roles/matrix-prometheus/tasks/setup_uninstall.yml b/roles/matrix-prometheus/tasks/setup_uninstall.yml new file mode 100644 index 00000000..0a4a8cb6 --- /dev/null +++ b/roles/matrix-prometheus/tasks/setup_uninstall.yml @@ -0,0 +1,31 @@ +--- + +- name: Check existence of matrix-prometheus service + stat: + path: "{{ matrix_systemd_path }}/matrix-prometheus.service" + register: matrix_prometheus_service_stat + +- name: Ensure matrix-prometheus is stopped + service: + name: matrix-prometheus + state: stopped + daemon_reload: yes + register: stopping_result + when: "matrix_prometheus_service_stat.stat.exists|bool" + +- name: Ensure matrix-prometheus.service doesn't exist + file: + path: "{{ matrix_systemd_path }}/matrix-prometheus.service" + state: absent + when: "matrix_prometheus_service_stat.stat.exists|bool" + +- name: Ensure systemd reloaded after matrix-prometheus.service removal + service: + daemon_reload: yes + when: "matrix_prometheus_service_stat.stat.exists|bool" + +- name: Ensure matrix-prometheus Docker image doesn't exist + docker_image: + name: "{{ matrix_prometheus_docker_image }}" + state: absent + when: "not matrix_prometheus_enabled|bool" From c8ab200cb1ded35d57b45514902ed807821e4b89 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 31 Jan 2021 19:23:12 +0200 Subject: [PATCH 21/24] Break dependency between matrix-prometheus and (matrix-prometheus-node-exporter, matrix-synapse) --- group_vars/matrix_servers | 4 ++++ roles/matrix-prometheus/defaults/main.yml | 12 ++++++++++-- .../matrix-prometheus/tasks/setup_install.yml | 18 ++++++++++-------- .../tasks/validate_config.yml | 4 ++-- .../templates/prometheus.yml.j2 | 6 +++--- 5 files changed, 29 insertions(+), 15 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index ad700597..a8bddf6e 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1405,6 +1405,10 @@ matrix_prometheus_enabled: false # Prometheus' HTTP port to the local host. matrix_prometheus_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else '127.0.0.1:9090' }}" +matrix_prometheus_scraper_synapse_enabled: "{{ matrix_synapse_enabled and matrix_synapse_metrics_enabled }}" +matrix_prometheus_scraper_synapse_targets: ['matrix-synapse:{{ matrix_synapse_metrics_port }}'] +matrix_prometheus_scraper_synapse_rules_synapse_tag: "{{ matrix_synapse_docker_image_tag }}" + matrix_prometheus_scraper_node_enabled: "{{ matrix_prometheus_node_exporter_enabled }}" ###################################################################### diff --git a/roles/matrix-prometheus/defaults/main.yml b/roles/matrix-prometheus/defaults/main.yml index c07c3801..56018ba6 100644 --- a/roles/matrix-prometheus/defaults/main.yml +++ b/roles/matrix-prometheus/defaults/main.yml @@ -6,8 +6,6 @@ matrix_prometheus_enabled: false matrix_prometheus_docker_image: "docker.io/prom/prometheus:v2.24.1" matrix_prometheus_docker_image_force_pull: "{{ matrix_prometheus_docker_image.endswith(':latest') }}" -matrix_synapse_prometheus_rules_download_url: "https://raw.githubusercontent.com/matrix-org/synapse/{{ matrix_synapse_docker_image_tag }}/contrib/prometheus/synapse-v2.rules" - matrix_prometheus_base_path: "{{ matrix_base_data_path }}/prometheus" matrix_prometheus_config_path: "{{ matrix_prometheus_base_path }}/config" matrix_prometheus_data_path: "{{ matrix_prometheus_base_path }}/data" @@ -26,6 +24,16 @@ matrix_prometheus_systemd_wanted_services_list: [] # Takes an ":" or "" value (e.g. "127.0.0.1:9090"), or empty string to not expose. matrix_prometheus_container_http_host_bind_port: '' +# Tells whether the "synapse" scraper configuration is enabled. +matrix_prometheus_scraper_synapse_enabled: false + +# Tells whether to download and load a Synapse rules file +matrix_prometheus_scraper_synapse_rules_enabled: "{{ matrix_prometheus_scraper_synapse_enabled }}" +matrix_prometheus_scraper_synapse_rules_synapse_tag: "master" +matrix_prometheus_scraper_synapse_rules_download_url: "https://raw.githubusercontent.com/matrix-org/synapse/{{ matrix_prometheus_scraper_synapse_rules_synapse_tag }}/contrib/prometheus/synapse-v2.rules" + +matrix_prometheus_scraper_synapse_targets: [] + # Tells whether the "node" scraper configuration is enabled. # This configuration aims to scrape the current node (this server). matrix_prometheus_scraper_node_enabled: false diff --git a/roles/matrix-prometheus/tasks/setup_install.yml b/roles/matrix-prometheus/tasks/setup_install.yml index b69e349d..8aee5178 100644 --- a/roles/matrix-prometheus/tasks/setup_install.yml +++ b/roles/matrix-prometheus/tasks/setup_install.yml @@ -34,22 +34,24 @@ matrix_prometheus_scraper_node_targets: ["{{ matrix_docker_network_info.network.IPAM.Config[0].Gateway }}:9100"] when: "matrix_prometheus_scraper_node_enabled|bool and matrix_prometheus_scraper_node_targets|length == 0" -- name: Ensure prometheus.yml installed - copy: - content: "{{ matrix_prometheus_configuration|to_nice_yaml }}" - dest: "{{ matrix_prometheus_config_path }}/prometheus.yml" - mode: 0644 - owner: "{{ matrix_user_username }}" - group: "{{ matrix_user_groupname }}" - name: Download synapse-v2.rules get_url: - url: "{{ matrix_synapse_prometheus_rules_download_url }}" + url: "{{ matrix_prometheus_scraper_synapse_rules_download_url }}" dest: "{{ matrix_prometheus_config_path }}/synapse-v2.rules" force: true mode: 0440 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" + when: "matrix_prometheus_scraper_synapse_rules_enabled|bool" + +- name: Ensure prometheus.yml installed + copy: + content: "{{ matrix_prometheus_configuration|to_nice_yaml }}" + dest: "{{ matrix_prometheus_config_path }}/prometheus.yml" + mode: 0644 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" - name: Ensure matrix-prometheus.service installed template: diff --git a/roles/matrix-prometheus/tasks/validate_config.yml b/roles/matrix-prometheus/tasks/validate_config.yml index 713646ae..9fcfe12b 100644 --- a/roles/matrix-prometheus/tasks/validate_config.yml +++ b/roles/matrix-prometheus/tasks/validate_config.yml @@ -3,5 +3,5 @@ - name: Fail if Synapse metrics or Prometheus Node Exporter not enabled fail: msg: > - You need to enable `matrix_synapse_metrics_enabled` and/or `matrix_prometheus_node_exporter_enabled` for Prometheus grab metrics. - when: "not matrix_synapse_metrics_enabled and not matrix_prometheus_node_exporter_enabled" + You need to enable `matrix_prometheus_scraper_synapse_enabled` and/or `matrix_prometheus_scraper_node_enabled` for Prometheus grab metrics. + when: "not matrix_prometheus_scraper_synapse_enabled and not matrix_prometheus_scraper_node_enabled" diff --git a/roles/matrix-prometheus/templates/prometheus.yml.j2 b/roles/matrix-prometheus/templates/prometheus.yml.j2 index 4fe8394d..9502a08b 100644 --- a/roles/matrix-prometheus/templates/prometheus.yml.j2 +++ b/roles/matrix-prometheus/templates/prometheus.yml.j2 @@ -6,7 +6,7 @@ global: # Load rules once and periodically evaluate them according to the global 'evaluation_interval'. rule_files: - {% if matrix_synapse_metrics_enabled %} + {% if matrix_prometheus_scraper_synapse_rules_enabled %} - 'synapse-v2.rules' {% endif %} @@ -26,11 +26,11 @@ scrape_configs: static_configs: - targets: ['localhost:9090'] - {% if matrix_synapse_metrics_enabled %} + {% if matrix_prometheus_scraper_synapse_enabled %} - job_name: 'synapse' metrics_path: '/_synapse/metrics' static_configs: - - targets: ['matrix-synapse:{{ matrix_synapse_metrics_port }}'] + - targets: {{ matrix_prometheus_scraper_synapse_targets|to_json }} {% endif %} {% if matrix_prometheus_scraper_node_enabled %} From eb9aac0ac9682104955a3498951fd02d102bb370 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 31 Jan 2021 19:43:47 +0200 Subject: [PATCH 22/24] Minor docs updates --- docs/configuring-playbook-prometheus-grafana.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/configuring-playbook-prometheus-grafana.md b/docs/configuring-playbook-prometheus-grafana.md index 0c759692..2010b1b5 100644 --- a/docs/configuring-playbook-prometheus-grafana.md +++ b/docs/configuring-playbook-prometheus-grafana.md @@ -8,17 +8,19 @@ You can enable this with the following settings in your configuration file (`inv matrix_prometheus_enabled: true matrix_synapse_metrics_enabled: true + matrix_prometheus_node_exporter_enabled: true matrix_grafana_enabled: true matrix_grafana_anonymous_access: false -matrix_grafana_default_admin_user: yourname +# This has no relation to your Matrix user id. It can be any username you'd like. +matrix_grafana_default_admin_user: some_username_chosen_by_you # Passwords containing special characters may be troublesome. # Changing the password subsequently won't work. -matrix_grafana_default_admin_password: securelongpassword +matrix_grafana_default_admin_password: some_strong_password_chosen_by_you ``` The dashboards will by default be available on the `stats.` subdomain, proxied via Nginx. @@ -28,7 +30,7 @@ The dashboards will by default be available on the `stats.` subdoma Name | Description -----|---------- `matrix_prometheus_enabled`|Prometheus is a time series database. It holds all the data we're going to talk about. -`matrix_synapse_metrics_enabled`|Enables metrics specific to Synapse +`matrix_synapse_metrics_enabled`|Tell the Synapse server to expose metrics. This also cascades to other variables, which makes Prometheus collect said metrics `matrix_prometheus_node_exporter_enabled`|Node Exporter is an addon of sorts to Prometheus that collects generic system information such as CPU, memory, filesystem, and even system temperatures `matrix_grafana_enabled`|Grafana is the visual component. It shows the dashboards with the graphs that we're interested in `matrix_grafana_anonymous_access`|By default you need to log in to see graphs. If you want to publicly share your graphs (e.g. when asking for help in [`#synapse:matrix.org`](https://matrix.to/#/#synapse:matrix.org?via=matrix.org&via=privacytools.io&via=mozilla.org)) you'll want to enable this option. @@ -36,7 +38,7 @@ Name | Description ## Security and privacy -Metrics and resulting graphs can contain a lot if information. This includes system specs but also usage patterns. This applies especially to small personal/family scale homeservers. Someone might be able to figure out when you wake up and go to sleep by looking at the graphs over time. Think about this before enabling anonymous access. And you should really not forget to change your Grafana password. +Metrics and resulting graphs can contain a lot of information. This includes system specs but also usage patterns. This applies especially to small personal/family scale homeservers. Someone might be able to figure out when you wake up and go to sleep by looking at the graphs over time. Think about this before enabling anonymous access. And you should really not forget to change your Grafana password. Most of our docker containers run with limited system access, but the `prometheus-node-exporter` has access to the host network stack and (readonly) root filesystem. This is required to report on them. If you don't like that, you can set `matrix_prometheus_node_exporter_enabled: false` (which is actually the default). You will still get Synapse metrics with this container disabled. Both of the dashboards will always be enabled, so you can still look at historical data after disabling either source. From 2b47258c6cc382218cb659b3882bed247e304807 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 12 Feb 2021 13:47:53 +0200 Subject: [PATCH 23/24] Do not auto-expose metrics on matrix.DOMAIN/_synapse/metrics .. and other documentation improvements. --- ...configuring-playbook-prometheus-grafana.md | 23 +++++++++++++++---- group_vars/matrix_servers | 8 ++++++- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/docs/configuring-playbook-prometheus-grafana.md b/docs/configuring-playbook-prometheus-grafana.md index 2010b1b5..006c99e9 100644 --- a/docs/configuring-playbook-prometheus-grafana.md +++ b/docs/configuring-playbook-prometheus-grafana.md @@ -7,8 +7,6 @@ You can enable this with the following settings in your configuration file (`inv ```yaml matrix_prometheus_enabled: true -matrix_synapse_metrics_enabled: true - matrix_prometheus_node_exporter_enabled: true matrix_grafana_enabled: true @@ -25,23 +23,40 @@ matrix_grafana_default_admin_password: some_strong_password_chosen_by_you The dashboards will by default be available on the `stats.` subdomain, proxied via Nginx. + ## What does it do? Name | Description -----|---------- `matrix_prometheus_enabled`|Prometheus is a time series database. It holds all the data we're going to talk about. -`matrix_synapse_metrics_enabled`|Tell the Synapse server to expose metrics. This also cascades to other variables, which makes Prometheus collect said metrics `matrix_prometheus_node_exporter_enabled`|Node Exporter is an addon of sorts to Prometheus that collects generic system information such as CPU, memory, filesystem, and even system temperatures -`matrix_grafana_enabled`|Grafana is the visual component. It shows the dashboards with the graphs that we're interested in +`matrix_grafana_enabled`|Grafana is the visual component. It shows (on the `stats.` subdomain) the dashboards with the graphs that we're interested in `matrix_grafana_anonymous_access`|By default you need to log in to see graphs. If you want to publicly share your graphs (e.g. when asking for help in [`#synapse:matrix.org`](https://matrix.to/#/#synapse:matrix.org?via=matrix.org&via=privacytools.io&via=mozilla.org)) you'll want to enable this option. `matrix_grafana_default_admin_user`
`matrix_grafana_default_admin_password`|By default Grafana creates a user with `admin` as the username and password. If you feel this is insecure and you want to change it beforehand, you can do that here + ## Security and privacy Metrics and resulting graphs can contain a lot of information. This includes system specs but also usage patterns. This applies especially to small personal/family scale homeservers. Someone might be able to figure out when you wake up and go to sleep by looking at the graphs over time. Think about this before enabling anonymous access. And you should really not forget to change your Grafana password. Most of our docker containers run with limited system access, but the `prometheus-node-exporter` has access to the host network stack and (readonly) root filesystem. This is required to report on them. If you don't like that, you can set `matrix_prometheus_node_exporter_enabled: false` (which is actually the default). You will still get Synapse metrics with this container disabled. Both of the dashboards will always be enabled, so you can still look at historical data after disabling either source. + +## Collecting metrics to an external Prometheus server + +If you wish, you could expose homeserver metrics without enabling (installing) Prometheus and Grafana via the playbook. + +To do this, you may be interested in the following variables: + `matrix_synapse_metrics_enabled` to `true` + +Name | Description +-----|---------- +`matrix_synapse_metrics_enabled`|Set this to `true` to make Synapse expose metrics (locally, on the container network) +`matrix_nginx_proxy_proxy_synapse_metrics`|Set this to `true` to make matrix-nginx-proxy expose the Synapse metrics at `https://matrix.DOMAIN/_synapse/metrics` +`matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled`|Set this to `true` to password-protect (using HTTP Basic Auth) `https://matrix.DOMAIN/_synapse/metrics` (the username is always `prometheus`, the password is defined in `matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_key`) +`matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_key`|Set this to a password to use for HTTP Basic Auth for protecting `https://matrix.DOMAIN/_synapse/metrics` (the username is always `prometheus` - it's not configurable) + + ## More inforation - [Understanding Synapse Performance Issues Through Grafana Graphs](https://github.com/matrix-org/synapse/wiki/Understanding-Synapse-Performance-Issues-Through-Grafana-Graphs) at the Synapse Github Wiki diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index a8bddf6e..6d54f01d 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -992,7 +992,10 @@ matrix_nginx_proxy_proxy_matrix_federation_api_addr_sans_container: "127.0.0.1:8 matrix_nginx_proxy_container_federation_host_bind_port: "{{ matrix_federation_public_port }}" -matrix_nginx_proxy_proxy_synapse_metrics: "{{ matrix_synapse_metrics_enabled }}" +# This used to be hooked to `matrix_synapse_metrics_enabled`, but we don't do it anymore. +# The fact that someone wishes to enable Synapse metrics does not necessarily mean they want to make them public. +# A local Prometheus can consume them over the container network. +matrix_nginx_proxy_proxy_synapse_metrics: false matrix_nginx_proxy_proxy_synapse_metrics_addr_with_container: "matrix-synapse:{{ matrix_synapse_metrics_port }}" matrix_nginx_proxy_proxy_synapse_metrics_addr_sans_container: "127.0.0.1:{{ matrix_synapse_metrics_port }}" @@ -1300,6 +1303,9 @@ matrix_synapse_tls_private_key_path: ~ matrix_synapse_federation_port_openid_resource_required: "{{ not matrix_synapse_federation_enabled and (matrix_dimension_enabled or matrix_ma1sd_enabled) }}" +# If someone instals Prometheus via the playbook, they most likely wish to monitor Synapse. +matrix_synapse_metrics_enabled: "{{ matrix_prometheus_enabled }}" + matrix_synapse_email_enabled: "{{ matrix_mailer_enabled }}" matrix_synapse_email_smtp_host: "matrix-mailer" matrix_synapse_email_smtp_port: 8025 From 890e4ad1af0e89f14c9f87148919636d8628944e Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 12 Feb 2021 14:02:53 +0200 Subject: [PATCH 24/24] Announce Prometheus/Grafana --- CHANGELOG.md | 9 +++++++++ docs/configuring-playbook-prometheus-grafana.md | 12 ++++++------ docs/configuring-playbook.md | 2 ++ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e23e58d..a31fbc16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# 2021-02-12 + +## Monitoring/metrics support using Prometheus and Grafana + +Thanks to [@Peetz0r](https://github.com/Peetz0r), the playbook can now install a bunch of tools for monitoring your Matrix server: the [Prometheus](https://prometheus.io) time-series database server, the Prometheus [node-exporter](https://prometheus.io/docs/guides/node-exporter/) host metrics exporter, and the [Grafana](https://grafana.com/) web UI. + +To get get these installed, follow our [Enabling metrics and graphs (Prometheus, Grafana) for your Matrix server](docs/configuring-playbook-prometheus-grafana.md) docs page. + + # 2021-01-31 ## Etherpad support diff --git a/docs/configuring-playbook-prometheus-grafana.md b/docs/configuring-playbook-prometheus-grafana.md index 006c99e9..a10497cc 100644 --- a/docs/configuring-playbook-prometheus-grafana.md +++ b/docs/configuring-playbook-prometheus-grafana.md @@ -14,6 +14,7 @@ matrix_grafana_enabled: true matrix_grafana_anonymous_access: false # This has no relation to your Matrix user id. It can be any username you'd like. +# Changing the username subsequently won't work. matrix_grafana_default_admin_user: some_username_chosen_by_you # Passwords containing special characters may be troublesome. @@ -21,16 +22,16 @@ matrix_grafana_default_admin_user: some_username_chosen_by_you matrix_grafana_default_admin_password: some_strong_password_chosen_by_you ``` -The dashboards will by default be available on the `stats.` subdomain, proxied via Nginx. +By default, a [Grafana](https://grafana.com/) web user-interface will be available at `https://stats.`. ## What does it do? Name | Description -----|---------- -`matrix_prometheus_enabled`|Prometheus is a time series database. It holds all the data we're going to talk about. -`matrix_prometheus_node_exporter_enabled`|Node Exporter is an addon of sorts to Prometheus that collects generic system information such as CPU, memory, filesystem, and even system temperatures -`matrix_grafana_enabled`|Grafana is the visual component. It shows (on the `stats.` subdomain) the dashboards with the graphs that we're interested in +`matrix_prometheus_enabled`|[Prometheus](https://prometheus.io) is a time series database. It holds all the data we're going to talk about. +`matrix_prometheus_node_exporter_enabled`|[Node Exporter](https://prometheus.io/docs/guides/node-exporter/) is an addon of sorts to Prometheus that collects generic system information such as CPU, memory, filesystem, and even system temperatures +`matrix_grafana_enabled`|[Grafana](https://grafana.com/) is the visual component. It shows (on the `stats.` subdomain) the dashboards with the graphs that we're interested in `matrix_grafana_anonymous_access`|By default you need to log in to see graphs. If you want to publicly share your graphs (e.g. when asking for help in [`#synapse:matrix.org`](https://matrix.to/#/#synapse:matrix.org?via=matrix.org&via=privacytools.io&via=mozilla.org)) you'll want to enable this option. `matrix_grafana_default_admin_user`
`matrix_grafana_default_admin_password`|By default Grafana creates a user with `admin` as the username and password. If you feel this is insecure and you want to change it beforehand, you can do that here @@ -44,10 +45,9 @@ Most of our docker containers run with limited system access, but the `prometheu ## Collecting metrics to an external Prometheus server -If you wish, you could expose homeserver metrics without enabling (installing) Prometheus and Grafana via the playbook. +If you wish, you could expose homeserver metrics without enabling (installing) Prometheus and Grafana via the playbook. This may be useful for hooking Matrix services to an external Prometheus/Grafana installation. To do this, you may be interested in the following variables: - `matrix_synapse_metrics_enabled` to `true` Name | Description -----|---------- diff --git a/docs/configuring-playbook.md b/docs/configuring-playbook.md index 90dc01c5..70060292 100644 --- a/docs/configuring-playbook.md +++ b/docs/configuring-playbook.md @@ -35,6 +35,8 @@ When you're done with all the configuration you'd like to do, continue with [Ins - [Setting up Dynamic DNS](configuring-playbook-dynamic-dns.md) (optional) +- [Enabling metrics and graphs (Prometheus, Grafana) for your Matrix server](configuring-playbook-prometheus-grafana.md) (optional) + ### Core service adjustments - [Configuring Synapse](configuring-playbook-synapse.md) (optional)