From 13ef9e85cf664a03942e3d38280238988a247a2f Mon Sep 17 00:00:00 2001 From: Peetz0r Date: Fri, 29 Jan 2021 05:29:25 +0100 Subject: [PATCH] Prometheus Initial attempt. Seems to work fine. Only tested on debian amd64 so far --- group_vars/matrix_servers | 21 +++++ roles/matrix-prometheus/defaults/main.yml | 51 ++++++++++ roles/matrix-prometheus/tasks/init.yml | 5 + roles/matrix-prometheus/tasks/main.yml | 14 +++ roles/matrix-prometheus/tasks/setup.yml | 93 +++++++++++++++++++ .../tasks/validate_config.yml | 7 ++ .../templates/prometheus.yml.j2 | 35 +++++++ .../systemd/matrix-prometheus.service.j2 | 42 +++++++++ setup.yml | 1 + 9 files changed, 269 insertions(+) create mode 100644 roles/matrix-prometheus/defaults/main.yml create mode 100644 roles/matrix-prometheus/tasks/init.yml create mode 100644 roles/matrix-prometheus/tasks/main.yml create mode 100644 roles/matrix-prometheus/tasks/setup.yml create mode 100644 roles/matrix-prometheus/tasks/validate_config.yml create mode 100644 roles/matrix-prometheus/templates/prometheus.yml.j2 create mode 100644 roles/matrix-prometheus/templates/systemd/matrix-prometheus.service.j2 diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 5d76a60c..833089a2 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1368,6 +1368,27 @@ matrix_synapse_admin_container_self_build: "{{ matrix_architecture != 'amd64' }} +###################################################################### +# +# matrix-prometheus +# +###################################################################### + +matrix_prometheus_enabled: false + +# Normally, matrix-nginx-proxy is enabled and nginx can reach Prometheus over the container network. +# If matrix-nginx-proxy is not enabled, or you otherwise have a need for it, you can expose +# Prometheus' HTTP port to the local host. +matrix_prometheus_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else '127.0.0.1:9090' }}" + +###################################################################### +# +# /matrix-prometheus +# +###################################################################### + + + ###################################################################### # # matrix-registration diff --git a/roles/matrix-prometheus/defaults/main.yml b/roles/matrix-prometheus/defaults/main.yml new file mode 100644 index 00000000..10424424 --- /dev/null +++ b/roles/matrix-prometheus/defaults/main.yml @@ -0,0 +1,51 @@ +# matrix-prometheus is an open-source systems monitoring and alerting toolkit +# See: https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md + +matrix_prometheus_enabled: false + +matrix_prometheus_docker_image: "docker.io/prom/prometheus:v2.24.1" +matrix_prometheus_docker_image_force_pull: "{{ matrix_prometheus_docker_image.endswith(':latest') }}" + +matrix_synapse_prometheus_rules_download_url: "https://raw.githubusercontent.com/matrix-org/synapse/{{ matrix_synapse_docker_image_tag }}/contrib/prometheus/synapse-v2.rules" + +matrix_prometheus_base_path: "{{ matrix_base_data_path }}/prometheus" +matrix_prometheus_config_path: "{{ matrix_prometheus_base_path }}/config" +matrix_prometheus_data_path: "{{ matrix_prometheus_base_path }}/data" + +# A list of extra arguments to pass to the container +matrix_prometheus_container_extra_arguments: [] + +# List of systemd services that matrix-prometheus.service depends on +matrix_prometheus_systemd_required_services_list: ['docker.service'] + +# List of systemd services that matrix-prometheus.service wants +matrix_prometheus_systemd_wanted_services_list: [] + +# Controls whether the matrix-prometheus container exposes its HTTP port (tcp/9090 in the container). +# +# Takes an ":" or "" value (e.g. "127.0.0.1:9090"), or empty string to not expose. +matrix_prometheus_container_http_host_bind_port: '' + + +# Default prometheus configuration template which covers the generic use case. +# You can customize it by controlling the various variables inside it. +# +# For a more advanced customization, you can extend the default (see `matrix_prometheus_configuration_extension_yaml`) +# or completely replace this variable with your own template. +matrix_prometheus_configuration_yaml: "{{ lookup('template', 'templates/prometheus.yml.j2') }}" + +matrix_prometheus_configuration_extension_yaml: | + # Your custom YAML configuration goes here. + # This configuration extends the default starting configuration (`matrix_prometheus_configuration_yaml`). + # + # You can override individual variables from the default configuration, or introduce new ones. + # + # If you need something more special, you can take full control by + # completely redefining `matrix_prometheus_configuration_yaml`. + +matrix_prometheus_configuration_extension: "{{ matrix_prometheus_configuration_extension_yaml|from_yaml if matrix_prometheus_configuration_extension_yaml|from_yaml is mapping else {} }}" + +# Holds the final configuration (a combination of the default and its extension). +# You most likely don't need to touch this variable. Instead, see `matrix_prometheus_configuration_yaml`. +matrix_prometheus_configuration: "{{ matrix_prometheus_configuration_yaml|from_yaml|combine(matrix_prometheus_configuration_extension, recursive=True) }}" + diff --git a/roles/matrix-prometheus/tasks/init.yml b/roles/matrix-prometheus/tasks/init.yml new file mode 100644 index 00000000..12fae831 --- /dev/null +++ b/roles/matrix-prometheus/tasks/init.yml @@ -0,0 +1,5 @@ +- set_fact: + matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-prometheus.service'] }}" + when: matrix_prometheus_enabled|bool + + diff --git a/roles/matrix-prometheus/tasks/main.yml b/roles/matrix-prometheus/tasks/main.yml new file mode 100644 index 00000000..2290048f --- /dev/null +++ b/roles/matrix-prometheus/tasks/main.yml @@ -0,0 +1,14 @@ +- import_tasks: "{{ role_path }}/tasks/init.yml" + tags: + - always + +- import_tasks: "{{ role_path }}/tasks/validate_config.yml" + when: run_setup|bool + tags: + - setup-all + - setup-prometheus + +- import_tasks: "{{ role_path }}/tasks/setup.yml" + tags: + - setup-all + - setup-prometheus diff --git a/roles/matrix-prometheus/tasks/setup.yml b/roles/matrix-prometheus/tasks/setup.yml new file mode 100644 index 00000000..1746b961 --- /dev/null +++ b/roles/matrix-prometheus/tasks/setup.yml @@ -0,0 +1,93 @@ +--- + +# +# Tasks related to setting up matrix-prometheus +# + +- name: Ensure matrix-prometheus image is pulled + docker_image: + name: "{{ matrix_prometheus_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_prometheus_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_prometheus_docker_image_force_pull }}" + when: "matrix_prometheus_enabled|bool" + +- name: Ensure Prometheus paths exists + file: + path: "{{ item }}" + state: directory + mode: 0750 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + with_items: + - "{{ matrix_prometheus_base_path }}" + - "{{ matrix_prometheus_config_path }}" + - "{{ matrix_prometheus_data_path }}" + when: matrix_prometheus_enabled|bool + +- name: Ensure prometheus.yml installed + copy: + content: "{{ matrix_prometheus_configuration|to_nice_yaml }}" + dest: "{{ matrix_prometheus_config_path }}/prometheus.yml" + mode: 0644 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + when: matrix_prometheus_enabled|bool + +- name: Download synapse-v2.rules + get_url: + url: "{{ matrix_synapse_prometheus_rules_download_url }}" + dest: "{{ matrix_prometheus_config_path }}/synapse-v2.rules" + force: true + mode: 0440 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + when: matrix_prometheus_enabled|bool + + +- name: Ensure matrix-prometheus.service installed + template: + src: "{{ role_path }}/templates/systemd/matrix-prometheus.service.j2" + dest: "{{ matrix_systemd_path }}/matrix-prometheus.service" + mode: 0644 + register: matrix_prometheus_systemd_service_result + when: matrix_prometheus_enabled|bool + +- name: Ensure systemd reloaded after matrix-prometheus.service installation + service: + daemon_reload: yes + when: "matrix_prometheus_enabled|bool and matrix_prometheus_systemd_service_result.changed" + +# +# Tasks related to getting rid of matrix-prometheus (if it was previously enabled) +# + +- name: Check existence of matrix-prometheus service + stat: + path: "{{ matrix_systemd_path }}/matrix-prometheus.service" + register: matrix_prometheus_service_stat + +- name: Ensure matrix-prometheus is stopped + service: + name: matrix-prometheus + state: stopped + daemon_reload: yes + register: stopping_result + when: "not matrix_prometheus_enabled|bool and matrix_prometheus_service_stat.stat.exists" + +- name: Ensure matrix-prometheus.service doesn't exist + file: + path: "{{ matrix_systemd_path }}/matrix-prometheus.service" + state: absent + when: "not matrix_prometheus_enabled|bool and matrix_prometheus_service_stat.stat.exists" + +- name: Ensure systemd reloaded after matrix-prometheus.service removal + service: + daemon_reload: yes + when: "not matrix_prometheus_enabled|bool and matrix_prometheus_service_stat.stat.exists" + +- name: Ensure matrix-prometheus Docker image doesn't exist + docker_image: + name: "{{ matrix_prometheus_docker_image }}" + state: absent + when: "not matrix_prometheus_enabled|bool" diff --git a/roles/matrix-prometheus/tasks/validate_config.yml b/roles/matrix-prometheus/tasks/validate_config.yml new file mode 100644 index 00000000..b614b438 --- /dev/null +++ b/roles/matrix-prometheus/tasks/validate_config.yml @@ -0,0 +1,7 @@ +--- + +- name: Fail if Synapse metrics not enabled + fail: + msg: > + You need to enable `matrix_synapse_metrics_enabled` for Prometheus grab metrics. + when: "not matrix_synapse_metrics_enabled" diff --git a/roles/matrix-prometheus/templates/prometheus.yml.j2 b/roles/matrix-prometheus/templates/prometheus.yml.j2 new file mode 100644 index 00000000..7b90baeb --- /dev/null +++ b/roles/matrix-prometheus/templates/prometheus.yml.j2 @@ -0,0 +1,35 @@ +#jinja2: lstrip_blocks: "True" +global: + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: + {% if matrix_synapse_metrics_enabled %} + - 'synapse-v2.rules' + {% endif %} + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + - job_name: 'prometheus' + + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 5s + scrape_timeout: 5s + + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + + static_configs: + - targets: ['localhost:9090'] + + {% if matrix_synapse_metrics_enabled %} + - job_name: 'synapse' + metrics_path: '/_synapse/metrics' + static_configs: + - targets: ['matrix-synapse:{{ matrix_synapse_metrics_port }}'] + {% endif %} + diff --git a/roles/matrix-prometheus/templates/systemd/matrix-prometheus.service.j2 b/roles/matrix-prometheus/templates/systemd/matrix-prometheus.service.j2 new file mode 100644 index 00000000..dd3ac72c --- /dev/null +++ b/roles/matrix-prometheus/templates/systemd/matrix-prometheus.service.j2 @@ -0,0 +1,42 @@ +#jinja2: lstrip_blocks: "True" +[Unit] +Description=matrix-prometheus +{% for service in matrix_prometheus_systemd_required_services_list %} +Requires={{ service }} +After={{ service }} +{% endfor %} +{% for service in matrix_prometheus_systemd_wanted_services_list %} +Wants={{ service }} +{% endfor %} +DefaultDependencies=no + +[Service] +Type=simple +Environment="HOME={{ matrix_systemd_unit_home_path }}" +ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus 2>/dev/null' +ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus 2>/dev/null' + + +ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-prometheus \ + --log-driver=none \ + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + --cap-drop=ALL \ + --network={{ matrix_docker_network }} \ + {% if matrix_prometheus_container_http_host_bind_port %} + -p {{ matrix_prometheus_container_http_host_bind_port }}:9090 \ + {% endif %} + -v {{ matrix_prometheus_config_path }}:/etc/prometheus:z \ + -v {{ matrix_prometheus_data_path }}:/prometheus:z \ + {% for arg in matrix_prometheus_container_extra_arguments %} + {{ arg }} \ + {% endfor %} + {{ matrix_prometheus_docker_image }} + +ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus 2>/dev/null' +ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus 2>/dev/null' +Restart=always +RestartSec=30 +SyslogIdentifier=matrix-prometheus + +[Install] +WantedBy=multi-user.target diff --git a/setup.yml b/setup.yml index 9bb1788f..7965185c 100755 --- a/setup.yml +++ b/setup.yml @@ -28,6 +28,7 @@ - matrix-bot-matrix-reminder-bot - matrix-synapse - matrix-synapse-admin + - matrix-prometheus - matrix-registration - matrix-client-element - matrix-jitsi