diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | LICENSE | 2 | ||||
| -rw-r--r-- | defaults/main.yml | 55 | ||||
| -rw-r--r-- | handlers/main.yml | 8 | ||||
| -rw-r--r-- | readme.md | 11 | ||||
| -rw-r--r-- | tasks/alertmanager.yml | 2 | ||||
| -rw-r--r-- | tasks/blackbox.yml | 55 | ||||
| -rw-r--r-- | tasks/discord.yml | 42 | ||||
| -rw-r--r-- | tasks/main.yml | 46 | ||||
| -rw-r--r-- | templates/alertmanager.runit | 2 | ||||
| -rw-r--r-- | templates/alertmanager_discord.runit | 6 | ||||
| -rw-r--r-- | templates/blackbox_exporter.runit | 7 | ||||
| -rw-r--r-- | templates/prometheus.runit | 2 |
13 files changed, 217 insertions, 22 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0d20b64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.pyc @@ -1,4 +1,4 @@ -Copyright (c) 2019 Mitchell Riedstra +Copyright (c) 2021 Mitchell Riedstra Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/defaults/main.yml b/defaults/main.yml index 7d67f8e..baef38c 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -6,10 +6,10 @@ prometheus_data_dir: '{{prometheus_home_dir}}/data' prometheus_retention_time: "730d" prometheus_log: '{{prometheus_home_dir}}/log' prometheus_user: prometheus -prometheus_version: "2.22.1" +prometheus_version: "2.30.3" # Should resolve to "linux" prometheus_os: "{{ansible_facts['system']|lower}}" -prometheus_checksum: 9001a9cb939e0a6d9f2b67d22506c620bc9457777272fced43274b032ba35f44 +prometheus_checksum: 1ccd386d05f73a98b69aa5e0ed31fffac95cd9dadf7df1540daf2f182c5287e2 prometheus_architecture: amd64 prometheus_url: "https://github.com/prometheus/prometheus/releases/download/v{{prometheus_version}}/prometheus-{{prometheus_version}}.{{prometheus_os}}-{{prometheus_architecture}}.tar.gz" @@ -22,15 +22,28 @@ prometheus_opts: | --web.listen-address "{{prometheus_ui_listen}}" --web.enable-admin-api +# Be sure to set `--web.config.file={{prometheus_home_dir}}/conf/web_conf.yml` +# if you enable this +# prometheus_web_config: +# tls_server_config: +# cert_file: '{{prometheus_home_dir}}/conf/crt' +# key_file: '{{prometheus_home_dir}}/conf/key' +# client_ca_file: '{{prometheus_home_dir}}/conf/ca.crt' +# client_auth_type: RequireAndVerifyClientCert -# Filtered through 'to_nice_yaml' +# Anything defined in this map is written out verbatim to the conf/ dir +# prometheus_other_config: +# filename: | +# Some content in here + +# Filtered through 'to_nice_json' # prometheus_config: alertmanager: true -alertmanager_version: '0.21.0' +alertmanager_version: '0.23.0' alertmanager_checksum_alg: sha256 -alertmanager_checksum: 9ccd863937436fd6bfe650e22521a7f2e6a727540988eef515dde208f9aef232 +alertmanager_checksum: 77793c4d9bb92be98f7525f8bc50cb8adb8c5de2e944d5500e90ab13918771fc alertmanager_url: 'https://github.com/prometheus/alertmanager/releases/download/v{{alertmanager_version}}/alertmanager-{{alertmanager_version}}.{{prometheus_os}}-{{prometheus_architecture}}.tar.gz' alertmanager_port: '9093' # Note that this is also UDP @@ -39,6 +52,7 @@ alertmanager_data_dir: '{{prometheus_home_dir}}/alertmanager-data' alertmanager_log: '{{prometheus_home_dir}}/alertmanager.log' # Newlines are automatically replaced with spaces +# you can reuse the same --web.config.file from prometheus here alertmanager_opts: | --config.file "{{prometheus_home_dir}}/conf/alertmanager.yml" --storage.path {{alertmanager_data_dir}} @@ -53,9 +67,6 @@ alertmanager_opts: | # - prometheus1.example.com # alertmanager_nodes: "{{query('inventory_hostnames', 'prometheus')}}" -# Append the configured port number, used in the config in a few spots -alertmanager_nodes_w_port: '{{alertmanager_nodes | forEachAppend(":" + alertmanager_cluster_port)}}' - # alertmanager_conf: # global: # # The API URL to use for Slack notifications. @@ -69,3 +80,31 @@ alertmanager_nodes_w_port: '{{alertmanager_nodes | forEachAppend(":" + alertmana # slack_configs: # - channel: '#{{slack_alerts_channel}}' # send_resolved: true + + +blackbox_exporter: false +blackbox_exporter_version: 0.19.0 +blackbox_exporter_checksum: af2ae1394c4f9b46962ac1510e1dacac78115c11e625991fb6c54825d2240896 +blackbox_exporter_url: 'https://github.com/prometheus/blackbox_exporter/releases/download/v{{blackbox_exporter_version}}/blackbox_exporter-{{blackbox_exporter_version}}.{{prometheus_os}}-{{prometheus_architecture}}.tar.gz' +blackbox_exporter_bind: 127.0.0.1:9115 +blackbox_exporter_log: '{{prometheus_home_dir}}/blackbox_exporter.log' +blackbox_exporter_opts: | + --web.listen-address="{{blackbox_exporter_bind}}" + --config.file={{prometheus_home_dir}}/conf/blackbox_exporter.yml +# Check out: +# https://github.com/prometheus/blackbox_exporter/blob/master/example.yml +blackbox_exporter_conf: + + +alertmanager_discord: false +alertmanager_discord_url: 'https://git.riedstra.dev/pub/alertmanager-discord/plain/build/alertmanager-discord-{{prometheus_os}}-{{prometheus_architecture}}.gz?h=binaries' +# Linux amd64 +alertmanager_discord_checksum: 3cd33d08824f140d93d7cdbfcb054be730b542d2afd33e9b47723b8270809325 +blackbox_exporter_checksum_alg: sha256 +alertmanager_discord_port: '9098' +alertmanager_discord_webhook_url: "<changeme>" +alertmanager_discord_opts: | + -listen.address "127.0.0.1:{{alertmanager_discord_port}}" + -webhook.url "{{alertmanager_discord_webhook_url}}" + +alertmanager_discord_log: '{{prometheus_home_dir}}/alertmanager_discord.log' diff --git a/handlers/main.yml b/handlers/main.yml index abaa110..d7482b5 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -6,3 +6,11 @@ runit: name: alertmanager state: restarted +- name: Restart Blackbox Exporter + runit: + name: blackbox_exporter + state: restarted +- name: Restart Alertmanager Discord + runit: + name: alertmanager_discord + state: restarted @@ -1,13 +1,8 @@ -# Prometheus role for Alpine, Void Linux, Ubuntu, and CentOS - -You can easily override the default Prometheus template with -`prometheus_template_source: production/prometheus.yml` in your `group_vars` or -so. +# Prometheus role for systems using `runit` Updating and or changing versions of Prometheus is as simple as changing the version and checksum in your vars. -Works well with my Netdata role which is available here: - -https://git.riedstra.us/ansible +Check over `defaults/main.yml` for a heavily commented example +Be sure to check out the `node_exporter` role as well diff --git a/tasks/alertmanager.yml b/tasks/alertmanager.yml index 4c9e4d3..f89e7ed 100644 --- a/tasks/alertmanager.yml +++ b/tasks/alertmanager.yml @@ -12,7 +12,7 @@ #!/bin/sh set -e if ! [ -e "{{prometheus_home_dir}}/{{alertmanager_extract_dir}}" ] ; then - su - '{{prometheus_user}}' -c 'tar xzf alertmanager-{{alertmanager_version}}.tgz' + su - '{{prometheus_user}}' -c 'cd "{{prometheus_home_dir}}"; tar xzf alertmanager-{{alertmanager_version}}.tgz' exit 50 fi register: res diff --git a/tasks/blackbox.yml b/tasks/blackbox.yml new file mode 100644 index 0000000..c6bcd43 --- /dev/null +++ b/tasks/blackbox.yml @@ -0,0 +1,55 @@ +--- +- name: Download Blackbox Exporter + get_url: + url: "{{blackbox_exporter_url}}" + dest: "{{prometheus_home_dir}}/blackbox_exporter-{{blackbox_exporter_version}}.tgz" + mode: '0600' + owner: "{{prometheus_user}}" + checksum: '{{blackbox_exporter_checksum_alg}}:{{blackbox_exporter_checksum}}' +- set_fact: "blackbox_exporter_extract_dir=blackbox_exporter-{{blackbox_exporter_version}}.{{prometheus_os}}-{{prometheus_architecture}}" +- name: Extract Blackbox Exporter + shell: | + #!/bin/sh + set -e + if ! [ -e "{{prometheus_home_dir}}/{{blackbox_exporter_extract_dir}}" ] ; then + su - '{{prometheus_user}}' -c 'cd "{{prometheus_home_dir}}"; tar xzf blackbox_exporter-{{blackbox_exporter_version}}.tgz' + exit 50 + fi + register: res + changed_when: res is defined and res.rc == 50 + ignore_errors: true + notify: Restart Blackbox Exporter +- name: Link Blackbox Exporter directroy + file: + state: link + src: '{{blackbox_exporter_extract_dir}}' + dest: '{{prometheus_home_dir}}/blackbox_exporter' + force: yes +- name: Write Blackbox Exporter configuration file + copy: + content: '{{blackbox_exporter_conf | to_nice_yaml}}' + dest: '{{prometheus_home_dir}}/conf/blackbox_exporter.yml' + owner: '{{prometheus_user}}' + mode: '0600' + notify: Restart Blackbox Exporter + tags: + - configuration +- name: Create Runit Directory + file: + state: directory + dest: /etc/sv/blackbox_exporter + owner: root + mode: '0755' +- name: Write runit service + template: + src: blackbox_exporter.runit + dest: /etc/sv/blackbox_exporter/run + owner: root + mode: '0755' + tags: + - configuration +- name: Enable Runit service + file: + state: link + src: /etc/sv/blackbox_exporter + dest: /var/service/blackbox_exporter diff --git a/tasks/discord.yml b/tasks/discord.yml new file mode 100644 index 0000000..aaf8c9b --- /dev/null +++ b/tasks/discord.yml @@ -0,0 +1,42 @@ +--- +- name: Download Alertmanager Discord + get_url: + url: "{{alertmanager_discord_url}}" + dest: "{{prometheus_home_dir}}/alertmanager_discord.gz" + mode: '0600' + owner: "{{prometheus_user}}" + checksum: 'sha256:{{alertmanager_discord_checksum}}' +- name: Extract Alertmanager Discord + shell: | + #!/bin/sh + set -e + cd "{{prometheus_home_dir}}" + gzip -dc < alertmanager_discord.gz > alertmanager_discord + if ! [ -x alertmanager_discord ] ; then + chmod +x alertmanager_discord + fi + chown '{{prometheus_user}}' alertmanager_discord + exit 50 + register: res + changed_when: res is defined and res.rc == 50 + ignore_errors: true + notify: Restart Alertmanager Discord +- name: Create Runit Directory + file: + state: directory + dest: /etc/sv/alertmanager_discord + owner: root + mode: '0755' +- name: Write runit service + template: + src: alertmanager_discord.runit + dest: /etc/sv/alertmanager_discord/run + owner: root + mode: '0755' + tags: + - configuration +- name: Enable Runit service + file: + state: link + src: /etc/sv/alertmanager_discord + dest: /var/service/alertmanager_discord diff --git a/tasks/main.yml b/tasks/main.yml index 19ca131..e9a017a 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -30,11 +30,12 @@ #!/bin/sh set -e if ! [ -e "{{prometheus_home_dir}}/{{prometheus_extract_dir}}" ] ; then - su - '{{prometheus_user}}' -c 'tar xzf prometheus-{{prometheus_version}}.tgz' + su - '{{prometheus_user}}' -c 'cd "{{prometheus_home_dir}}"; tar xzf prometheus-{{prometheus_version}}.tgz' exit 50 fi register: res changed_when: res is defined and res.rc == 50 + # failed_when: res is defined and ( res.rc != 50 or res.rc != 0 ) ignore_errors: true notify: Restart Prometheus - name: Link prometheus directory @@ -53,13 +54,34 @@ - prometheus.yml - name: Write prometheus configuration file copy: - content: '{{prometheus_config | to_nice_yaml}}' + content: '{{prometheus_config | to_nice_json}}' dest: '{{prometheus_home_dir}}/conf/prometheus.yml' owner: '{{prometheus_user}}' mode: '0600' notify: Restart prometheus tags: - configuration +- name: Write prometheus web configuration file + copy: + content: '{{prometheus_web_config | to_nice_json}}' + dest: '{{prometheus_home_dir}}/conf/web_conf.yml' + owner: '{{prometheus_user}}' + mode: '0600' + when: prometheus_web_config is defined + notify: Restart prometheus + tags: + - configuration +- name: Write other config files + copy: + content: '{{item.content}}' + dest: '{{prometheus_home_dir}}/conf/{{item.filename}}' + owner: '{{prometheus_user}}' + mode: '0600' + loop: '{{prometheus_other_config | dict2items(key_name="filename", value_name="content")}}' + when: prometheus_other_config is defined + notify: Restart prometheus + tags: + - configuration - name: Create Runit Directory file: state: directory @@ -72,6 +94,8 @@ dest: /etc/sv/prometheus/run owner: root mode: '0755' + tags: + - configuration - name: Enable Runit service file: state: link @@ -86,3 +110,21 @@ tags: - alertmanager when: alertmanager +- name: Include Blackbox Exporter tasks + include_tasks: + file: blackbox.yml + apply: + tags: + - blackbox + tags: + - blackbox + when: blackbox_exporter +- name: Include Discord tasks + include_tasks: + file: discord.yml + apply: + tags: + - discord + tags: + - discord + when: alertmanager_discord diff --git a/templates/alertmanager.runit b/templates/alertmanager.runit index e300d47..4c53493 100644 --- a/templates/alertmanager.runit +++ b/templates/alertmanager.runit @@ -1,6 +1,6 @@ #!/bin/sh +exec 3>>{{alertmanager_log}} 2>&3 1>&3 exec chpst -u {{prometheus_user}} \ {{prometheus_home_dir}}/alertmanager/alertmanager \ {{alertmanager_opts | replace('\n', ' ')}} \ - >> {{alertmanager_log}} 2>&1 diff --git a/templates/alertmanager_discord.runit b/templates/alertmanager_discord.runit new file mode 100644 index 0000000..28f82a8 --- /dev/null +++ b/templates/alertmanager_discord.runit @@ -0,0 +1,6 @@ +#!/bin/sh +exec 3>>{{alertmanager_discord_log}} 2>&3 1>&3 +exec chpst -u {{prometheus_user}} \ + {{prometheus_home_dir}}/alertmanager_discord \ + {{alertmanager_discord_opts | replace('\n', ' ')}} + diff --git a/templates/blackbox_exporter.runit b/templates/blackbox_exporter.runit new file mode 100644 index 0000000..115bc7b --- /dev/null +++ b/templates/blackbox_exporter.runit @@ -0,0 +1,7 @@ +#!/bin/sh +exec 3>>{{blackbox_exporter_log}} 2>&3 1>&3 +exec chpst -u {{prometheus_user}} \ + {{prometheus_home_dir}}/blackbox_exporter/blackbox_exporter \ + {{blackbox_exporter_opts | replace('\n', ' ')}} \ + + diff --git a/templates/prometheus.runit b/templates/prometheus.runit index b14e2d1..9866d08 100644 --- a/templates/prometheus.runit +++ b/templates/prometheus.runit @@ -1,5 +1,5 @@ #!/bin/sh +exec 3>>{{prometheus_log}} 2>&3 1>&3 exec chpst -u {{prometheus_user}} \ {{prometheus_home_dir}}/prometheus/prometheus \ {{prometheus_opts | replace('\n', ' ')}} \ - > {{prometheus_log}} 2>&1 |