Update prometheus, alertmanager and add a few new optionsHEADmaster

TLS Support for everything if you configure the vars properly.
Blackbox exporter.
Alertmanager discord for sending alerts there via webhooks.

Runit services for all of them.

13 files changed, 217 insertions, 22 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..0d20b64
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+*.pyc
diff --git a/LICENSE b/LICENSE
index eaf4522..78f7235 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,4 +1,4 @@
-Copyright (c) 2019 Mitchell Riedstra
+Copyright (c) 2021 Mitchell Riedstra
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/defaults/main.yml b/defaults/main.yml
index 7d67f8e..baef38c 100644
--- a/defaults/main.yml
+++ b/defaults/main.yml
@@ -6,10 +6,10 @@ prometheus_data_dir: '{{prometheus_home_dir}}/data'
 prometheus_retention_time: "730d"
 prometheus_log: '{{prometheus_home_dir}}/log'
 prometheus_user: prometheus
-prometheus_version: "2.22.1"
+prometheus_version: "2.30.3"
 # Should resolve to "linux"
 prometheus_os: "{{ansible_facts['system']|lower}}"
-prometheus_checksum: 9001a9cb939e0a6d9f2b67d22506c620bc9457777272fced43274b032ba35f44
+prometheus_checksum: 1ccd386d05f73a98b69aa5e0ed31fffac95cd9dadf7df1540daf2f182c5287e2
 prometheus_architecture: amd64
 prometheus_url: "https://github.com/prometheus/prometheus/releases/download/v{{prometheus_version}}/prometheus-{{prometheus_version}}.{{prometheus_os}}-{{prometheus_architecture}}.tar.gz"
 
@@ -22,15 +22,28 @@ prometheus_opts: |
        --web.listen-address "{{prometheus_ui_listen}}"
        --web.enable-admin-api
 
+# Be sure to set `--web.config.file={{prometheus_home_dir}}/conf/web_conf.yml`
+# if you enable this
+# prometheus_web_config:
+#   tls_server_config:
+#     cert_file: '{{prometheus_home_dir}}/conf/crt'
+#     key_file: '{{prometheus_home_dir}}/conf/key'
+#     client_ca_file: '{{prometheus_home_dir}}/conf/ca.crt'
+#     client_auth_type: RequireAndVerifyClientCert
 
-# Filtered through 'to_nice_yaml'
+# Anything defined in this map is written out verbatim to the conf/ dir
+# prometheus_other_config:
+#   filename: |
+#     Some content in here
+
+# Filtered through 'to_nice_json'
 # prometheus_config:
 
 alertmanager: true
 
-alertmanager_version: '0.21.0'
+alertmanager_version: '0.23.0'
 alertmanager_checksum_alg: sha256
-alertmanager_checksum: 9ccd863937436fd6bfe650e22521a7f2e6a727540988eef515dde208f9aef232
+alertmanager_checksum: 77793c4d9bb92be98f7525f8bc50cb8adb8c5de2e944d5500e90ab13918771fc
 alertmanager_url: 'https://github.com/prometheus/alertmanager/releases/download/v{{alertmanager_version}}/alertmanager-{{alertmanager_version}}.{{prometheus_os}}-{{prometheus_architecture}}.tar.gz'
 alertmanager_port: '9093'
 # Note that this is also UDP
@@ -39,6 +52,7 @@ alertmanager_data_dir: '{{prometheus_home_dir}}/alertmanager-data'
 alertmanager_log: '{{prometheus_home_dir}}/alertmanager.log'
 
 # Newlines are automatically replaced with spaces
+# you can reuse the same --web.config.file from prometheus here
 alertmanager_opts: |
   --config.file "{{prometheus_home_dir}}/conf/alertmanager.yml"
   --storage.path {{alertmanager_data_dir}}
@@ -53,9 +67,6 @@ alertmanager_opts: |
 #   - prometheus1.example.com
 # alertmanager_nodes: "{{query('inventory_hostnames', 'prometheus')}}"
 
-# Append the configured port number, used in the config in a few spots
-alertmanager_nodes_w_port: '{{alertmanager_nodes | forEachAppend(":" + alertmanager_cluster_port)}}'
-
 # alertmanager_conf:
 #   global:
 #     # The API URL to use for Slack notifications.
@@ -69,3 +80,31 @@ alertmanager_nodes_w_port: '{{alertmanager_nodes | forEachAppend(":" + alertmana
 #       slack_configs:
 #         - channel: '#{{slack_alerts_channel}}'
 #           send_resolved: true
+
+
+blackbox_exporter: false
+blackbox_exporter_version: 0.19.0
+blackbox_exporter_checksum: af2ae1394c4f9b46962ac1510e1dacac78115c11e625991fb6c54825d2240896
+blackbox_exporter_url: 'https://github.com/prometheus/blackbox_exporter/releases/download/v{{blackbox_exporter_version}}/blackbox_exporter-{{blackbox_exporter_version}}.{{prometheus_os}}-{{prometheus_architecture}}.tar.gz'
+blackbox_exporter_bind: 127.0.0.1:9115
+blackbox_exporter_log: '{{prometheus_home_dir}}/blackbox_exporter.log'
+blackbox_exporter_opts: |
+  --web.listen-address="{{blackbox_exporter_bind}}"
+  --config.file={{prometheus_home_dir}}/conf/blackbox_exporter.yml
+# Check out:
+# https://github.com/prometheus/blackbox_exporter/blob/master/example.yml
+blackbox_exporter_conf:
+  
+
+alertmanager_discord: false
+alertmanager_discord_url: 'https://git.riedstra.dev/pub/alertmanager-discord/plain/build/alertmanager-discord-{{prometheus_os}}-{{prometheus_architecture}}.gz?h=binaries'
+# Linux amd64
+alertmanager_discord_checksum: 3cd33d08824f140d93d7cdbfcb054be730b542d2afd33e9b47723b8270809325
+blackbox_exporter_checksum_alg: sha256
+alertmanager_discord_port: '9098'
+alertmanager_discord_webhook_url: "<changeme>"
+alertmanager_discord_opts: |
+  -listen.address "127.0.0.1:{{alertmanager_discord_port}}"
+  -webhook.url "{{alertmanager_discord_webhook_url}}"
+  
+alertmanager_discord_log: '{{prometheus_home_dir}}/alertmanager_discord.log'
diff --git a/handlers/main.yml b/handlers/main.yml
index abaa110..d7482b5 100644
--- a/handlers/main.yml
+++ b/handlers/main.yml
@@ -6,3 +6,11 @@
   runit:
     name: alertmanager
     state: restarted
+- name: Restart Blackbox Exporter
+  runit:
+    name: blackbox_exporter
+    state: restarted
+- name: Restart Alertmanager Discord
+  runit:
+    name: alertmanager_discord
+    state: restarted
diff --git a/readme.md b/readme.md
index f563577..12d106e 100644
--- a/readme.md
+++ b/readme.md
@@ -1,13 +1,8 @@
-# Prometheus role for Alpine, Void Linux, Ubuntu, and CentOS
-
-You can easily override the default Prometheus template with
-`prometheus_template_source: production/prometheus.yml` in your `group_vars` or
-so.
+# Prometheus role for systems using `runit`
 
 Updating and or changing versions of Prometheus is as simple as changing
 the version and checksum in your vars.
 
-Works well with my Netdata role which is available here:
-
-https://git.riedstra.us/ansible
+Check over `defaults/main.yml` for a heavily commented example
 
+Be sure to check out the `node_exporter` role as well
diff --git a/tasks/alertmanager.yml b/tasks/alertmanager.yml
index 4c9e4d3..f89e7ed 100644
--- a/tasks/alertmanager.yml
+++ b/tasks/alertmanager.yml
@@ -12,7 +12,7 @@
     #!/bin/sh
     set -e
     if ! [ -e "{{prometheus_home_dir}}/{{alertmanager_extract_dir}}" ] ; then
-      su - '{{prometheus_user}}' -c 'tar xzf alertmanager-{{alertmanager_version}}.tgz'
+      su - '{{prometheus_user}}' -c 'cd "{{prometheus_home_dir}}"; tar xzf alertmanager-{{alertmanager_version}}.tgz'
       exit 50
     fi
   register: res
diff --git a/tasks/blackbox.yml b/tasks/blackbox.yml
new file mode 100644
index 0000000..c6bcd43
--- /dev/null
+++ b/tasks/blackbox.yml
@@ -0,0 +1,55 @@
+---
+- name: Download Blackbox Exporter
+  get_url:
+    url: "{{blackbox_exporter_url}}"
+    dest: "{{prometheus_home_dir}}/blackbox_exporter-{{blackbox_exporter_version}}.tgz"
+    mode: '0600'
+    owner: "{{prometheus_user}}"
+    checksum: '{{blackbox_exporter_checksum_alg}}:{{blackbox_exporter_checksum}}'
+- set_fact: "blackbox_exporter_extract_dir=blackbox_exporter-{{blackbox_exporter_version}}.{{prometheus_os}}-{{prometheus_architecture}}"
+- name: Extract Blackbox Exporter
+  shell: |
+    #!/bin/sh
+    set -e
+    if ! [ -e "{{prometheus_home_dir}}/{{blackbox_exporter_extract_dir}}" ] ; then
+      su - '{{prometheus_user}}' -c 'cd "{{prometheus_home_dir}}"; tar xzf blackbox_exporter-{{blackbox_exporter_version}}.tgz'
+      exit 50
+    fi
+  register: res
+  changed_when: res is defined and res.rc == 50
+  ignore_errors: true
+  notify: Restart Blackbox Exporter
+- name: Link Blackbox Exporter directroy
+  file:
+    state: link
+    src: '{{blackbox_exporter_extract_dir}}'
+    dest: '{{prometheus_home_dir}}/blackbox_exporter'
+    force: yes
+- name: Write Blackbox Exporter configuration file
+  copy:
+    content: '{{blackbox_exporter_conf | to_nice_yaml}}'
+    dest: '{{prometheus_home_dir}}/conf/blackbox_exporter.yml'
+    owner: '{{prometheus_user}}'
+    mode: '0600'
+  notify: Restart Blackbox Exporter
+  tags:
+    - configuration
+- name: Create Runit Directory
+  file:
+    state: directory
+    dest: /etc/sv/blackbox_exporter
+    owner: root
+    mode: '0755'
+- name: Write runit service
+  template:
+    src: blackbox_exporter.runit
+    dest: /etc/sv/blackbox_exporter/run
+    owner: root
+    mode: '0755'
+  tags:
+    - configuration
+- name: Enable Runit service
+  file:
+    state: link
+    src: /etc/sv/blackbox_exporter
+    dest: /var/service/blackbox_exporter
diff --git a/tasks/discord.yml b/tasks/discord.yml
new file mode 100644
index 0000000..aaf8c9b
--- /dev/null
+++ b/tasks/discord.yml
@@ -0,0 +1,42 @@
+---
+- name: Download Alertmanager Discord
+  get_url:
+    url: "{{alertmanager_discord_url}}"
+    dest: "{{prometheus_home_dir}}/alertmanager_discord.gz"
+    mode: '0600'
+    owner: "{{prometheus_user}}"
+    checksum: 'sha256:{{alertmanager_discord_checksum}}'
+- name: Extract Alertmanager Discord
+  shell: |
+    #!/bin/sh
+    set -e
+    cd "{{prometheus_home_dir}}"
+    gzip -dc < alertmanager_discord.gz > alertmanager_discord
+    if ! [ -x alertmanager_discord ] ; then
+      chmod +x alertmanager_discord
+    fi
+    chown '{{prometheus_user}}' alertmanager_discord
+    exit 50
+  register: res
+  changed_when: res is defined and res.rc == 50
+  ignore_errors: true
+  notify: Restart Alertmanager Discord
+- name: Create Runit Directory
+  file:
+    state: directory
+    dest: /etc/sv/alertmanager_discord
+    owner: root
+    mode: '0755'
+- name: Write runit service
+  template:
+    src: alertmanager_discord.runit
+    dest: /etc/sv/alertmanager_discord/run
+    owner: root
+    mode: '0755'
+  tags:
+    - configuration
+- name: Enable Runit service
+  file:
+    state: link
+    src: /etc/sv/alertmanager_discord
+    dest: /var/service/alertmanager_discord
diff --git a/tasks/main.yml b/tasks/main.yml
index 19ca131..e9a017a 100644
--- a/tasks/main.yml
+++ b/tasks/main.yml
@@ -30,11 +30,12 @@
     #!/bin/sh
     set -e
     if ! [ -e "{{prometheus_home_dir}}/{{prometheus_extract_dir}}" ] ; then
-      su - '{{prometheus_user}}' -c 'tar xzf prometheus-{{prometheus_version}}.tgz'
+      su - '{{prometheus_user}}' -c 'cd "{{prometheus_home_dir}}"; tar xzf prometheus-{{prometheus_version}}.tgz'
       exit 50
     fi
   register: res
   changed_when: res is defined and res.rc == 50
+  # failed_when: res is defined and ( res.rc != 50 or res.rc != 0 )
   ignore_errors: true
   notify: Restart Prometheus
 - name: Link prometheus directory
@@ -53,13 +54,34 @@
     - prometheus.yml
 - name: Write prometheus configuration file
   copy:
-    content: '{{prometheus_config | to_nice_yaml}}'
+    content: '{{prometheus_config | to_nice_json}}'
     dest: '{{prometheus_home_dir}}/conf/prometheus.yml'
     owner: '{{prometheus_user}}'
     mode: '0600'
   notify: Restart prometheus
   tags:
     - configuration
+- name: Write prometheus web configuration file
+  copy:
+    content: '{{prometheus_web_config | to_nice_json}}'
+    dest: '{{prometheus_home_dir}}/conf/web_conf.yml'
+    owner: '{{prometheus_user}}'
+    mode: '0600'
+  when: prometheus_web_config is defined
+  notify: Restart prometheus
+  tags:
+    - configuration
+- name: Write other config files
+  copy:
+    content: '{{item.content}}'
+    dest: '{{prometheus_home_dir}}/conf/{{item.filename}}'
+    owner: '{{prometheus_user}}'
+    mode: '0600'
+  loop: '{{prometheus_other_config | dict2items(key_name="filename", value_name="content")}}'
+  when: prometheus_other_config is defined
+  notify: Restart prometheus
+  tags:
+    - configuration
 - name: Create Runit Directory
   file:
     state: directory
@@ -72,6 +94,8 @@
     dest: /etc/sv/prometheus/run
     owner: root
     mode: '0755'
+  tags:
+    - configuration
 - name: Enable Runit service
   file:
     state: link
@@ -86,3 +110,21 @@
   tags:
     - alertmanager
   when: alertmanager
+- name: Include Blackbox Exporter tasks
+  include_tasks:
+    file: blackbox.yml
+    apply:
+      tags:
+        - blackbox
+  tags:
+    - blackbox
+  when: blackbox_exporter
+- name: Include Discord tasks
+  include_tasks:
+    file: discord.yml
+    apply:
+      tags:
+        - discord
+  tags:
+    - discord
+  when: alertmanager_discord
diff --git a/templates/alertmanager.runit b/templates/alertmanager.runit
index e300d47..4c53493 100644
--- a/templates/alertmanager.runit
+++ b/templates/alertmanager.runit
@@ -1,6 +1,6 @@
 #!/bin/sh
+exec 3>>{{alertmanager_log}} 2>&3 1>&3
 exec chpst -u {{prometheus_user}} \
 	{{prometheus_home_dir}}/alertmanager/alertmanager \
 	{{alertmanager_opts | replace('\n', ' ')}} \
-	>> {{alertmanager_log}} 2>&1
 
diff --git a/templates/alertmanager_discord.runit b/templates/alertmanager_discord.runit
new file mode 100644
index 0000000..28f82a8
--- /dev/null
+++ b/templates/alertmanager_discord.runit
@@ -0,0 +1,6 @@
+#!/bin/sh
+exec 3>>{{alertmanager_discord_log}} 2>&3 1>&3
+exec chpst -u {{prometheus_user}} \
+	{{prometheus_home_dir}}/alertmanager_discord \
+	{{alertmanager_discord_opts | replace('\n', ' ')}}
+
diff --git a/templates/blackbox_exporter.runit b/templates/blackbox_exporter.runit
new file mode 100644
index 0000000..115bc7b
--- /dev/null
+++ b/templates/blackbox_exporter.runit
@@ -0,0 +1,7 @@
+#!/bin/sh
+exec 3>>{{blackbox_exporter_log}} 2>&3 1>&3
+exec chpst -u {{prometheus_user}} \
+	{{prometheus_home_dir}}/blackbox_exporter/blackbox_exporter \
+	{{blackbox_exporter_opts | replace('\n', ' ')}} \
+
+
diff --git a/templates/prometheus.runit b/templates/prometheus.runit
index b14e2d1..9866d08 100644
--- a/templates/prometheus.runit
+++ b/templates/prometheus.runit
@@ -1,5 +1,5 @@
 #!/bin/sh
+exec 3>>{{prometheus_log}} 2>&3 1>&3
 exec chpst -u {{prometheus_user}} \
 	{{prometheus_home_dir}}/prometheus/prometheus \
 	{{prometheus_opts | replace('\n', ' ')}} \
-	> {{prometheus_log}} 2>&1