Create configs for monitoring server

.gitignore

@@ -1 +1,2 @@
 ansible.cfg
+roles/cloudalchemy*

README.md

@@ -22,12 +22,11 @@ Please note: Replace <ENVIRONMENT> with either **production** or **staging** in
 
 Requirements: jmespath (deployer host), unzip (target host)
 
-### Grafana
+Adjust inventory and config variables in `environments/monitoring/group_vars/monitoring/main.yml` before prodceeding.
 
-- `ansible-galaxy install cloudalchemy.grafana`
-- `ansible-playbook -i environments/monitoring/inventory.yml playbooks/install_grafana.yml`
-
-### Prometheus
+### Prometheus + Grafana + Alert Manager
 
+- `ansible-galaxy install cloudalchemy.grafana`
 - `ansible-galaxy install cloudalchemy.prometheus`
-- `ansible-playbook -i environments/monitoring/inventory.yml playbooks/install_prometheus.yml`
+- `ansible-galaxy install cloudalchemy.alertmanager`
+- `ansible-playbook -i environments/monitoring/inventory.yml playbooks/install_monitoring.yml`

environments/monitoring/group_vars/monitoring/main.yml

@@ -0,0 +1,52 @@
+domains:
+  prometheus: prometheus.palava.tv
+  grafana: grafana.palava.tv
+  alertmanager: alerts.palava.tv
+
+### PROMETHEUS ###
+
+prometheus_version: latest
+prometheus_web_listen_address: "127.0.0.1:9090"
+prometheus_web_external_url: "https://{{ domains.prometheus }}"
+prometheus_storage_retention: 30d
+prometheus_scrape_jobs:
+  - job_name: "signal-tower"
+    metrics_path: /metrics
+    basic_auth:
+      username: "TODO"
+      password: "TODOTODOTODO"
+    params:
+      module: [http_2xx]
+    static_configs:
+      - targets:
+          - machine.palava.tv
+
+### GRAFANA ###
+
+grafana_version: latest
+grafana_address: 127.0.0.1
+grafana_port: 7000
+grafana_url: "https://{{ domains.grafana }}"
+grafana_security:
+  admin_user: TODO
+  admin_password: "TODO"
+grafana_datasources:
+  - name: prometheus
+    type: prometheus
+    url: "http://{{ prometheus_web_listen_address }}"
+    basicAuth: false
+    basicAuthUser: "TODO"
+    basicAuthPassword: "TODO"
+
+### ALERTMANAGER ###
+
+alertmanager_version: 0.23.0
+alertmanager_web_listen_address: 127.0.0.1:9093
+alertmanager_web_external_url: "https://{{ domains.alertmanager }}"
+#alertmanager_receivers: TODO
+alertmanager_route:
+  group_by: ["alertname", "cluster", "service"]
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 3h
+  receiver: slack

environments/monitoring/inventory.yml

@@ -0,0 +1,3 @@
+monitoring:
+  hosts:
+    128.140.124.42: null

environments/production/group_vars/all/main.yml

@@ -1,31 +1,8 @@
 ---
-
 palava_signaltower_install_dir: /srv/signaltower-production
 palava_signaltower_log_dir: /var/log/signaltower-production
 palava_environment: production
 # Used to configure the TURN server (turn role) to enable TURN in the signaltower role
 # palava_signaltower_turn_secret: SOME_SECRET_KEY
 
 palava_signaltower_autostart: yes
-
-# Prometheus vars
-prometheus_version: 2.22.0
-prometheus_web_listen_address: '127.0.0.1:9090'
-prometheus_scrape_jobs:
-- job_name: 'signal-tower'
-  metrics_path: /metrics
-  params:
-    module: [http_2xx]
-  static_configs:
-    - targets:
-      - localhost:4233
-# Grafana vars
-grafana_security:
-  admin_user: admin
-  admin_password: "admin"
-grafana_datasources:
-  - name: prometheus
-    type: prometheus
-    access: proxy
-    url: 'http://{{ prometheus_web_listen_address }}'
-    basicAuth: false

environments/production/inventory.yml

@@ -1,5 +1,3 @@
----
-
 all:
   hosts:
     157.90.226.126: null

environments/staging/group_vars/all/main.yml

@@ -1,5 +1,4 @@
 ---
-
 palava_signaltower_install_dir: /srv/signaltower-staging
 palava_signaltower_log_dir: /var/log/signaltower-staging
 palava_environment: staging

environments/staging/inventory.yml

@@ -1,5 +1,3 @@
----
-
 all:
   hosts:
     157.90.226.126: null

playbooks/install_monitoring.yml

@@ -0,0 +1,12 @@
+- hosts: monitoring
+  strategy: debug
+  remote_user: root
+  become: yes
+  roles:
+    - name: common
+      include_role:
+        name: common
+        tasks_from: monitoring.yml
+    - cloudalchemy.prometheus
+    - cloudalchemy.grafana
+    - cloudalchemy.alertmanager

roles/common/tasks/monitoring.yml

@@ -0,0 +1,4 @@
+---
+- include: upgrade.yml
+- include: packages-monitoring.yml
+- include: ssh.yml

roles/common/tasks/packages-monitoring.yml

@@ -0,0 +1,18 @@
+- name: Install common packages (monitoring)
+  apt:
+    name:
+      - apt-transport-https
+      - apache2-utils
+      - fail2ban
+      - nginx
+      - unattended-upgrades
+      - vim
+    state: present
+
+- name: Make sure some packages are not installed
+  apt:
+    name:
+      - apache2
+      - cups
+      - telnet
+    state: absent

roles/nginx/tasks/monitoring-todo.yml

@@ -0,0 +1,71 @@
+#jinja2: lstrip_blocks: True
+
+## PROMETHEUS ##
+server {
+  listen 443 ssl http2;
+  server_name {{ prom.domains.prometheus }};
+  root /home/{{ user }}/www/prom/;
+
+  access_log /home/{{ user }}/log/{{ prom.domains.prometheus }}/nginx.access.log;
+  error_log /home/{{ user }}/log/{{ prom.domains.prometheus }}/nginx.error.log;
+
+  include ssl_extra;
+  ssl_certificate /etc/letsencrypt/live/{{ prom.domains.prometheus }}/fullchain.pem;
+  ssl_certificate_key /etc/letsencrypt/live/{{ prom.domains.prometheus }}/privkey.pem;
+
+  location / {
+    auth_basic "{{ stats.basic_auth.message|default('restricted access') }}";
+    auth_basic_user_file /etc/nginx/.htpasswd-stats;
+    proxy_pass http://localhost:9090/;
+  }
+}
+
+## GRAFANA ##
+server {
+  listen 443 ssl http2;
+  server_name {{ prom.domains.grafana }};
+  root /home/{{ user }}/www/prom/;
+
+  access_log /home/{{ user }}/log/{{ prom.domains.grafana }}/nginx.access.log;
+  error_log /home/{{ user }}/log/{{ prom.domains.grafana }}/nginx.error.log;
+
+  include ssl_extra;
+  ssl_certificate /etc/letsencrypt/live/{{ prom.domains.prometheus }}/fullchain.pem;
+  ssl_certificate_key /etc/letsencrypt/live/{{ prom.domains.prometheus }}/privkey.pem;
+
+  location / {
+    auth_basic "{{ stats.basic_auth.message|default('restricted access') }}";
+    auth_basic_user_file /etc/nginx/.htpasswd-stats;
+    proxy_pass http://localhost:7000/;
+  }
+}
+
+## ALERT MANAGER ##
+server {
+  listen 443 ssl http2;
+  server_name {{ prom.domains.alertmanager }};
+  root /home/{{ user }}/www/prom/;
+
+  access_log /home/{{ user }}/log/{{ prom.domains.alertmanager }}/nginx.access.log;
+  error_log /home/{{ user }}/log/{{ prom.domains.alertmanager }}/nginx.error.log;
+
+  include ssl_extra;
+  ssl_certificate /etc/letsencrypt/live/{{ prom.domains.prometheus }}/fullchain.pem;
+  ssl_certificate_key /etc/letsencrypt/live/{{ prom.domains.prometheus }}/privkey.pem;
+
+  location / {
+    auth_basic "{{ stats.basic_auth.message|default('restricted access') }}";
+    auth_basic_user_file /etc/nginx/.htpasswd-stats;
+    proxy_pass http://localhost:9093/;
+  }
+}
+
+## HTTP REDIRECTS ##
+server {
+  listen 80;
+  server_name {{ prom.domains.prometheus }} {{ prom.domains.grafana }} {{ prom.domains.alertmanager }};
+
+  location / {
+    return 301 https://$host$request_uri;
+  }
+}

roles/requirements.yml

@@ -1,5 +1,6 @@
 # from galaxy
 - name: cloudalchemy.prometheus
 - name: cloudalchemy.grafana
+- name: cloudalchemy.alertmanager
 - name: geerlingguy.certbot
-- name: community.general
+- name: community.general