Prometheus Alertmanager Fixes

chatops_deployment/ansible/roles/alertmanager/files/alertmanager.service

@@ -13,8 +13,8 @@ ExecStart=/opt/alertmanager/alertmanager \
   --storage.path=/opt/alertmanager/data \
   --web.config.file=/opt/alertmanager/web.yml
 Restart=always
-StandardOutput=append:/opt/alertmanager/alertmanager.log
-StandardError=append:/opt/alertmanager/alertmanager.log
+StandardOutput=append:/var/log/alertmanager/alertmanager.log
+StandardError=append:/var/log/alertmanager/alertmanager.log
 
 
 [Install]

chatops_deployment/ansible/roles/alertmanager/handlers/main.yml

@@ -0,0 +1,23 @@
+- name: Move Alertmanager binaries
+  become: true
+  ansible.builtin.copy:
+    src: "/tmp/alertmanager-{{ alertmanager_version }}.linux-amd64/"
+    dest: "/opt/alertmanager"
+    mode: preserve
+    owner: alertmanager
+    group: alertmanager
+    remote_src: true
+
+- name: Start Alertmanager
+  become: true
+  ansible.builtin.systemd_service:
+    name: alertmanager.service
+    state: started
+    daemon_reload: true
+    enabled: true
+
+- name: Restart Alertmanager
+  become: true
+  ansible.builtin.systemd_service:
+    name: alertmanager.service
+    state: restarted

chatops_deployment/ansible/roles/alertmanager/tasks/main.yml

@@ -22,42 +22,20 @@
     group: alertmanager
     system: true
 
-- name: Check if binaries exist
-  ansible.builtin.stat:
-    path: /opt/alertmanager
-  register: stat_binaries
-
-- name: Download and extract binaries
-  when: not stat_binaries.stat.exists
-  block:
-    - name: Download alertmanager binaries
-      ansible.builtin.get_url:
-        url: >
-          https://github.com/prometheus/alertmanager/releases/download/v{{ alertmanager_version }}/
-          alertmanager-{{ alertmanager_version }}.linux-amd64.tar.gz
-        dest: /tmp/alertmanager-{{ alertmanager_version }}.linux-amd64.tar.gz
-        mode: "0774"
-
-    - name: Extract alertmanager
-      ansible.builtin.unarchive:
-        src: "/tmp/alertmanager-{{ alertmanager_version }}.linux-amd64.tar.gz"
-        dest: "/tmp/alertmanager-{{ alertmanager_version }}.linux-amd64"
-        remote_src: true
-        creates: "/tmp/alertmanager-{{ alertmanager_version }}.linux-amd64"
-
-    - name: Create alertmanager directory
-      become: true
-      ansible.builtin.file:
-        path: /opt/alertmanager
-        state: directory
-        owner: alertmanager
-        group: alertmanager
-        mode: "0770"
-
-    - name: Move alertmanager binaries
-      ansible.builtin.shell: "cp -r /tmp/alertmanager-{{ alertmanager_version }}.linux-amd64/* /opt/alertmanager"
-      register: _
-      changed_when: _.rc == 0
+- name: Download and extract Alertmanager
+  become: true
+  ansible.builtin.unarchive:
+    src: "
+      https://github.com/prometheus/alertmanager/releases/download/v{{ alertmanager_version }}/\
+      alertmanager-{{ alertmanager_version }}.linux-amd64.tar.gz
+      "
+    dest: /tmp
+    remote_src: true
+    creates: "/opt/alertmanager"
+    mode: "0774"
+    notify:
+      - Move Alertmanager binaries
+      - Start Alertmanager
 
 - name: Copy alertmanager service file
   become: true
@@ -67,6 +45,8 @@
     owner: alertmanager
     group: alertmanager
     mode: "0774"
+    notify:
+      - Restart Alertmanager
 
 - name: Template alertmanager config
   become: true
@@ -76,14 +56,16 @@
     owner: alertmanager
     group: alertmanager
     mode: "0770"
+    notify:
+      - Restart Alertmanager
   loop:
     - alertmanager.yml.j2
     - web.yml.j2
 
-- name: Start alertmanager service
-  become: true
-  ansible.builtin.systemd_service:
-    name: alertmanager.service
-    state: restarted
-    daemon_reload: true
-    enabled: true
+- name: Create Alertmanager log directory
+  ansible.builtin.file:
+    path: /var/log/alertmanager
+    state: directory
+    owner: alertmanager
+    group: alertmanager
+    mode: "0770"

chatops_deployment/ansible/roles/prometheus/files/prometheus.service

@@ -12,8 +12,8 @@ ExecStart=/opt/prometheus/prometheus \
   --storage.tsdb.path=/opt/prometheus/data \
   --storage.tsdb.retention.time=30d \
   --web.config.file=/opt/prometheus/web.yml
-StandardOutput=append:/opt/prometheus/prometheus.log
-StandardError=append:/opt/prometheus/prometheus.log
+StandardOutput=append:/var/log/prometheus/prometheus.log
+StandardError=append:/var/log/prometheus/prometheus.log
 
 [Install]
 WantedBy=multi-user.target

chatops_deployment/ansible/roles/prometheus/files/rules.yml

@@ -8,7 +8,7 @@ groups:
           severity: critical
         annotations:
           summary: "ChatOps container is detected down on {{ $labels.instance }}"
-          description: "Container has been not running for more than 10 seconds."
+          description: "Container has been not running for more than 30 seconds."
 
       - alert: SystemdServiceDown
         expr: systemd_unit_state{name=~"grafana-server.service|haproxy.service",state=~"failed|inactive"} == 1

chatops_deployment/ansible/roles/prometheus/handlers/main.yml

@@ -0,0 +1,23 @@
+- name: Move Prometheus binaries
+  become: true
+  ansible.builtin.copy:
+    src: "/tmp/prometheus-{{ prometheus_version }}.linux-amd64/"
+    dest: "/opt/prometheus"
+    mode: preserve
+    owner: prometheus
+    group: prometheus
+    remote_src: true
+
+- name: Start Prometheus
+  become: true
+  ansible.builtin.systemd_service:
+    name: prometheus.service
+    state: started
+    daemon_reload: true
+    enabled: true
+
+- name: Restart Prometheus
+  become: true
+  ansible.builtin.systemd_service:
+    name: prometheus.service
+    state: restarted

chatops_deployment/ansible/roles/prometheus/tasks/main.yml

@@ -22,55 +22,22 @@
     group: prometheus
     system: true
 
-- name: Check if binaries exist
-  ansible.builtin.stat:
-    path: /opt/prometheus
-  register: stat_binaries
-
-- name: Download and extract binaries
-  when: not stat_binaries.stat.exists
-  block:
-    - name: Download prometheus binaries
-      ansible.builtin.get_url:
-        url: >
-          https://github.com/prometheus/prometheus/releases/download/v{{ prometheus_version }}/
-          prometheus-{{ prometheus_version }}.linux-amd64.tar.gz
-        dest: "/tmp/prometheus-{{ prometheus_version }}.linux-amd64.tar.gz"
-        owner: ubuntu
-        group: ubuntu
-        mode: "0774"
-
-    - name: Extract prometheus
-      ansible.builtin.unarchive:
-        src: /tmp/prometheus-{{ prometheus_version }}.linux-amd64.tar.gz
-        dest: /tmp
-        remote_src: true
-        creates: /tmp/prometheus-{{ prometheus_version }}.linux-amd64
-
-    - name: Create prometheus directory
-      become: true
-      ansible.builtin.file:
-        path: /opt/prometheus
-        state: directory
-        owner: prometheus
-        group: prometheus
-        mode: "0774"
-
-    - name: Move prometheus binaries
-      ansible.builtin.shell: "cp -r /tmp/prometheus-{{ prometheus_version }}.linux-amd64/* /opt/prometheus"
-      register: _
-      changed_when: _.rc == 0
-
-- name: Create prometheus data directory
+- name: Download and extract Prometheus
   become: true
-  ansible.builtin.file:
-    path: /var/prometheus/data
-    state: directory
-    owner: prometheus
-    group: prometheus
+  ansible.builtin.unarchive:
+    src: "
+      https://github.com/prometheus/prometheus/releases/download/v{{ prometheus_version }}/\
+      prometheus-{{ prometheus_version }}.linux-amd64.tar.gz
+    "
+    dest: /tmp
+    remote_src: true
+    creates: "/opt/prometheus"
     mode: "0774"
+    notify:
+      - Move Prometheus binaries
+      - Start Prometheus
 
-- name: Attach data volume
+- name: Attach volume to Prometheus data directory
   become: true
   ansible.posix.mount:
     boot: true
@@ -96,6 +63,8 @@
     owner: prometheus
     group: prometheus
     mode: "0774"
+    notify:
+      - Start Prometheus
 
 - name: Copy prometheus rules file
   become: true
@@ -105,6 +74,8 @@
     owner: prometheus
     group: prometheus
     mode: "0774"
+    notify:
+      - Restart Prometheus
 
 - name: Template prometheus config
   become: true
@@ -114,14 +85,16 @@
     owner: prometheus
     group: prometheus
     mode: "0774"
+    notify:
+      - Restart Prometheus
   loop:
     - prometheus.yml.j2
     - web.yml.j2
 
-- name: Start prometheus service
-  become: true
-  ansible.builtin.systemd_service:
-    name: prometheus.service
-    state: restarted
-    daemon_reload: true
-    enabled: true
+- name: Create Prometheus log directory
+  ansible.builtin.file:
+    path: /var/log/prometheus
+    state: directory
+    owner: prometheus
+    group: prometheus
+    mode: "0770"