From 2bb8f2cbc4b68b9331ed20de5846caa5c7bb7bd9 Mon Sep 17 00:00:00 2001 From: Kalibh Halford Date: Thu, 22 May 2025 08:55:07 +0100 Subject: [PATCH 1/8] MAINT: Move alert manager logs to /var/log Moving alertmanager logs to /var/log as this is more conventional --- .../ansible/roles/alertmanager/files/alertmanager.service | 4 ++-- .../ansible/roles/alertmanager/tasks/main.yml | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/chatops_deployment/ansible/roles/alertmanager/files/alertmanager.service b/chatops_deployment/ansible/roles/alertmanager/files/alertmanager.service index 43c84100..7bd07bd1 100644 --- a/chatops_deployment/ansible/roles/alertmanager/files/alertmanager.service +++ b/chatops_deployment/ansible/roles/alertmanager/files/alertmanager.service @@ -13,8 +13,8 @@ ExecStart=/opt/alertmanager/alertmanager \ --storage.path=/opt/alertmanager/data \ --web.config.file=/opt/alertmanager/web.yml Restart=always -StandardOutput=append:/opt/alertmanager/alertmanager.log -StandardError=append:/opt/alertmanager/alertmanager.log +StandardOutput=append:/var/log/alertmanager/alertmanager.log +StandardError=append:/var/log/alertmanager/alertmanager.log [Install] diff --git a/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml b/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml index dec8051f..c5b6864c 100644 --- a/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml +++ b/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml @@ -79,6 +79,14 @@ loop: - alertmanager.yml.j2 - web.yml.j2 + - +- name: Create Alertmanager log directory + ansible.builtin.file: + path: /var/log/alertmanager + state: directory + owner: alertmanager + group: alertmanager + mode: "0774" - name: Start alertmanager service become: true From d091a6b590865e3f89e2fce21d1d0fd27dba5630 Mon Sep 17 00:00:00 2001 From: Kalibh Halford Date: Thu, 22 May 2025 08:53:29 +0100 Subject: [PATCH 2/8] MAINT: Move prometheus logs to /var/log Moving prometheus logs to /var/log as this is more conventional --- .../ansible/roles/prometheus/files/prometheus.service | 4 ++-- .../ansible/roles/prometheus/tasks/main.yml | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/chatops_deployment/ansible/roles/prometheus/files/prometheus.service b/chatops_deployment/ansible/roles/prometheus/files/prometheus.service index d1d19997..dea64e3a 100644 --- a/chatops_deployment/ansible/roles/prometheus/files/prometheus.service +++ b/chatops_deployment/ansible/roles/prometheus/files/prometheus.service @@ -12,8 +12,8 @@ ExecStart=/opt/prometheus/prometheus \ --storage.tsdb.path=/opt/prometheus/data \ --storage.tsdb.retention.time=30d \ --web.config.file=/opt/prometheus/web.yml -StandardOutput=append:/opt/prometheus/prometheus.log -StandardError=append:/opt/prometheus/prometheus.log +StandardOutput=append:/var/log/prometheus/prometheus.log +StandardError=append:/var/log/prometheus/prometheus.log [Install] WantedBy=multi-user.target diff --git a/chatops_deployment/ansible/roles/prometheus/tasks/main.yml b/chatops_deployment/ansible/roles/prometheus/tasks/main.yml index ab9ab64a..c18eef06 100644 --- a/chatops_deployment/ansible/roles/prometheus/tasks/main.yml +++ b/chatops_deployment/ansible/roles/prometheus/tasks/main.yml @@ -118,6 +118,14 @@ - prometheus.yml.j2 - web.yml.j2 +- name: Create Prometheus log directory + ansible.builtin.file: + path: /var/log/prometheus + state: directory + owner: prometheus + group: prometheus + mode: "0774" + - name: Start prometheus service become: true ansible.builtin.systemd_service: From cd840d1abbbae2b03fef7530f17fd7384b09124e Mon Sep 17 00:00:00 2001 From: Kalibh Halford Date: Thu, 22 May 2025 09:34:02 +0100 Subject: [PATCH 3/8] MAINT: Simplify the Alertmanager install We can remove a lot of tasks as they can be achieved by already used tasks. In the old changes the following tasks run: - Check if the binaries exist (stat) - Download the binaries (get_url) - Extract the binaries (unarchive) - Create the binary directory (file) - Move the extracted binaries into the folder (shell) In the new changes the following tasks run: - Unarchive - Check the downloaded files exists with "creates" - Download the files with "src" as a URL - Extract the files to destination "/tmp" - Copy - Copy the extracted files into the binaries directory - Create that directory if it doesn't exist --- .../ansible/roles/alertmanager/tasks/main.yml | 55 +++++++------------ 1 file changed, 20 insertions(+), 35 deletions(-) diff --git a/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml b/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml index c5b6864c..8d0d21ab 100644 --- a/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml +++ b/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml @@ -22,42 +22,27 @@ group: alertmanager system: true -- name: Check if binaries exist - ansible.builtin.stat: - path: /opt/alertmanager - register: stat_binaries - -- name: Download and extract binaries - when: not stat_binaries.stat.exists - block: - - name: Download alertmanager binaries - ansible.builtin.get_url: - url: > - https://github.com/prometheus/alertmanager/releases/download/v{{ alertmanager_version }}/ - alertmanager-{{ alertmanager_version }}.linux-amd64.tar.gz - dest: /tmp/alertmanager-{{ alertmanager_version }}.linux-amd64.tar.gz - mode: "0774" - - - name: Extract alertmanager - ansible.builtin.unarchive: - src: "/tmp/alertmanager-{{ alertmanager_version }}.linux-amd64.tar.gz" - dest: "/tmp/alertmanager-{{ alertmanager_version }}.linux-amd64" - remote_src: true - creates: "/tmp/alertmanager-{{ alertmanager_version }}.linux-amd64" - - - name: Create alertmanager directory - become: true - ansible.builtin.file: - path: /opt/alertmanager - state: directory - owner: alertmanager - group: alertmanager - mode: "0770" +- name: Download and extract Alertmanager + become: true + ansible.builtin.unarchive: + src: " + https://github.com/prometheus/alertmanager/releases/download/v{{ alertmanager_version }}/\ + alertmanager-{{ alertmanager_version }}.linux-amd64.tar.gz + " + dest: /tmp + remote_src: true + creates: "/tmp/alertmanager-{{ alertmanager_version }}.linux-amd64" + mode: "0774" - - name: Move alertmanager binaries - ansible.builtin.shell: "cp -r /tmp/alertmanager-{{ alertmanager_version }}.linux-amd64/* /opt/alertmanager" - register: _ - changed_when: _.rc == 0 +- name: Move Alertmanager binaries + become: true + ansible.builtin.copy: + src: "/tmp/alertmanager-{{ alertmanager_version }}.linux-amd64/" + dest: "/opt/alertmanager" + mode: preserve + owner: alertmanager + group: alertmanager + remote_src: true - name: Copy alertmanager service file become: true From bfedddb5803f7c17a5f4c536be5216e194747723 Mon Sep 17 00:00:00 2001 From: Kalibh Halford Date: Thu, 22 May 2025 09:34:16 +0100 Subject: [PATCH 4/8] MAINT: Simplify Prometheus install See previous commit --- .../ansible/roles/prometheus/tasks/main.yml | 62 ++++++------------- 1 file changed, 18 insertions(+), 44 deletions(-) diff --git a/chatops_deployment/ansible/roles/prometheus/tasks/main.yml b/chatops_deployment/ansible/roles/prometheus/tasks/main.yml index c18eef06..2b2358ca 100644 --- a/chatops_deployment/ansible/roles/prometheus/tasks/main.yml +++ b/chatops_deployment/ansible/roles/prometheus/tasks/main.yml @@ -22,55 +22,29 @@ group: prometheus system: true -- name: Check if binaries exist - ansible.builtin.stat: - path: /opt/prometheus - register: stat_binaries - -- name: Download and extract binaries - when: not stat_binaries.stat.exists - block: - - name: Download prometheus binaries - ansible.builtin.get_url: - url: > - https://github.com/prometheus/prometheus/releases/download/v{{ prometheus_version }}/ - prometheus-{{ prometheus_version }}.linux-amd64.tar.gz - dest: "/tmp/prometheus-{{ prometheus_version }}.linux-amd64.tar.gz" - owner: ubuntu - group: ubuntu - mode: "0774" - - - name: Extract prometheus - ansible.builtin.unarchive: - src: /tmp/prometheus-{{ prometheus_version }}.linux-amd64.tar.gz - dest: /tmp - remote_src: true - creates: /tmp/prometheus-{{ prometheus_version }}.linux-amd64 - - - name: Create prometheus directory - become: true - ansible.builtin.file: - path: /opt/prometheus - state: directory - owner: prometheus - group: prometheus - mode: "0774" - - - name: Move prometheus binaries - ansible.builtin.shell: "cp -r /tmp/prometheus-{{ prometheus_version }}.linux-amd64/* /opt/prometheus" - register: _ - changed_when: _.rc == 0 +- name: Download and extract Prometheus + become: true + ansible.builtin.unarchive: + src: " + https://github.com/prometheus/prometheus/releases/download/v{{ prometheus_version }}/\ + prometheus-{{ prometheus_version }}.linux-amd64.tar.gz + " + dest: /tmp + remote_src: true + creates: "/tmp/prometheus-{{ prometheus_version }}.linux-amd64" + mode: "0774" -- name: Create prometheus data directory +- name: Move Prometheus binaries become: true - ansible.builtin.file: - path: /var/prometheus/data - state: directory + ansible.builtin.copy: + src: "/tmp/prometheus-{{ prometheus_version }}.linux-amd64/" + dest: "/opt/prometheus" + mode: preserve owner: prometheus group: prometheus - mode: "0774" + remote_src: true -- name: Attach data volume +- name: Attach volume to Prometheus data directory become: true ansible.posix.mount: boot: true From 4735e4fab7a318d5471d320556dcebbe32aa9b8e Mon Sep 17 00:00:00 2001 From: Kalibh Halford Date: Thu, 15 May 2025 15:50:20 +0100 Subject: [PATCH 5/8] DOC: Fix a typo for timing alerts The alert will trigger after 30 seconds not 10. Updated the description to reflect that --- chatops_deployment/ansible/roles/prometheus/files/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chatops_deployment/ansible/roles/prometheus/files/rules.yml b/chatops_deployment/ansible/roles/prometheus/files/rules.yml index 064eab63..5a3e1da8 100644 --- a/chatops_deployment/ansible/roles/prometheus/files/rules.yml +++ b/chatops_deployment/ansible/roles/prometheus/files/rules.yml @@ -8,7 +8,7 @@ groups: severity: critical annotations: summary: "ChatOps container is detected down on {{ $labels.instance }}" - description: "Container has been not running for more than 10 seconds." + description: "Container has been not running for more than 30 seconds." - alert: SystemdServiceDown expr: systemd_unit_state{name=~"grafana-server.service|haproxy.service",state=~"failed|inactive"} == 1 From 6fa4ef3cc857f501efaf3f53dea8a51ba20ae28d Mon Sep 17 00:00:00 2001 From: Kalibh Halford Date: Mon, 2 Jun 2025 10:58:59 +0100 Subject: [PATCH 6/8] MAINT: Reduce permission scope of logs Reducing the permission scope of the alertmanager and prometheus logs. The public do not need read access to the logs. Any user who would need to read the logs should already be in the prometheus/alertmanager group. --- chatops_deployment/ansible/roles/alertmanager/tasks/main.yml | 2 +- chatops_deployment/ansible/roles/prometheus/tasks/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml b/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml index 8d0d21ab..1d6fdef3 100644 --- a/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml +++ b/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml @@ -71,7 +71,7 @@ state: directory owner: alertmanager group: alertmanager - mode: "0774" + mode: "0770" - name: Start alertmanager service become: true diff --git a/chatops_deployment/ansible/roles/prometheus/tasks/main.yml b/chatops_deployment/ansible/roles/prometheus/tasks/main.yml index 2b2358ca..5c19dfec 100644 --- a/chatops_deployment/ansible/roles/prometheus/tasks/main.yml +++ b/chatops_deployment/ansible/roles/prometheus/tasks/main.yml @@ -98,7 +98,7 @@ state: directory owner: prometheus group: prometheus - mode: "0774" + mode: "0770" - name: Start prometheus service become: true From e5bd8a3112e64b4f5b34d74d7f99ce85ee2d8597 Mon Sep 17 00:00:00 2001 From: Kalibh Halford Date: Mon, 2 Jun 2025 11:03:35 +0100 Subject: [PATCH 7/8] MAINT: Change unarchive target check directory When running the playbook for a second time, if the binaries were already removed from `/tmp` (e.g. system reboot) then the binaries will be re-downloaded and extracted even if `/opt/` already exists. We can fix this by checking if `/opt/` exists instead and only run the copy module if the unarchive module made changes. --- chatops_deployment/ansible/roles/alertmanager/tasks/main.yml | 4 +++- chatops_deployment/ansible/roles/prometheus/tasks/main.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml b/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml index 1d6fdef3..b37916ab 100644 --- a/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml +++ b/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml @@ -31,11 +31,13 @@ " dest: /tmp remote_src: true - creates: "/tmp/alertmanager-{{ alertmanager_version }}.linux-amd64" + creates: "/opt/alertmanager" mode: "0774" + register: unarchive_result - name: Move Alertmanager binaries become: true + when: unarchive_result.changed ansible.builtin.copy: src: "/tmp/alertmanager-{{ alertmanager_version }}.linux-amd64/" dest: "/opt/alertmanager" diff --git a/chatops_deployment/ansible/roles/prometheus/tasks/main.yml b/chatops_deployment/ansible/roles/prometheus/tasks/main.yml index 5c19dfec..59631402 100644 --- a/chatops_deployment/ansible/roles/prometheus/tasks/main.yml +++ b/chatops_deployment/ansible/roles/prometheus/tasks/main.yml @@ -31,11 +31,13 @@ " dest: /tmp remote_src: true - creates: "/tmp/prometheus-{{ prometheus_version }}.linux-amd64" + creates: "/opt/prometheus" mode: "0774" + register: unarchive_result - name: Move Prometheus binaries become: true + when: unarchive_result.changed ansible.builtin.copy: src: "/tmp/prometheus-{{ prometheus_version }}.linux-amd64/" dest: "/opt/prometheus" From fcb7a987f18b4fb6d5129df237b3d710e1eef5bb Mon Sep 17 00:00:00 2001 From: Kalibh Halford Date: Mon, 2 Jun 2025 11:37:00 +0100 Subject: [PATCH 8/8] MAINT: Fix linting no-handlers When a task is run based on a previous task causing a change on the target host it should be run as a handler. Moving the handler tasks into handlers to fix the lint warning. https://ansible.readthedocs.io/projects/lint/rules/no-handler/#problematic-code --- .../roles/alertmanager/handlers/main.yml | 23 +++++++++++++++ .../ansible/roles/alertmanager/tasks/main.yml | 29 +++++-------------- .../roles/prometheus/handlers/main.yml | 23 +++++++++++++++ .../ansible/roles/prometheus/tasks/main.yml | 29 ++++++------------- 4 files changed, 63 insertions(+), 41 deletions(-) create mode 100644 chatops_deployment/ansible/roles/alertmanager/handlers/main.yml create mode 100644 chatops_deployment/ansible/roles/prometheus/handlers/main.yml diff --git a/chatops_deployment/ansible/roles/alertmanager/handlers/main.yml b/chatops_deployment/ansible/roles/alertmanager/handlers/main.yml new file mode 100644 index 00000000..2987a2c5 --- /dev/null +++ b/chatops_deployment/ansible/roles/alertmanager/handlers/main.yml @@ -0,0 +1,23 @@ +- name: Move Alertmanager binaries + become: true + ansible.builtin.copy: + src: "/tmp/alertmanager-{{ alertmanager_version }}.linux-amd64/" + dest: "/opt/alertmanager" + mode: preserve + owner: alertmanager + group: alertmanager + remote_src: true + +- name: Start Alertmanager + become: true + ansible.builtin.systemd_service: + name: alertmanager.service + state: started + daemon_reload: true + enabled: true + +- name: Restart Alertmanager + become: true + ansible.builtin.systemd_service: + name: alertmanager.service + state: restarted diff --git a/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml b/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml index b37916ab..15eac35a 100644 --- a/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml +++ b/chatops_deployment/ansible/roles/alertmanager/tasks/main.yml @@ -33,18 +33,9 @@ remote_src: true creates: "/opt/alertmanager" mode: "0774" - register: unarchive_result - -- name: Move Alertmanager binaries - become: true - when: unarchive_result.changed - ansible.builtin.copy: - src: "/tmp/alertmanager-{{ alertmanager_version }}.linux-amd64/" - dest: "/opt/alertmanager" - mode: preserve - owner: alertmanager - group: alertmanager - remote_src: true + notify: + - Move Alertmanager binaries + - Start Alertmanager - name: Copy alertmanager service file become: true @@ -54,6 +45,8 @@ owner: alertmanager group: alertmanager mode: "0774" + notify: + - Restart Alertmanager - name: Template alertmanager config become: true @@ -63,10 +56,12 @@ owner: alertmanager group: alertmanager mode: "0770" + notify: + - Restart Alertmanager loop: - alertmanager.yml.j2 - web.yml.j2 - - + - name: Create Alertmanager log directory ansible.builtin.file: path: /var/log/alertmanager @@ -74,11 +69,3 @@ owner: alertmanager group: alertmanager mode: "0770" - -- name: Start alertmanager service - become: true - ansible.builtin.systemd_service: - name: alertmanager.service - state: restarted - daemon_reload: true - enabled: true diff --git a/chatops_deployment/ansible/roles/prometheus/handlers/main.yml b/chatops_deployment/ansible/roles/prometheus/handlers/main.yml new file mode 100644 index 00000000..89a587fe --- /dev/null +++ b/chatops_deployment/ansible/roles/prometheus/handlers/main.yml @@ -0,0 +1,23 @@ +- name: Move Prometheus binaries + become: true + ansible.builtin.copy: + src: "/tmp/prometheus-{{ prometheus_version }}.linux-amd64/" + dest: "/opt/prometheus" + mode: preserve + owner: prometheus + group: prometheus + remote_src: true + +- name: Start Prometheus + become: true + ansible.builtin.systemd_service: + name: prometheus.service + state: started + daemon_reload: true + enabled: true + +- name: Restart Prometheus + become: true + ansible.builtin.systemd_service: + name: prometheus.service + state: restarted diff --git a/chatops_deployment/ansible/roles/prometheus/tasks/main.yml b/chatops_deployment/ansible/roles/prometheus/tasks/main.yml index 59631402..3f7792b9 100644 --- a/chatops_deployment/ansible/roles/prometheus/tasks/main.yml +++ b/chatops_deployment/ansible/roles/prometheus/tasks/main.yml @@ -33,18 +33,9 @@ remote_src: true creates: "/opt/prometheus" mode: "0774" - register: unarchive_result - -- name: Move Prometheus binaries - become: true - when: unarchive_result.changed - ansible.builtin.copy: - src: "/tmp/prometheus-{{ prometheus_version }}.linux-amd64/" - dest: "/opt/prometheus" - mode: preserve - owner: prometheus - group: prometheus - remote_src: true + notify: + - Move Prometheus binaries + - Start Prometheus - name: Attach volume to Prometheus data directory become: true @@ -72,6 +63,8 @@ owner: prometheus group: prometheus mode: "0774" + notify: + - Start Prometheus - name: Copy prometheus rules file become: true @@ -81,6 +74,8 @@ owner: prometheus group: prometheus mode: "0774" + notify: + - Restart Prometheus - name: Template prometheus config become: true @@ -90,6 +85,8 @@ owner: prometheus group: prometheus mode: "0774" + notify: + - Restart Prometheus loop: - prometheus.yml.j2 - web.yml.j2 @@ -101,11 +98,3 @@ owner: prometheus group: prometheus mode: "0770" - -- name: Start prometheus service - become: true - ansible.builtin.systemd_service: - name: prometheus.service - state: restarted - daemon_reload: true - enabled: true