From a74482079a0c2c559ef097c7426bcc8473d31a53 Mon Sep 17 00:00:00 2001 From: Thomas Boni Date: Sun, 6 Sep 2026 11:53:24 +0000 Subject: [PATCH] feat(chart): expose the worker task watchdog as worker.taskMaxRuntime Opening an analysis enumerates every governed project from GitLab, and that enumeration is a single worker task. The worker watchdog kills the process if any one task exceeds JOBS_WORKER_TASK_MAX_RUNTIME, which defaults to 3m in the application. The chart had no way to set it, so every deployment inherited the default. On an estate of a few thousand projects that enumeration runs well past 3m: on our test instance 3708 projects took 80 GitLab calls averaging 4.7s, about 6m20 in total. The worker is killed mid-list, the task is recovered and retried, and the analysis is accepted but never leaves zero tasks. It is a restart loop with no error surfaced to the user. The value renders only on the worker deployment, since only the process started with --worker reads it, and is omitted entirely when set to null so an operator can fall back to the application default. Default 20m gives headroom over the measured time rather than sitting just above it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WkEoFQP5bxWXS5qgkYxmu8 --- charts/plumber/Chart.yaml | 4 ++-- charts/plumber/templates/deployment.yaml | 11 +++++++++++ charts/plumber/values.yaml | 8 ++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/charts/plumber/Chart.yaml b/charts/plumber/Chart.yaml index c1afe89..ed708b1 100644 --- a/charts/plumber/Chart.yaml +++ b/charts/plumber/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: plumber description: Helm chart for Plumber type: application -version: "1.5.0" -appVersion: "1.5.0" +version: "1.5.1" +appVersion: "1.5.1" home: https://github.com/getplumber/platform/ maintainers: - name: devpro diff --git a/charts/plumber/templates/deployment.yaml b/charts/plumber/templates/deployment.yaml index 2820d3c..863386a 100644 --- a/charts/plumber/templates/deployment.yaml +++ b/charts/plumber/templates/deployment.yaml @@ -107,6 +107,17 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.hostIP + {{- if and (eq .name $.Values.worker.name) .taskMaxRuntime }} + {{- /* + Watchdog: the worker exits if any single task runs longer than + this. Opening an analysis enumerates every governed project from + GitLab as one task, so the limit has to exceed that enumeration or + the worker is killed mid-list, the task is retried, and the + analysis never starts. Only the worker reads it. + */}} + - name: JOBS_WORKER_TASK_MAX_RUNTIME + value: "{{ .taskMaxRuntime }}" + {{- end }} {{- if eq .type "backend" }} - name: JOBS_LISTEN_ADDR value: "0.0.0.0" diff --git a/charts/plumber/values.yaml b/charts/plumber/values.yaml index 286b552..d9ead2a 100644 --- a/charts/plumber/values.yaml +++ b/charts/plumber/values.yaml @@ -188,6 +188,14 @@ worker: revisionHistoryLimit: 5 args: - "--worker" + # -- Hard timeout for a single worker task, after which the worker exits and + # restarts (watchdog). Opening an analysis enumerates every governed project + # from GitLab as one task, so this has to be longer than that enumeration + # takes. On an estate of a few thousand projects it runs into several + # minutes, past the application default of 3m, and the symptom is an analysis + # that is accepted but stays at zero tasks while the worker restarts in a + # loop. Raise it further if your GitLab is slow or your estate is very large. + taskMaxRuntime: "20m" livenessEndpoint: "/job/health/alive" readinesspoint: "/job/health/ready" extraEnv: []