fix(backtest): scope job sweep to this terminal, stop leaking tester processes, tail the real log

Dockerfile.test

@@ -25,7 +25,7 @@ COPY tests ./tests
 # docker-compose.yml.j2 is here because the compose-generation test renders the
 # REAL template — a stub would assert nothing about what actually ships.
 COPY requirements-api.txt requirements-mcpunifier.txt docker-compose.yml.example docker-compose.yml.j2 run.sh ./
-COPY scripts/config_helper.py scripts/start.bat scripts/check_health.py scripts/healthcheck.sh scripts/verify_binaries.py ./scripts/
+COPY scripts/config_helper.py scripts/start.bat scripts/check_health.py scripts/healthcheck.sh scripts/verify_binaries.py scripts/wickworks-healthcheck.py ./scripts/
 COPY assets/binaries.lock.json ./assets/
 
 ENV PYTHONPATH=/app

docker-compose.yml.example

@@ -36,6 +36,13 @@ services:
   # ONLY from the mt5 container (and from the Windows VM via the dockurr
   # gateway 20.20.20.1:8000). No ports published; nothing else on the
   # docker network can talk to it.
+  #
+  # netns-shared sidecars are NOT re-joined when the owning VM container is
+  # recreated: Docker leaves wickworks running in the old, now-orphaned
+  # network namespace, where its own loopback healthcheck still passes while
+  # the VM can no longer reach it (every TA call then 502s). This healthcheck
+  # detects the orphan and kills wickworks so `restart: unless-stopped`
+  # recreates it into the current mt5 netns. See scripts/wickworks-healthcheck.py.
   wickworks:
     image: psyb0t/wickworks:v0.3.1
     restart: unless-stopped
@@ -44,6 +51,14 @@ services:
       LOG_LEVEL: INFO
       MAX_BARS: "5000"
       MIN_BARS: "50"
+    volumes:
+      - ./scripts/wickworks-healthcheck.py:/wickworks-healthcheck.py:ro
+    healthcheck:
+      test: ["CMD", "python", "/wickworks-healthcheck.py"]
+      interval: 15s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     depends_on:
       - mt5
 

docker-compose.yml.j2

@@ -72,6 +72,20 @@ services:
       LOG_LEVEL: INFO
       MAX_BARS: "5000"
       MIN_BARS: "50"
+    volumes:
+      - ./scripts/wickworks-healthcheck.py:/wickworks-healthcheck.py:ro
+    # netns-shared sidecars are NOT re-joined when the owning VM container is
+    # recreated: Docker leaves wickworks running in the old, now-orphaned
+    # network namespace, where its own loopback healthcheck still passes while
+    # the VM can no longer reach it (every TA call then 502s). This healthcheck
+    # detects the orphan and kills wickworks so `restart: unless-stopped`
+    # recreates it into the current mt5 netns. See scripts/wickworks-healthcheck.py.
+    healthcheck:
+      test: ["CMD", "python", "/wickworks-healthcheck.py"]
+      interval: 15s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     depends_on:
       - {{ vm.service }}
 

mt5api/backtest/handler.py

@@ -32,6 +32,7 @@
 from mt5api.config import (
     ACCOUNT,
     BROKER,
+    INSTANCE,
     LOG_DIR,
     TERMINAL_DIR,
     TERMINAL_PATH,
@@ -254,21 +255,38 @@ def _tail(text, limit=DIAGNOSTIC_TAIL_CHARS):
 
 
 def _tail_terminal_log(lines=20):
+    """Tail of the terminal's most recently written run log.
+
+    Picked by modification time, not by name. The logs are named `<date>.log`,
+    but the directory also holds `metaeditor.log`, which sorts after every one
+    of them ("m" > "2") and never changes — so an alphabetical pick attached a
+    months-old compile tail to every failure message and hid the actual reason
+    the run died.
+    """
     log_dir = os.path.join(TERMINAL_DIR, "logs")
     if not os.path.isdir(log_dir):
         return ""
 
     try:
-        candidates = sorted(
-            file_name for file_name in os.listdir(log_dir) if file_name.endswith(".log")
-        )
+        candidates = [
+            entry
+            for entry in os.scandir(log_dir)
+            if entry.is_file()
+            and entry.name.endswith(".log")
+            and entry.name.lower() != "metaeditor.log"
+        ]
     except OSError:
         return ""
 
     if not candidates:
         return ""
 
-    latest_path = os.path.join(log_dir, candidates[-1])
+    try:
+        newest = max(candidates, key=lambda entry: entry.stat().st_mtime)
+    except OSError:
+        return ""
+
+    latest_path = newest.path
     try:
         with open(latest_path, "r", encoding="utf-16-le", errors="replace") as handle:
             content = handle.read()
@@ -281,22 +299,64 @@ def _tail_terminal_log(lines=20):
     return "\n".join(tail_lines[-lines:])
 
 
-def _terminal_process_alive():
-    """True while a terminal64.exe belonging to THIS terminal directory runs.
+#: The tester runs as terminal64.exe plus one metatester64.exe per agent. The
+#: agents are what hold the localhost ports a later run needs, so a cleanup that
+#: only accounts for terminal64.exe leaves the terminal unusable.
+TESTER_PROCESS_NAMES = frozenset({"terminal64.exe", "metatester64.exe"})
+
+
+def _terminal_processes(names=TESTER_PROCESS_NAMES):
+    """Processes of the given names running from THIS terminal directory.
 
-    Matched by directory rather than by the PID we spawned on purpose: the
-    point of this check is to see the process MT5 started to *replace* the one
-    we launched, which we never get a handle on.
+    Matched by directory rather than by the PID we spawned on purpose: MT5 may
+    replace the process we launched (see _await_self_relaunch), and the agents
+    are never ours to begin with. Sibling terminals live in sibling directories,
+    so this never reaches across to another instance's processes.
     """
     for proc in psutil.process_iter(["name", "exe"]):
         try:
-            if (proc.info.get("name") or "").lower() != "terminal64.exe":
+            if (proc.info.get("name") or "").lower() not in names:
                 continue
             exe = proc.info.get("exe") or ""
             if exe and TERMINAL_DIR.lower() in exe.lower():
-                return True
+                yield proc
+        except (psutil.NoSuchProcess, psutil.AccessDenied):
+            continue
+
+
+def kill_terminal_processes(grace_seconds=10):
+    """Stop this terminal's tester processes. Returns how many were signalled.
+
+    Terminate first, then kill whatever is still standing, so MT5 gets the
+    chance to release its files cleanly before being shot.
+    """
+    victims = list(_terminal_processes())
+    if not victims:
+        return 0
+    for proc in victims:
+        try:
+            proc.terminate()
+        except (psutil.NoSuchProcess, psutil.AccessDenied):
+            continue
+    _, alive = psutil.wait_procs(victims, timeout=grace_seconds)
+    for proc in alive:
+        try:
+            proc.kill()
         except (psutil.NoSuchProcess, psutil.AccessDenied):
             continue
+    if alive:
+        psutil.wait_procs(alive, timeout=grace_seconds)
+    return len(victims)
+
+
+def _terminal_process_alive():
+    """True while a terminal64.exe belonging to THIS terminal directory runs.
+
+    Only terminal64.exe: this answers "is a run still going", and the agents
+    come and go within one.
+    """
+    for _ in _terminal_processes({"terminal64.exe"}):
+        return True
     return False
 
 
@@ -461,6 +521,9 @@ def run_backtest():
         "status": "queued",
         "broker": BROKER,
         "account": ACCOUNT,
+        # Sibling terminals share broker+account and differ only by instance, so
+        # without this the startup sweep cannot tell whose job it is looking at.
+        "instance": INSTANCE,
         "submittedAt": jobs.now_iso(),
         "startedAt": None,
         "finishedAt": None,
@@ -560,10 +623,24 @@ def _execute_job(job_id):
                     )
                 except subprocess.TimeoutExpired:
                     duration = round(time.time() - start_time, 3)
+                    # subprocess.run kills the process it started, but not the
+                    # metatester64.exe agents it spawned, and not a terminal MT5
+                    # relaunched in place of ours. Those keep the terminal's
+                    # localhost agent ports bound, so every later run on this
+                    # terminal dies with "bind error [10048]" until the host is
+                    # rebooted. Clear the whole directory's tester processes.
+                    killed = kill_terminal_processes()
+                    error = f"Backtest timed out after {job['timeoutSeconds']}s"
+                    if killed:
+                        error = f"{error} (killed {killed} leftover tester process(es))"
+                    log.warning(
+                        "backtest timed out broker=%s account=%s job=%s killed=%d",
+                        BROKER, ACCOUNT, job_id, killed,
+                    )
                     jobs.update_job(
                         job_id,
                         status="failed",
-                        error=f"Backtest timed out after {job['timeoutSeconds']}s",
+                        error=error,
                         durationSeconds=duration,
                         finishedAt=jobs.now_iso(),
                     )

mt5api/backtest/jobs.py

@@ -19,9 +19,13 @@
 from datetime import datetime, timezone
 
 from mt5api.config import (
+    ACCOUNT,
     BACKTEST_JOB_DIR,
     BACKTEST_JOB_RETENTION_SECONDS,
     BACKTEST_SWEEP_LOOKBACK_SECONDS,
+    BROKER,
+    INSTANCE,
+    normalize_instance,
 )
 from mt5api.logger import log
 
@@ -157,8 +161,29 @@ def public_payload(job: dict) -> dict:
     return payload
 
 
+def owns_job(job: dict) -> bool:
+    """Is this job ours, rather than a sibling terminal's?
+
+    Every backtest API on a host shares one job directory, so the sweep sees
+    every terminal's jobs. A job that does not name a terminal is treated as
+    ours: that is what a single-terminal install writes, and what every job
+    written before this field existed looks like, so the previous behaviour is
+    preserved exactly where there is nothing to distinguish.
+    """
+    broker = job.get("broker")
+    if broker is not None and broker != BROKER:
+        return False
+    account = job.get("account")
+    if account is not None and account != ACCOUNT:
+        return False
+    instance = job.get("instance")
+    if instance is not None and normalize_instance(instance) != normalize_instance(INSTANCE):
+        return False
+    return True
+
+
 def sweep_orphans(lookback_seconds: int | None = None) -> int:
-    """Mark any queued/running jobs on disk as failed.
+    """Mark this terminal's queued/running jobs on disk as failed.
 
     Called at API startup. Only state files touched within the last
     ``lookback_seconds`` are considered: a live job rewrites its file on every
@@ -167,6 +192,11 @@ def sweep_orphans(lookback_seconds: int | None = None) -> int:
     parsed tens of thousands of files on every boot — and because every backtest
     API on a VM shares the same job directory, that full scan repeated once per
     process. Returns the number of jobs swept.
+
+    Only jobs belonging to THIS terminal are swept. The directory is shared, so
+    sweeping everything meant restarting one terminal's API failed every other
+    terminal's in-flight backtest with "API restarted before completion" —
+    silently, and while those runs went on to finish perfectly well.
     """
     if lookback_seconds is None:
         lookback_seconds = SWEEP_LOOKBACK_SECONDS
@@ -198,6 +228,8 @@ def sweep_orphans(lookback_seconds: int | None = None) -> int:
             continue
         if job.get("status") not in ACTIVE_STATUSES:
             continue
+        if not owns_job(job):
+            continue
         job["status"] = "failed"
         job["error"] = "API restarted before completion"
         job["finishedAt"] = now_iso()

mt5api/main.py

@@ -140,6 +140,29 @@ def _run_backtest_startup_cleanup():
         swept = backtest_jobs.sweep_orphans()
         if swept:
             log.warning("Backtest sweep marked %d orphaned job(s) as failed.", swept)
+
+        # Any tester process still running for this terminal is a leftover: we
+        # are the only thing that launches one here, and we have only just
+        # started. It holds this terminal's localhost agent ports, so leaving it
+        # makes every subsequent run fail with "bind error [10048]" until the
+        # host is rebooted.
+        #
+        # Deliberately not conditional on the sweep having found anything. A run
+        # whose state file predates the sweep lookback — an API killed while a
+        # long test was live — is never swept, and gating the kill on `swept`
+        # would leave exactly that process holding the ports.
+        #
+        # Backtest mode only: this cleanup thread runs in every mode, and in
+        # live mode the terminal is meant to stay up.
+        if MODE == "backtest":
+            from mt5api.backtest.handler import kill_terminal_processes
+
+            killed = kill_terminal_processes()
+            if killed:
+                log.warning(
+                    "Backtest startup killed %d leftover tester process(es).", killed
+                )
+
         pruned = backtest_jobs.prune_old_jobs()
         if pruned:
             log.info("Backtest retention retired %d old job(s).", pruned)