From c6881ece5933a6078a3135a827261ca3bbd2e8a6 Mon Sep 17 00:00:00 2001 From: DK09876 Date: Thu, 11 Jun 2026 12:06:14 -0700 Subject: [PATCH 1/7] feat(zed): threads.db reader + instructions-file writer (validated vs real Zed) Core of the Zed integration, both validated against a real Zed install: - threads_db.py: reads Zed's SQLite threads.db (all 4 on-disk formats, zstd streaming frames, folder_paths->project). Caught/fixed a real zstd frame bug that synthetic fixtures had masked. - rules_file.py: writes a fenced HINDSIGHT memory block into the instruction file Zed actually reads (.rules / AGENTS.md / ...), without hijacking a user's existing file. Verified Zed auto-injects it into every conversation. 27 unit tests; reader additionally verified against a live threads.db. --- .../zed/hindsight_zed/__init__.py | 0 .../zed/hindsight_zed/rules_file.py | 126 ++++++++ .../zed/hindsight_zed/threads_db.py | 305 ++++++++++++++++++ .../zed/scripts/validate_real_zed.py | 64 ++++ hindsight-integrations/zed/tests/__init__.py | 0 .../zed/tests/test_rules_file.py | 111 +++++++ .../zed/tests/test_threads_db.py | 235 ++++++++++++++ 7 files changed, 841 insertions(+) create mode 100644 hindsight-integrations/zed/hindsight_zed/__init__.py create mode 100644 hindsight-integrations/zed/hindsight_zed/rules_file.py create mode 100644 hindsight-integrations/zed/hindsight_zed/threads_db.py create mode 100644 hindsight-integrations/zed/scripts/validate_real_zed.py create mode 100644 hindsight-integrations/zed/tests/__init__.py create mode 100644 hindsight-integrations/zed/tests/test_rules_file.py create mode 100644 hindsight-integrations/zed/tests/test_threads_db.py diff --git a/hindsight-integrations/zed/hindsight_zed/__init__.py b/hindsight-integrations/zed/hindsight_zed/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/hindsight-integrations/zed/hindsight_zed/rules_file.py b/hindsight-integrations/zed/hindsight_zed/rules_file.py new file mode 100644 index 000000000..b93f222a8 --- /dev/null +++ b/hindsight-integrations/zed/hindsight_zed/rules_file.py @@ -0,0 +1,126 @@ +"""Manage Hindsight's recalled-memory block inside a project's Zed instructions file. + +Zed always includes a project's instructions file in every agent conversation, +but it reads **only the first matching file** from a fixed priority list. So we +must not blindly create ``.rules`` — if a project already has an ``AGENTS.md`` +(or ``CLAUDE.md``, ``.cursorrules``, …) that the user maintains, creating a +higher-priority ``.rules`` would silently suppress it. + +Instead we find the instruction file Zed will *actually* read (the first that +exists), and write our memories into a fenced ```` … +```` block inside it, leaving the rest untouched. Only if +the project has no instruction file at all do we create ``.rules``. + +Priority order verified against zed.dev/docs/ai/instructions. +""" + +from pathlib import Path +from typing import Optional + +# Zed's project instruction files, highest priority first. Zed uses the first +# one that exists; ours must target that same file so the block is actually read. +INSTRUCTION_FILES = ( + ".rules", + ".cursorrules", + ".windsurfrules", + ".clinerules", + ".github/copilot-instructions.md", + "AGENT.md", + "AGENTS.md", + "CLAUDE.md", + "GEMINI.md", +) + +# Fallback file created when a project has no instruction file yet. +DEFAULT_INSTRUCTION_FILE = ".rules" + +BEGIN_MARKER = "" +END_MARKER = "" + + +def resolve_instruction_file(project: Path) -> Path: + """Return the instruction file Zed will read for *project*. + + The first existing file in priority order; if none exists, the path where we + should create our own (``.rules``). The returned path may not exist yet. + """ + for name in INSTRUCTION_FILES: + candidate = project / name + if candidate.is_file(): + return candidate + return project / DEFAULT_INSTRUCTION_FILE + + +def _strip_block(text: str) -> str: + """Remove an existing HINDSIGHT block (and its surrounding blank lines).""" + start = text.find(BEGIN_MARKER) + if start == -1: + return text + end = text.find(END_MARKER, start) + if end == -1: + # Malformed (begin without end) — drop from the marker onward. + return text[:start].rstrip() + "\n" + end += len(END_MARKER) + before = text[:start].rstrip() + after = text[end:].lstrip() + if before and after: + return f"{before}\n\n{after}" + return (before or after).rstrip() + ("\n" if (before or after) else "") + + +def render_block(memory_text: str) -> str: + """Render the fenced HINDSIGHT block for *memory_text* (no trailing newline).""" + body = memory_text.strip() + return f"{BEGIN_MARKER}\n{body}\n{END_MARKER}" + + +def write_memory_block(project: Path, memory_text: str, *, preamble: Optional[str] = None) -> Path: + """Write/replace Hindsight's memory block in *project*'s instruction file. + + Preserves any user-authored content in the file and only rewrites our fenced + block. Returns the path written. An empty ``memory_text`` removes the block + (see :func:`clear_memory_block`) so stale memory never lingers. + """ + if not memory_text.strip(): + return clear_memory_block(project) + + target = resolve_instruction_file(project) + existing = target.read_text(encoding="utf-8") if target.is_file() else "" + base = _strip_block(existing).rstrip() + + block_body = memory_text.strip() + if preamble: + block_body = f"{preamble.strip()}\n\n{block_body}" + block = render_block(block_body) + + # Our block goes at the top so memories lead the instructions, with the + # user's existing content following. + if base: + new_text = f"{block}\n\n{base}\n" + else: + new_text = f"{block}\n" + + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(new_text, encoding="utf-8") + return target + + +def clear_memory_block(project: Path) -> Path: + """Remove Hindsight's block from *project*'s instruction file, if present. + + Leaves the rest of the file intact. If removing the block empties a file we + created (``.rules`` with nothing but our block), the file is deleted. + """ + target = resolve_instruction_file(project) + if not target.is_file(): + return target + existing = target.read_text(encoding="utf-8") + if BEGIN_MARKER not in existing: + return target + stripped = _strip_block(existing).strip() + if not stripped and target.name == DEFAULT_INSTRUCTION_FILE: + # The file held only our block and we created it — remove it entirely. + target.unlink() + return target + target.write_text((stripped + "\n") if stripped else "", encoding="utf-8") + return target diff --git a/hindsight-integrations/zed/hindsight_zed/threads_db.py b/hindsight-integrations/zed/hindsight_zed/threads_db.py new file mode 100644 index 000000000..84417ad5e --- /dev/null +++ b/hindsight-integrations/zed/hindsight_zed/threads_db.py @@ -0,0 +1,305 @@ +"""Reader for Zed's agent thread database (``threads.db``). + +Zed stores every AI-assistant conversation in a SQLite database. This module +opens that database read-only, decompresses each thread, and extracts a plain +``[role] text`` transcript plus the project paths the thread belongs to — so a +background process can retain finished conversations into Hindsight. + +The on-disk format has gone through several revisions and Zed only rewrites a +row to the current version when that thread is next saved, so a live database +can hold a mix of all of them at once. We parse, in order of the top-level +``version`` field: + + - ``"0.3.0"`` (current): messages are externally-tagged ``{"User": {...}}`` / + ``{"Agent": {...}}`` objects; the role is implied by the variant key, and + text lives in a ``content`` array of ``{"Text": "..."}`` blocks. A unit + ``Resume`` message serializes as the bare string ``"Resume"`` rather than an + object, so message elements may be either strings or dicts. + - ``"0.2.0"`` / ``"0.1.0"`` (legacy): messages have an explicit lowercase + ``role`` and a ``segments`` array of ``{"type": "text", "text": "..."}``. + - no ``version`` field (oldest legacy): messages have an explicit ``role`` and + a flat ``text`` string. + +Source of truth: zed-industries/zed ``crates/agent/src/db.rs``, +``legacy_thread.rs``, and ``thread.rs`` (verified against ``main``). +""" + +import json +import os +import sqlite3 +import sys +import zlib +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Optional + +try: + import zstandard # type: ignore +except ImportError: # pragma: no cover - resolved at import time, exercised in tests via fallback + zstandard = None + + +# ── Thread / message models ────────────────────────────────────────────────── + + +@dataclass +class ThreadMessage: + """A single turn extracted from a Zed thread.""" + + role: str # "user" | "assistant" | "system" + text: str + + +@dataclass +class ZedThread: + """A parsed Zed conversation thread.""" + + id: str + title: str + updated_at: str + messages: list[ThreadMessage] = field(default_factory=list) + # Absolute project paths this thread was opened against (from the + # ``folder_paths`` column). Used to map a thread to its per-project bank. + folder_paths: list[str] = field(default_factory=list) + + +# ── Database location ───────────────────────────────────────────────────────── + + +def default_threads_db_path() -> Path: + """Return the platform-default path to Zed's ``threads.db``. + + Mirrors Zed's ``paths::data_dir().join("threads").join("threads.db")``. + """ + override = os.environ.get("ZED_THREADS_DB") + if override: + return Path(override) + + home = Path.home() + if sys.platform == "darwin": + return home / "Library" / "Application Support" / "Zed" / "threads" / "threads.db" + if sys.platform == "win32": + base = os.environ.get("LOCALAPPDATA") + root = Path(base) if base else home / "AppData" / "Local" + return root / "Zed" / "threads" / "threads.db" + # Linux / *nix — XDG_DATA_HOME or ~/.local/share, lowercase "zed". + xdg = os.environ.get("XDG_DATA_HOME") + root = Path(xdg) if xdg else home / ".local" / "share" + return root / "zed" / "threads" / "threads.db" + + +# ── Blob decoding ───────────────────────────────────────────────────────────── + + +def _decompress(data: bytes, data_type: str) -> str: + """Decode a thread ``data`` blob into its inner JSON string. + + ``data_type`` is ``"zstd"`` (current writes) or ``"json"`` (uncompressed). + """ + if data_type == "json": + return data.decode("utf-8") + if data_type == "zstd": + if zstandard is not None: + # Stream-decode rather than ZstdDecompressor.decompress(): Zed writes + # frames via zstd::encode_all, whose header does NOT declare the + # decompressed content size, and the one-shot decompress() refuses + # such frames ("could not determine content size in frame header"). + # The streaming reader has no such requirement. + import io + + with zstandard.ZstdDecompressor().stream_reader(io.BytesIO(data)) as reader: + return reader.read().decode("utf-8") + # Fallback so the package has no hard runtime dep: a zstd frame can be + # decoded by zlib only if it is not actually zstd. We never expect this + # path in practice, but raise a clear error rather than silently fail. + raise RuntimeError( + "thread is zstd-compressed but the 'zstandard' package is not installed" + ) + raise ValueError(f"unknown thread data_type: {data_type!r}") + + +# ── Message extraction (per version) ────────────────────────────────────────── + + +def _text_from_user_content(content: Any) -> str: + """Join the text of a current-format ``User`` message ``content`` array.""" + parts: list[str] = [] + for block in content or []: + if not isinstance(block, dict): + continue + if "Text" in block and isinstance(block["Text"], str): + parts.append(block["Text"]) + elif "Mention" in block and isinstance(block["Mention"], dict): + mention_text = block["Mention"].get("content") + if isinstance(mention_text, str): + parts.append(mention_text) + # "Image" blocks carry no text — skipped. + return "\n".join(p for p in parts if p) + + +def _text_from_agent_content(content: Any) -> str: + """Join the text of a current-format ``Agent`` message ``content`` array. + + Only plain ``Text`` blocks are kept — ``Thinking``, ``RedactedThinking`` and + ``ToolUse`` are model-internal and excluded from the retained transcript. + """ + parts: list[str] = [] + for block in content or []: + if isinstance(block, dict) and "Text" in block and isinstance(block["Text"], str): + parts.append(block["Text"]) + return "\n".join(p for p in parts if p) + + +def _messages_from_current(messages: Any) -> list[ThreadMessage]: + """Parse the current (``0.3.0``) externally-tagged message array.""" + out: list[ThreadMessage] = [] + for msg in messages or []: + # Unit variants (e.g. ``Resume``) serialize as a bare string. + if isinstance(msg, str): + continue + if not isinstance(msg, dict): + continue + if "User" in msg and isinstance(msg["User"], dict): + text = _text_from_user_content(msg["User"].get("content")) + if text.strip(): + out.append(ThreadMessage(role="user", text=text)) + elif "Agent" in msg and isinstance(msg["Agent"], dict): + text = _text_from_agent_content(msg["Agent"].get("content")) + if text.strip(): + out.append(ThreadMessage(role="assistant", text=text)) + # "Compaction" / "Resume" carry no user-facing transcript text. + return out + + +def _text_from_segments(segments: Any) -> str: + """Join the text of legacy ``segments`` (``0.1.0`` / ``0.2.0``).""" + parts: list[str] = [] + for seg in segments or []: + if isinstance(seg, dict) and seg.get("type") == "text": + text = seg.get("text") + if isinstance(text, str): + parts.append(text) + return "\n".join(p for p in parts if p) + + +def _messages_from_legacy(messages: Any) -> list[ThreadMessage]: + """Parse legacy messages that carry an explicit ``role``. + + Handles both the ``segments`` array (``0.1.0`` / ``0.2.0``) and the oldest + flat ``text`` field (no ``version``). ``system`` messages are dropped to + match Zed's own upgrade behaviour. + """ + out: list[ThreadMessage] = [] + for msg in messages or []: + if not isinstance(msg, dict): + continue + role = msg.get("role") + if role not in ("user", "assistant"): + continue # drop "system" and anything unexpected + if "segments" in msg: + text = _text_from_segments(msg.get("segments")) + else: + raw = msg.get("text") + text = raw if isinstance(raw, str) else "" + if text.strip(): + out.append(ThreadMessage(role=role, text=text)) + return out + + +def parse_thread_json(raw: str) -> Optional[list[ThreadMessage]]: + """Parse a decompressed thread JSON string into transcript messages. + + Returns ``None`` if the version is unrecognized (so the caller can skip it). + """ + try: + doc = json.loads(raw) + except (json.JSONDecodeError, ValueError): + return None + if not isinstance(doc, dict): + return None + + version = doc.get("version") + messages = doc.get("messages") + if version == "0.3.0": + return _messages_from_current(messages) + if version in ("0.2.0", "0.1.0") or version is None: + return _messages_from_legacy(messages) + # Unknown future version — skip rather than guess. + return None + + +def _thread_title(doc: dict, column_summary: str) -> str: + """Resolve a thread title, preferring the JSON over the column.""" + for key in ("title", "summary"): + val = doc.get(key) + if isinstance(val, str) and val.strip(): + return val + return column_summary or "Untitled" + + +def _folder_paths(raw: Optional[str]) -> list[str]: + """Parse the ``folder_paths`` column (a JSON-serialized list of paths).""" + if not raw: + return [] + try: + parsed = json.loads(raw) + except (json.JSONDecodeError, ValueError): + return [] + if isinstance(parsed, list): + return [p for p in parsed if isinstance(p, str)] + return [] + + +# ── Public API ──────────────────────────────────────────────────────────────── + + +def read_threads(db_path: Path, since: Optional[str] = None) -> list[ZedThread]: + """Read and parse all threads from ``db_path``. + + Opens the database read-only so it is safe to run alongside a live Zed. + When ``since`` (an RFC3339 string) is given, only threads with a strictly + greater ``updated_at`` are returned — the cheap way to poll for new activity. + """ + if not db_path.exists(): + return [] + + uri = f"file:{db_path}?mode=ro&immutable=1" + conn = sqlite3.connect(uri, uri=True) + try: + cols = {row[1] for row in conn.execute("PRAGMA table_info(threads)")} + has_folders = "folder_paths" in cols + select = "SELECT id, summary, updated_at, data_type, data" + ( + ", folder_paths" if has_folders else "" + ) + " FROM threads" + params: tuple = () + if since is not None: + select += " WHERE updated_at > ?" + params = (since,) + rows = conn.execute(select, params).fetchall() + finally: + conn.close() + + threads: list[ZedThread] = [] + for row in rows: + thread_id, summary, updated_at, data_type, data = row[0], row[1], row[2], row[3], row[4] + folder_raw = row[5] if has_folders and len(row) > 5 else None + try: + raw = _decompress(data, data_type) + doc = json.loads(raw) + except (RuntimeError, ValueError, json.JSONDecodeError): + continue + if not isinstance(doc, dict): + continue + messages = parse_thread_json(raw) + if messages is None: + continue + threads.append( + ZedThread( + id=str(thread_id), + title=_thread_title(doc, summary), + updated_at=str(updated_at), + messages=messages, + folder_paths=_folder_paths(folder_raw), + ) + ) + return threads diff --git a/hindsight-integrations/zed/scripts/validate_real_zed.py b/hindsight-integrations/zed/scripts/validate_real_zed.py new file mode 100644 index 000000000..b1adabb78 --- /dev/null +++ b/hindsight-integrations/zed/scripts/validate_real_zed.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Validate the Zed threads.db reader against a *real* Zed database. + +Run this after having at least one AI conversation in Zed. It opens your actual +threads.db, parses every thread with the same reader the integration uses, and +prints what it extracted — so you can eyeball whether the parsed transcript +matches the conversation you actually had. + +A healthy result: thread count > 0, titles you recognize, and the last thread's +transcript reading back the messages you sent/received. If it finds 0 threads, +0 messages, or garbled text, the on-disk format differs from what the reader +expects and the reader needs adjusting. + + python scripts/validate_real_zed.py + python scripts/validate_real_zed.py /path/to/threads.db # explicit path +""" + +import sys +from pathlib import Path + +# Make the package importable when run from the integration directory. +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from hindsight_zed.threads_db import default_threads_db_path, read_threads # noqa: E402 + + +def main() -> int: + db = Path(sys.argv[1]) if len(sys.argv) > 1 else default_threads_db_path() + print(f"threads.db: {db}") + if not db.exists(): + print(" ✗ not found — open Zed, have one AI conversation, then re-run.") + return 1 + + threads = read_threads(db) + print(f" parsed {len(threads)} thread(s)\n") + if not threads: + print(" ✗ 0 threads parsed. Either no conversations yet, or the on-disk") + print(" format differs from what the reader expects (a real bug to fix).") + return 1 + + threads.sort(key=lambda t: t.updated_at, reverse=True) + for t in threads[:5]: + print(f"- {t.title!r} ({len(t.messages)} msgs, updated {t.updated_at})") + if t.folder_paths: + print(f" project: {t.folder_paths}") + + last = threads[0] + print(f"\n=== most recent thread transcript: {last.title!r} ===") + if not last.messages: + print(" ✗ thread parsed but 0 messages extracted — format mismatch in the") + print(" messages array. Compare against a raw dump (see step B4).") + return 1 + for m in last.messages: + snippet = m.text.replace("\n", " ") + if len(snippet) > 200: + snippet = snippet[:200] + "…" + print(f" [{m.role}] {snippet}") + + print("\n✓ Reader works against your real Zed database.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/hindsight-integrations/zed/tests/__init__.py b/hindsight-integrations/zed/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/hindsight-integrations/zed/tests/test_rules_file.py b/hindsight-integrations/zed/tests/test_rules_file.py new file mode 100644 index 000000000..c468e1d65 --- /dev/null +++ b/hindsight-integrations/zed/tests/test_rules_file.py @@ -0,0 +1,111 @@ +"""Tests for the instruction-file memory-block writer.""" + +from pathlib import Path + +from hindsight_zed.rules_file import ( + BEGIN_MARKER, + END_MARKER, + clear_memory_block, + resolve_instruction_file, + write_memory_block, +) + + +def test_resolve_prefers_existing_high_priority_file(tmp_path): + # A project with AGENTS.md but no .rules: Zed reads AGENTS.md, so must we. + (tmp_path / "AGENTS.md").write_text("# my agent rules\n") + assert resolve_instruction_file(tmp_path).name == "AGENTS.md" + + +def test_resolve_dot_rules_wins_when_present(tmp_path): + (tmp_path / "AGENTS.md").write_text("x") + (tmp_path / ".rules").write_text("y") + assert resolve_instruction_file(tmp_path).name == ".rules" + + +def test_resolve_falls_back_to_dot_rules(tmp_path): + # Empty project → we create .rules. + assert resolve_instruction_file(tmp_path).name == ".rules" + + +def test_write_creates_dot_rules_when_no_instruction_file(tmp_path): + target = write_memory_block(tmp_path, "- user likes pytest") + assert target.name == ".rules" + text = target.read_text() + assert BEGIN_MARKER in text and END_MARKER in text + assert "- user likes pytest" in text + + +def test_write_does_not_hijack_existing_agents_md(tmp_path): + # The footgun the real Zed test surfaced: don't create .rules and suppress + # the user's AGENTS.md — write into AGENTS.md instead, preserving it. + agents = tmp_path / "AGENTS.md" + agents.write_text("# Project rules\n\nAlways use tabs.\n") + target = write_memory_block(tmp_path, "- user likes pytest") + assert target.name == "AGENTS.md" + assert not (tmp_path / ".rules").exists() + text = agents.read_text() + assert "Always use tabs." in text # user content preserved + assert "- user likes pytest" in text # our block added + assert text.index(BEGIN_MARKER) < text.index("Always use tabs.") # memory leads + + +def test_rewrite_replaces_block_not_duplicate(tmp_path): + write_memory_block(tmp_path, "- old memory") + write_memory_block(tmp_path, "- new memory") + text = (tmp_path / ".rules").read_text() + assert text.count(BEGIN_MARKER) == 1 # exactly one block + assert "- new memory" in text + assert "- old memory" not in text + + +def test_rewrite_preserves_user_content_around_block(tmp_path): + agents = tmp_path / "AGENTS.md" + agents.write_text("# Rules\n\nUse spaces.\n") + write_memory_block(tmp_path, "- mem v1") + write_memory_block(tmp_path, "- mem v2") + text = agents.read_text() + assert "Use spaces." in text + assert "- mem v2" in text and "- mem v1" not in text + assert text.count(BEGIN_MARKER) == 1 + + +def test_clear_removes_block_keeps_user_content(tmp_path): + agents = tmp_path / "AGENTS.md" + agents.write_text("# Rules\n\nUse spaces.\n") + write_memory_block(tmp_path, "- mem") + clear_memory_block(tmp_path) + text = agents.read_text() + assert BEGIN_MARKER not in text + assert "Use spaces." in text + + +def test_clear_deletes_self_created_rules_file(tmp_path): + # If we created .rules and it held only our block, removing leaves nothing → + # delete the file rather than leave an empty one. + write_memory_block(tmp_path, "- mem") + assert (tmp_path / ".rules").exists() + clear_memory_block(tmp_path) + assert not (tmp_path / ".rules").exists() + + +def test_empty_memory_text_clears(tmp_path): + write_memory_block(tmp_path, "- mem") + write_memory_block(tmp_path, " ") # empty → clears + assert not (tmp_path / ".rules").exists() + + +def test_preamble_included(tmp_path): + write_memory_block(tmp_path, "- fact", preamble="Relevant memories:") + text = (tmp_path / ".rules").read_text() + assert "Relevant memories:" in text + assert text.index("Relevant memories:") < text.index("- fact") + + +def test_malformed_block_recovered(tmp_path): + # A begin marker with no end (file got truncated) must not corrupt rewrite. + (tmp_path / ".rules").write_text(f"{BEGIN_MARKER}\n- half written") + write_memory_block(tmp_path, "- clean") + text = (tmp_path / ".rules").read_text() + assert text.count(BEGIN_MARKER) == 1 + assert "- clean" in text and "- half written" not in text diff --git a/hindsight-integrations/zed/tests/test_threads_db.py b/hindsight-integrations/zed/tests/test_threads_db.py new file mode 100644 index 000000000..d6fa14c48 --- /dev/null +++ b/hindsight-integrations/zed/tests/test_threads_db.py @@ -0,0 +1,235 @@ +"""Tests for the Zed threads.db reader. + +Zed isn't installed in CI, so these synthesize real ``threads.db`` fixtures — +a SQLite database with zstd-compressed JSON blobs in every on-disk format the +reader must handle — and assert the extracted transcript. +""" + +import json +import sqlite3 +from pathlib import Path + +import pytest +import zstandard + +from hindsight_zed.threads_db import ( + ZedThread, + default_threads_db_path, + parse_thread_json, + read_threads, +) + + +# ── Fixture helpers ─────────────────────────────────────────────────────────── + + +def _make_db(tmp_path: Path) -> Path: + db = tmp_path / "threads.db" + conn = sqlite3.connect(db) + conn.execute( + """ + CREATE TABLE threads ( + id TEXT PRIMARY KEY, + summary TEXT NOT NULL, + updated_at TEXT NOT NULL, + data_type TEXT NOT NULL, + data BLOB NOT NULL, + parent_id TEXT, + folder_paths TEXT, + folder_paths_order TEXT, + created_at TEXT + ) + """ + ) + conn.commit() + conn.close() + return db + + +def _insert(db: Path, *, id, summary, updated_at, doc, compress=True, folder_paths=None): + raw = json.dumps(doc).encode("utf-8") + if compress: + # Mimic Zed exactly: it writes streaming zstd frames (zstd::encode_all) + # whose header does NOT declare the decompressed content size. Use the + # streaming compressobj rather than .compress() (which would write the + # size and mask the real-world frame the reader must handle). + obj = zstandard.ZstdCompressor(level=3).compressobj() + data = obj.compress(raw) + obj.flush() + data_type = "zstd" + else: + data, data_type = raw, "json" + conn = sqlite3.connect(db) + conn.execute( + "INSERT INTO threads (id, summary, updated_at, data_type, data, folder_paths) " + "VALUES (?, ?, ?, ?, ?, ?)", + (id, summary, updated_at, data_type, data, json.dumps(folder_paths) if folder_paths else None), + ) + conn.commit() + conn.close() + + +# ── Sample documents per format ─────────────────────────────────────────────── + +CURRENT_DOC = { + "version": "0.3.0", + "title": "Fix the parser bug", + "updated_at": "2026-06-10T14:22:05Z", + "messages": [ + {"User": {"id": "u1", "content": [{"Text": "Why does the reader crash on empty input?"}]}}, + { + "Agent": { + "content": [ + {"Thinking": {"text": "internal", "signature": None}}, + {"Text": "It crashes on a zero-length blob. Guard for empty data."}, + {"ToolUse": {"id": "t1", "name": "read_file", "input": {}}}, + ], + "tool_results": {}, + } + }, + "Resume", # unit variant → bare string, must not break parsing + {"User": {"id": "u2", "content": [{"Mention": {"uri": "x", "content": "see @reader.rs"}}]}}, + ], +} + +LEGACY_020_DOC = { + "version": "0.2.0", + "summary": "Old thread", + "updated_at": "2026-01-02T00:00:00Z", + "messages": [ + {"id": 0, "role": "user", "segments": [{"type": "text", "text": "hello from legacy"}]}, + {"id": 1, "role": "system", "segments": [{"type": "text", "text": "system prompt — dropped"}]}, + {"id": 2, "role": "assistant", "segments": [{"type": "text", "text": "legacy reply"}]}, + ], +} + +OLDEST_DOC = { + # no "version" key — messages carry a flat "text" field + "summary": "Oldest thread", + "updated_at": "2025-12-01T00:00:00Z", + "messages": [ + {"id": 0, "role": "user", "text": "oldest user line"}, + {"id": 1, "role": "assistant", "text": "oldest assistant line"}, + ], +} + + +# ── Per-format parsing ──────────────────────────────────────────────────────── + + +def test_current_format_roles_and_text(): + msgs = parse_thread_json(json.dumps(CURRENT_DOC)) + assert [m.role for m in msgs] == ["user", "assistant", "user"] + assert msgs[0].text == "Why does the reader crash on empty input?" + # Thinking/ToolUse excluded; only the plain Text block kept. + assert msgs[1].text == "It crashes on a zero-length blob. Guard for empty data." + # Mention contributes its resolved content text. + assert msgs[2].text == "see @reader.rs" + + +def test_resume_bare_string_does_not_break(): + # The "Resume" string element must be skipped, not crash. + msgs = parse_thread_json(json.dumps(CURRENT_DOC)) + assert all(isinstance(m.text, str) for m in msgs) + + +def test_legacy_020_segments_and_system_dropped(): + msgs = parse_thread_json(json.dumps(LEGACY_020_DOC)) + assert [m.role for m in msgs] == ["user", "assistant"] # system dropped + assert msgs[0].text == "hello from legacy" + assert msgs[1].text == "legacy reply" + + +def test_oldest_flat_text_format(): + msgs = parse_thread_json(json.dumps(OLDEST_DOC)) + assert [m.role for m in msgs] == ["user", "assistant"] + assert msgs[0].text == "oldest user line" + + +def test_unknown_version_skipped(): + assert parse_thread_json(json.dumps({"version": "9.9.9", "messages": []})) is None + + +def test_malformed_json_returns_none(): + assert parse_thread_json("{not json") is None + + +# ── End-to-end DB reading ───────────────────────────────────────────────────── + + +def test_read_threads_mixed_versions(tmp_path): + db = _make_db(tmp_path) + _insert(db, id="a", summary="cur", updated_at="2026-06-10T10:00:00Z", doc=CURRENT_DOC, + folder_paths=["/Users/me/proj-a"]) + _insert(db, id="b", summary="leg", updated_at="2026-06-10T09:00:00Z", doc=LEGACY_020_DOC) + _insert(db, id="c", summary="old", updated_at="2026-06-10T08:00:00Z", doc=OLDEST_DOC) + + threads = read_threads(db) + by_id = {t.id: t for t in threads} + assert set(by_id) == {"a", "b", "c"} + assert by_id["a"].title == "Fix the parser bug" + assert by_id["a"].folder_paths == ["/Users/me/proj-a"] + assert len(by_id["a"].messages) == 3 + assert len(by_id["b"].messages) == 2 + + +def test_read_threads_uncompressed_json_row(tmp_path): + db = _make_db(tmp_path) + _insert(db, id="j", summary="json", updated_at="2026-06-10T10:00:00Z", doc=CURRENT_DOC, compress=False) + threads = read_threads(db) + assert len(threads) == 1 + assert threads[0].messages[0].text.startswith("Why does the reader crash") + + +def test_read_threads_since_filter(tmp_path): + db = _make_db(tmp_path) + _insert(db, id="old", summary="o", updated_at="2026-06-10T08:00:00Z", doc=CURRENT_DOC) + _insert(db, id="new", summary="n", updated_at="2026-06-10T12:00:00Z", doc=CURRENT_DOC) + recent = read_threads(db, since="2026-06-10T10:00:00Z") + assert [t.id for t in recent] == ["new"] + + +def test_read_threads_missing_db_returns_empty(tmp_path): + assert read_threads(tmp_path / "nope.db") == [] + + +def test_read_threads_unknown_version_skipped_at_db_level(tmp_path): + db = _make_db(tmp_path) + _insert(db, id="future", summary="f", updated_at="2026-06-10T10:00:00Z", + doc={"version": "9.9.9", "messages": []}) + _insert(db, id="ok", summary="g", updated_at="2026-06-10T11:00:00Z", doc=CURRENT_DOC) + threads = read_threads(db) + assert [t.id for t in threads] == ["ok"] + + +def test_opens_readonly_does_not_lock(tmp_path): + # Reading must not block a concurrent writer — open read-only and verify we + # can still write to the db afterward. + db = _make_db(tmp_path) + _insert(db, id="a", summary="x", updated_at="2026-06-10T10:00:00Z", doc=CURRENT_DOC) + read_threads(db) + # Writing still works (no lingering lock from the reader). + _insert(db, id="b", summary="y", updated_at="2026-06-10T11:00:00Z", doc=CURRENT_DOC) + assert len(read_threads(db)) == 2 + + +# ── Path resolution ─────────────────────────────────────────────────────────── + + +def test_default_path_env_override(monkeypatch): + monkeypatch.setenv("ZED_THREADS_DB", "/custom/threads.db") + assert default_threads_db_path() == Path("/custom/threads.db") + + +def test_default_path_macos(monkeypatch): + monkeypatch.delenv("ZED_THREADS_DB", raising=False) + monkeypatch.setattr("sys.platform", "darwin") + p = default_threads_db_path() + assert p.parts[-3:] == ("Zed", "threads", "threads.db") + + +def test_default_path_linux(monkeypatch): + monkeypatch.delenv("ZED_THREADS_DB", raising=False) + monkeypatch.delenv("XDG_DATA_HOME", raising=False) + monkeypatch.setattr("sys.platform", "linux") + p = default_threads_db_path() + assert p.parts[-3:] == ("zed", "threads", "threads.db") From a65c4192ff4a34262a2cc3e36c07af21cbdbee94 Mon Sep 17 00:00:00 2001 From: DK09876 Date: Thu, 11 Jun 2026 12:19:33 -0700 Subject: [PATCH 2/7] feat(zed): daemon, CLI, bank/config/content, packaging, CI + docs Completes the Zed integration: a background daemon (launchd/systemd) that polls Zed's threads.db and, per project, keeps a recalled-memory block fresh in the instruction file Zed always reads (auto-recall) and retains conversations (auto-retain). Per-project banks by default. - daemon.py / state.py: poll -> recall -> write block; retain w/ dedup - client.py / config.py / bank.py / content.py: HTTP client, typed config, per-git-repo bank derivation, transcript+memory formatting - cli.py: hindsight-zed init/run/status/uninstall (launchd + systemd) - pyproject (hindsight-zed, zstandard runtime dep), README, settings.json - CI job + detect-changes filter, VALID_INTEGRATIONS, docs page + gallery entry - 58 deterministic tests + a gated requires_real_llm e2e; ruff clean Closes #2096 --- .github/workflows/test.yml | 34 +++ hindsight-docs/docs-integrations/zed.md | 45 ++++ hindsight-docs/src/data/integrations.json | 26 ++- hindsight-docs/static/img/icons/zed.svg | 1 + hindsight-integrations/zed/.gitignore | 3 + hindsight-integrations/zed/README.md | 77 +++++++ .../zed/hindsight_zed/__init__.py | 3 + .../zed/hindsight_zed/bank.py | 84 ++++++++ .../zed/hindsight_zed/cli.py | 200 ++++++++++++++++++ .../zed/hindsight_zed/client.py | 117 ++++++++++ .../zed/hindsight_zed/config.py | 137 ++++++++++++ .../zed/hindsight_zed/content.py | 53 +++++ .../zed/hindsight_zed/daemon.py | 134 ++++++++++++ .../zed/hindsight_zed/state.py | 43 ++++ hindsight-integrations/zed/pyproject.toml | 61 ++++++ hindsight-integrations/zed/settings.json | 13 ++ hindsight-integrations/zed/tests/test_bank.py | 52 +++++ hindsight-integrations/zed/tests/test_cli.py | 46 ++++ .../zed/tests/test_config.py | 45 ++++ .../zed/tests/test_content.py | 51 +++++ .../zed/tests/test_daemon.py | 164 ++++++++++++++ hindsight-integrations/zed/tests/test_e2e.py | 66 ++++++ .../zed/tests/test_state.py | 30 +++ scripts/release-integration.sh | 2 +- 24 files changed, 1478 insertions(+), 9 deletions(-) create mode 100644 hindsight-docs/docs-integrations/zed.md create mode 100644 hindsight-docs/static/img/icons/zed.svg create mode 100644 hindsight-integrations/zed/.gitignore create mode 100644 hindsight-integrations/zed/README.md create mode 100644 hindsight-integrations/zed/hindsight_zed/bank.py create mode 100644 hindsight-integrations/zed/hindsight_zed/cli.py create mode 100644 hindsight-integrations/zed/hindsight_zed/client.py create mode 100644 hindsight-integrations/zed/hindsight_zed/config.py create mode 100644 hindsight-integrations/zed/hindsight_zed/content.py create mode 100644 hindsight-integrations/zed/hindsight_zed/daemon.py create mode 100644 hindsight-integrations/zed/hindsight_zed/state.py create mode 100644 hindsight-integrations/zed/pyproject.toml create mode 100644 hindsight-integrations/zed/settings.json create mode 100644 hindsight-integrations/zed/tests/test_bank.py create mode 100644 hindsight-integrations/zed/tests/test_cli.py create mode 100644 hindsight-integrations/zed/tests/test_config.py create mode 100644 hindsight-integrations/zed/tests/test_content.py create mode 100644 hindsight-integrations/zed/tests/test_daemon.py create mode 100644 hindsight-integrations/zed/tests/test_e2e.py create mode 100644 hindsight-integrations/zed/tests/test_state.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b9d782b77..734d9da19 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,6 +47,7 @@ jobs: integrations-paperclip: ${{ steps.filter.outputs.integrations-paperclip }} integrations-opencode: ${{ steps.filter.outputs.integrations-opencode }} integrations-cursor: ${{ steps.filter.outputs.integrations-cursor }} + integrations-zed: ${{ steps.filter.outputs.integrations-zed }} integrations-n8n: ${{ steps.filter.outputs.integrations-n8n }} integrations-cloudflare-oauth-proxy: ${{ steps.filter.outputs.integrations-cloudflare-oauth-proxy }} integrations-superagent: ${{ steps.filter.outputs.integrations-superagent }} @@ -156,6 +157,8 @@ jobs: - 'hindsight-integrations/opencode/**' integrations-cursor: - 'hindsight-integrations/cursor/**' + integrations-zed: + - 'hindsight-integrations/zed/**' integrations-n8n: - 'hindsight-integrations/n8n/**' integrations-cloudflare-oauth-proxy: @@ -476,6 +479,37 @@ jobs: working-directory: ./hindsight-integrations/cursor run: python -m pytest tests/ -v + test-zed-integration: + needs: [detect-changes] + if: >- + github.event_name != 'pull_request_review' && + (github.event_name == 'workflow_dispatch' || + needs.detect-changes.outputs.integrations-zed == 'true' || + needs.detect-changes.outputs.ci == 'true') + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha || '' }} + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.11' + + - name: Install package and pytest + working-directory: ./hindsight-integrations/zed + # Installs the package (incl. the zstandard runtime dep) so the threads.db + # reader tests can decompress Zed's zstd blobs. + run: pip install -e . pytest + + - name: Run tests + working-directory: ./hindsight-integrations/zed + # PR CI runs only the deterministic bucket; the real-LLM E2E bucket + # (requires_real_llm) needs a live Hindsight server and runs separately. + run: python -m pytest tests/ -v -m "not requires_real_llm" + test-omo-integration: needs: [detect-changes] if: >- diff --git a/hindsight-docs/docs-integrations/zed.md b/hindsight-docs/docs-integrations/zed.md new file mode 100644 index 000000000..57a1dc970 --- /dev/null +++ b/hindsight-docs/docs-integrations/zed.md @@ -0,0 +1,45 @@ +--- +sidebar_position: 7 +title: "Zed Persistent Memory with Hindsight | Integration" +description: "Add automatic long-term memory to the Zed editor's AI assistant with Hindsight. Recalls relevant project memory into every conversation and retains sessions — no manual steps." +--- + +# Zed + +Automatic, always-on long-term memory for the [Zed](https://zed.dev) editor's AI assistant, powered by [Hindsight](https://vectorize.io/hindsight). When you chat with Zed's Agent Panel, relevant memory from past sessions on that project is injected automatically — no manual tool calls — and your conversations are retained so the next session builds on them. + +[View Changelog →](/changelog/integrations/zed) + +## How It Works + +Zed has no AI-conversation hook, but it **always includes a project's instruction file** (`.rules` / `AGENTS.md` / …) in every agent conversation, and it stores conversations in a local `threads.db`. `hindsight-zed` runs a small background daemon that uses both: + +- **Auto-recall (passive injection):** when a Zed conversation updates, the daemon recalls relevant memory for that project and writes it into a fenced `` block in the project's instruction file. Zed includes that file automatically, so memory "just shows up" on the next turn. The block is written into the file Zed actually reads — it never hijacks your existing `AGENTS.md`/`CLAUDE.md`. +- **Auto-retain (passive capture):** the daemon reads finished and updated threads from `threads.db` and retains their transcripts into the project's Hindsight bank. + +Memory is **per-project** by default — each git repository gets its own bank, so context from one codebase doesn't leak into another. + +## Setup + +```bash +pip install hindsight-zed +hindsight-zed init --api-token YOUR_HINDSIGHT_API_KEY +``` + +`init` writes config to `~/.hindsight/zed.json` and installs a background service (launchd on macOS, systemd user service on Linux). After that it's hands-off — open any project in Zed and memory works. + +Use a [Hindsight Cloud](https://hindsight.vectorize.io) key, or point at a self-hosted server with `--api-url http://localhost:8888`. To share one bank across all projects, pass `--fixed-bank-id my-memory`. + +## Commands + +| Command | Description | +| --- | --- | +| `hindsight-zed init` | One-time setup: config + background daemon | +| `hindsight-zed status` | Whether the daemon is running | +| `hindsight-zed uninstall` | Stop and remove the daemon | + +## Limitation + +Zed exposes no per-prompt hook, so injection is **periodic** (refreshed when a conversation updates) rather than recomputed against each individual keystroke. In practice the relevant project memory is present in context for every turn. + +See the [package README](https://github.com/vectorize-io/hindsight/tree/main/hindsight-integrations/zed) for full configuration options. diff --git a/hindsight-docs/src/data/integrations.json b/hindsight-docs/src/data/integrations.json index 555e12849..a89086cd4 100644 --- a/hindsight-docs/src/data/integrations.json +++ b/hindsight-docs/src/data/integrations.json @@ -3,7 +3,7 @@ { "id": "litellm", "name": "LiteLLM", - "description": "Add long-term memory to any LLM via LiteLLM proxy callbacks. Zero code changes — configure once and every model gets memory.", + "description": "Add long-term memory to any LLM via LiteLLM proxy callbacks. Zero code changes \u2014 configure once and every model gets memory.", "type": "official", "by": "hindsight", "category": "framework", @@ -150,6 +150,16 @@ "link": "/sdks/integrations/cursor", "icon": "/img/icons/cursor.svg" }, + { + "id": "zed", + "name": "Zed", + "description": "Automatic long-term memory for the Zed editor's AI assistant via Hindsight. Recalls relevant project memory into every conversation and retains sessions \u2014 no manual steps.", + "type": "official", + "by": "hindsight", + "category": "tool", + "link": "/sdks/integrations/zed", + "icon": "/img/icons/zed.svg" + }, { "id": "cline", "name": "Cline", @@ -233,7 +243,7 @@ { "id": "nemoclaw", "name": "NemoClaw", - "description": "Persistent memory for NemoClaw sandboxed agents. One-command setup — no code changes required.", + "description": "Persistent memory for NemoClaw sandboxed agents. One-command setup \u2014 no code changes required.", "type": "official", "by": "hindsight", "category": "framework", @@ -243,7 +253,7 @@ { "id": "paperclip", "name": "Paperclip", - "description": "Persistent memory for Paperclip AI agents. Install the plugin once and every agent gets automatic recall before runs and retain after — no code changes.", + "description": "Persistent memory for Paperclip AI agents. Install the plugin once and every agent gets automatic recall before runs and retain after \u2014 no code changes.", "type": "official", "by": "hindsight", "category": "framework", @@ -253,7 +263,7 @@ { "id": "right-agent", "name": "Right Agent", - "description": "Persistent memory for Right Agent — closed-box AI agents that run Claude Code inside OpenShell sandboxes. Hindsight is the native memory provider, selected during initialization.", + "description": "Persistent memory for Right Agent \u2014 closed-box AI agents that run Claude Code inside OpenShell sandboxes. Hindsight is the native memory provider, selected during initialization.", "type": "official", "by": "hindsight", "category": "framework", @@ -393,7 +403,7 @@ { "id": "omo", "name": "OMO (oh-my-openagent)", - "description": "Automatic long-term memory for OMO agent harness via Hindsight hooks. Recalls context before each prompt and retains session learnings — zero config for cloud users.", + "description": "Automatic long-term memory for OMO agent harness via Hindsight hooks. Recalls context before each prompt and retains session learnings \u2014 zero config for cloud users.", "type": "official", "by": "hindsight", "category": "tool", @@ -413,7 +423,7 @@ { "id": "obsidian", "name": "Obsidian", - "description": "Sync your Obsidian vault into Hindsight and chat with an agent grounded on your notes. Your vault stays the source of truth — every answer cites the note it came from.", + "description": "Sync your Obsidian vault into Hindsight and chat with an agent grounded on your notes. Your vault stays the source of truth \u2014 every answer cites the note it came from.", "type": "official", "by": "hindsight", "category": "tool", @@ -423,7 +433,7 @@ { "id": "hindclaw", "name": "HindClaw", - "description": "Production memory infrastructure for AI agents — server-side access control via Hindsight extensions, Terraform provider for users/groups/banks/permissions, and an OpenClaw plugin with auto-managed embed.", + "description": "Production memory infrastructure for AI agents \u2014 server-side access control via Hindsight extensions, Terraform provider for users/groups/banks/permissions, and an OpenClaw plugin with auto-managed embed.", "type": "community", "by": "mrkhachaturov", "category": "framework", @@ -443,7 +453,7 @@ { "id": "gemini-spark", "name": "Gemini Spark", - "description": "Persistent memory for Google's Gemini Spark assistant via MCP. Spark calls Hindsight's recall and retain tools through its built-in MCP client — works with Hindsight Cloud or self-hosted via an OAuth proxy.", + "description": "Persistent memory for Google's Gemini Spark assistant via MCP. Spark calls Hindsight's recall and retain tools through its built-in MCP client \u2014 works with Hindsight Cloud or self-hosted via an OAuth proxy.", "type": "official", "by": "hindsight", "category": "tool", diff --git a/hindsight-docs/static/img/icons/zed.svg b/hindsight-docs/static/img/icons/zed.svg new file mode 100644 index 000000000..589bdafb1 --- /dev/null +++ b/hindsight-docs/static/img/icons/zed.svg @@ -0,0 +1 @@ + diff --git a/hindsight-integrations/zed/.gitignore b/hindsight-integrations/zed/.gitignore new file mode 100644 index 000000000..77ac75498 --- /dev/null +++ b/hindsight-integrations/zed/.gitignore @@ -0,0 +1,3 @@ +.venv/ +__pycache__/ +*.pyc diff --git a/hindsight-integrations/zed/README.md b/hindsight-integrations/zed/README.md new file mode 100644 index 000000000..a383b04e1 --- /dev/null +++ b/hindsight-integrations/zed/README.md @@ -0,0 +1,77 @@ +# hindsight-zed + +Automatic, always-on long-term memory for the [Zed](https://zed.dev) editor's AI +assistant, powered by [Hindsight](https://github.com/vectorize-io/hindsight). + +When you chat with Zed's Agent Panel, relevant memory from past sessions on that +project is injected automatically — no manual tool calls, no slash commands — +and your conversations are retained so the next session builds on them. + +## How it works + +Zed has no AI-conversation hook, but it **always includes a project's +instruction file** (`.rules` / `AGENTS.md` / …) in every agent conversation, and +it stores conversations in a local `threads.db`. `hindsight-zed` runs a small +background daemon that uses both: + +- **Auto-recall (passive injection):** when a Zed conversation updates, the + daemon recalls relevant memory for that project and writes it into a fenced + `` block in the project's instruction file. Zed includes + that file automatically, so memory "just shows up" on the next turn. The block + is written into the file Zed actually reads — it never hijacks or overwrites + your existing `AGENTS.md`/`CLAUDE.md`. +- **Auto-retain (passive capture):** the daemon reads finished/updated threads + from `threads.db` and retains their transcripts into the project's bank. + +Memory is **per-project** by default — each git repo gets its own Hindsight bank, +so context from one codebase doesn't leak into another. + +## Install + +```bash +pip install hindsight-zed +hindsight-zed init --api-token YOUR_HINDSIGHT_API_KEY +``` + +`init` writes config to `~/.hindsight/zed.json` and installs a background service +(launchd on macOS, systemd user service on Linux) that runs automatically. After +that it's hands-off — open any project in Zed and memory works. + +Use a [Hindsight Cloud](https://hindsight.vectorize.io) key, or point at a +self-hosted server with `--api-url http://localhost:8888`. + +To use one shared bank across all projects instead of per-project: + +```bash +hindsight-zed init --api-token ... --fixed-bank-id my-memory +``` + +## Commands + +| Command | Description | +| --- | --- | +| `hindsight-zed init` | One-time setup: config + background daemon | +| `hindsight-zed status` | Whether the daemon is running | +| `hindsight-zed uninstall` | Stop and remove the daemon | +| `hindsight-zed run` | Run the daemon in the foreground (used by the service) | + +## Configuration + +Settings layer (later wins): defaults → `~/.hindsight/zed.json` → environment +variables (`HINDSIGHT_API_URL`, `HINDSIGHT_API_TOKEN`, +`HINDSIGHT_ZED_FIXED_BANK_ID`, `HINDSIGHT_ZED_AUTO_RECALL`, …). See +`hindsight_zed/config.py` for the full list. + +## On-demand tools (optional) + +The passive daemon above is the headline feature. For explicit recall/retain in +a conversation you can also add Hindsight's MCP server to Zed's `context_servers` +(see Zed's [MCP docs](https://zed.dev/docs/ai/mcp)) pointing at +`/mcp//`. + +## Limitation + +Zed exposes no per-prompt hook, so injection is **periodic** (refreshed when a +conversation updates), not query-aware on the exact keystroke. In practice the +relevant project memory is present in context; it just isn't recomputed against +each individual message the instant you send it. diff --git a/hindsight-integrations/zed/hindsight_zed/__init__.py b/hindsight-integrations/zed/hindsight_zed/__init__.py index e69de29bb..5c03e79c7 100644 --- a/hindsight-integrations/zed/hindsight_zed/__init__.py +++ b/hindsight-integrations/zed/hindsight_zed/__init__.py @@ -0,0 +1,3 @@ +"""Hindsight memory integration for the Zed editor.""" + +__version__ = "0.1.0" diff --git a/hindsight-integrations/zed/hindsight_zed/bank.py b/hindsight-integrations/zed/hindsight_zed/bank.py new file mode 100644 index 000000000..a79c2f011 --- /dev/null +++ b/hindsight-integrations/zed/hindsight_zed/bank.py @@ -0,0 +1,84 @@ +"""Per-project memory-bank derivation. + +Each project gets its own Hindsight bank, so memory from one codebase doesn't +bleed into another. The bank id is derived from the project's git repository +root (so all linked worktrees of a repo share one bank), falling back to the +directory basename when git is unavailable. +""" + +import re +import subprocess +from pathlib import Path +from typing import Optional + +from .config import ZedConfig + + +def _git_repo_root(directory: str) -> Optional[str]: + """Return the main worktree root for *directory*, or None if not a repo. + + ``git rev-parse --git-common-dir`` resolves to the *main* worktree's ``.git`` + even from a linked worktree, so all worktrees of a repo map to one bank. + """ + try: + out = subprocess.run( + ["git", "rev-parse", "--git-common-dir"], + cwd=directory, + capture_output=True, + text=True, + timeout=5, + ) + except (OSError, subprocess.SubprocessError): + return None + if out.returncode != 0: + return None + common = out.stdout.strip() + if not common: + return None + common_path = (Path(directory) / common).resolve() if not Path(common).is_absolute() else Path(common) + # ``/.git`` → parent is the worktree root; a bare ``repo.git`` → itself. + if common_path.name == ".git": + return str(common_path.parent) + return str(common_path) + + +def _slugify(name: str) -> str: + """Reduce a name to a stable, bank-safe slug.""" + slug = re.sub(r"[^a-zA-Z0-9._-]+", "-", name).strip("-").lower() + return slug or "project" + + +def project_name(directory: str) -> str: + """Derive a stable project name from a directory (git root basename).""" + root = _git_repo_root(directory) or directory + return _slugify(Path(root).name) + + +def bank_id_for_project(directory: str, config: ZedConfig) -> str: + """Resolve the bank id for a project directory. + + Honors a configured ``fixed_bank_id`` (single shared bank); otherwise + ``-`` (per-project). + """ + if config.fixed_bank_id: + return config.fixed_bank_id + name = project_name(directory) + prefix = config.bank_prefix.strip("-") + return f"{prefix}-{name}" if prefix else name + + +def bank_id_for_thread_paths(folder_paths: list, config: ZedConfig) -> Optional[str]: + """Resolve the bank id for a thread, from its ``folder_paths``. + + Returns None when the thread has no associated project folder (so the caller + can fall back to a default). Uses the first existing folder path. + """ + if config.fixed_bank_id: + return config.fixed_bank_id + for path in folder_paths or []: + if path and Path(path).exists(): + return bank_id_for_project(path, config) + # No usable folder — first path by name, or nothing. + if folder_paths: + return bank_id_for_project(folder_paths[0], config) + return None diff --git a/hindsight-integrations/zed/hindsight_zed/cli.py b/hindsight-integrations/zed/hindsight_zed/cli.py new file mode 100644 index 000000000..ff67500ab --- /dev/null +++ b/hindsight-integrations/zed/hindsight_zed/cli.py @@ -0,0 +1,200 @@ +"""CLI for the Hindsight Zed integration. + +One-time setup installs a small background daemon that watches Zed's thread +database and, per project, keeps a recalled-memory block fresh in the +instruction file Zed always reads (auto-recall) and retains conversations +(auto-retain). After ``init`` it is fully hands-off — no per-project steps. +""" + +import argparse +import json +import plistlib +import subprocess +import sys +from pathlib import Path +from typing import Optional + +from . import __version__ +from .config import USER_CONFIG_FILE, load_config + +LAUNCHD_LABEL = "io.vectorize.hindsight-zed" +LAUNCHD_PLIST = Path.home() / "Library" / "LaunchAgents" / f"{LAUNCHD_LABEL}.plist" +SYSTEMD_UNIT = Path.home() / ".config" / "systemd" / "user" / "hindsight-zed.service" +LOG_DIR = Path.home() / ".hindsight" + + +def _daemon_command() -> list: + """The command the service runs: this CLI's ``run`` subcommand.""" + # Use the same interpreter that's running so the install works under uv/venv. + return [sys.executable, "-m", "hindsight_zed.cli", "run"] + + +def _scaffold_config(api_url: Optional[str], api_token: Optional[str], fixed_bank_id: Optional[str]) -> None: + if USER_CONFIG_FILE.is_file(): + print(f" Config already exists at {USER_CONFIG_FILE}, leaving as-is.") + return + config: dict = {} + if api_url: + config["hindsightApiUrl"] = api_url + if api_token: + config["hindsightApiToken"] = api_token + if fixed_bank_id: + config["fixedBankId"] = fixed_bank_id + USER_CONFIG_FILE.parent.mkdir(parents=True, exist_ok=True) + USER_CONFIG_FILE.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8") + print(f" Wrote config to {USER_CONFIG_FILE}") + + +def _install_launchd() -> None: + LOG_DIR.mkdir(parents=True, exist_ok=True) + plist = { + "Label": LAUNCHD_LABEL, + "ProgramArguments": _daemon_command(), + "RunAtLoad": True, + "KeepAlive": True, + "StandardOutPath": str(LOG_DIR / "zed-daemon.log"), + "StandardErrorPath": str(LOG_DIR / "zed-daemon.log"), + } + LAUNCHD_PLIST.parent.mkdir(parents=True, exist_ok=True) + with open(LAUNCHD_PLIST, "wb") as f: + plistlib.dump(plist, f) + # Reload if already loaded, then load. + subprocess.run(["launchctl", "unload", str(LAUNCHD_PLIST)], capture_output=True) + result = subprocess.run(["launchctl", "load", str(LAUNCHD_PLIST)], capture_output=True, text=True) + if result.returncode != 0: + print(f" warning: launchctl load failed: {result.stderr.strip()}", file=sys.stderr) + else: + print(f" Installed and started launchd agent ({LAUNCHD_LABEL})") + + +def _install_systemd() -> None: + LOG_DIR.mkdir(parents=True, exist_ok=True) + cmd = " ".join(_daemon_command()) + unit = ( + "[Unit]\n" + "Description=Hindsight memory daemon for Zed\n\n" + "[Service]\n" + f"ExecStart={cmd}\n" + "Restart=always\n" + "RestartSec=5\n\n" + "[Install]\n" + "WantedBy=default.target\n" + ) + SYSTEMD_UNIT.parent.mkdir(parents=True, exist_ok=True) + SYSTEMD_UNIT.write_text(unit, encoding="utf-8") + subprocess.run(["systemctl", "--user", "daemon-reload"], capture_output=True) + result = subprocess.run( + ["systemctl", "--user", "enable", "--now", "hindsight-zed.service"], capture_output=True, text=True + ) + if result.returncode != 0: + print(f" warning: systemctl enable failed: {result.stderr.strip()}", file=sys.stderr) + print(" (you may need: systemctl --user enable --now hindsight-zed.service)") + else: + print(" Installed and started systemd user service (hindsight-zed.service)") + + +def _install_daemon() -> None: + if sys.platform == "darwin": + _install_launchd() + elif sys.platform.startswith("linux"): + _install_systemd() + else: + print(f" Automatic daemon install isn't supported on {sys.platform}.") + print(f" Run it yourself (e.g. at login): {' '.join(_daemon_command())}") + + +def _uninstall_daemon() -> None: + if sys.platform == "darwin" and LAUNCHD_PLIST.exists(): + subprocess.run(["launchctl", "unload", str(LAUNCHD_PLIST)], capture_output=True) + LAUNCHD_PLIST.unlink() + print(f" Removed launchd agent ({LAUNCHD_LABEL})") + elif sys.platform.startswith("linux") and SYSTEMD_UNIT.exists(): + subprocess.run(["systemctl", "--user", "disable", "--now", "hindsight-zed.service"], capture_output=True) + SYSTEMD_UNIT.unlink() + subprocess.run(["systemctl", "--user", "daemon-reload"], capture_output=True) + print(" Removed systemd user service") + else: + print(" No installed daemon found.") + + +def cmd_init(args: argparse.Namespace) -> None: + print("Setting up Hindsight for Zed ...") + _scaffold_config(args.api_url, args.api_token, args.fixed_bank_id) + + # Fail fast if the server is unreachable, so the user fixes config now. + cfg = load_config() + client_ok = True + try: + from .client import HindsightClient + + client_ok = HindsightClient(cfg.hindsight_api_url, cfg.hindsight_api_token).health_check() + except Exception: + client_ok = False + if not client_ok: + print(f" warning: could not reach Hindsight at {cfg.hindsight_api_url}.") + print(" Check --api-url / --api-token; the daemon will keep retrying.") + + if not args.no_daemon: + _install_daemon() + + print() + print("Done. Hindsight now runs automatically for every project you open in Zed.") + print("Open a project, chat with the Agent Panel, and relevant memory is injected") + print("via the project's instruction file; conversations are retained as you go.") + + +def cmd_uninstall(args: argparse.Namespace) -> None: + _uninstall_daemon() + + +def cmd_run(args: argparse.Namespace) -> None: + from .daemon import run + + run() + + +def cmd_status(args: argparse.Namespace) -> None: + if sys.platform == "darwin": + out = subprocess.run(["launchctl", "list", LAUNCHD_LABEL], capture_output=True, text=True) + print("running" if out.returncode == 0 else "not running") + elif sys.platform.startswith("linux"): + out = subprocess.run( + ["systemctl", "--user", "is-active", "hindsight-zed.service"], capture_output=True, text=True + ) + print(out.stdout.strip() or "unknown") + else: + print("status unavailable on this platform") + + +def main() -> None: + parser = argparse.ArgumentParser(prog="hindsight-zed", description="Hindsight memory for Zed") + parser.add_argument("--version", action="version", version=f"hindsight-zed {__version__}") + sub = parser.add_subparsers(dest="command") + + init_p = sub.add_parser("init", help="Set up the daemon and config (one-time)") + init_p.add_argument("--api-url", default=None, help="Hindsight API URL (default: cloud)") + init_p.add_argument("--api-token", default=None, help="Hindsight API token") + init_p.add_argument( + "--fixed-bank-id", default=None, help="Use one shared bank for all projects (default: per-project)" + ) + init_p.add_argument("--no-daemon", action="store_true", help="Write config but don't install the daemon") + init_p.set_defaults(func=cmd_init) + + run_p = sub.add_parser("run", help="Run the daemon in the foreground (used by the service)") + run_p.set_defaults(func=cmd_run) + + status_p = sub.add_parser("status", help="Show whether the daemon is running") + status_p.set_defaults(func=cmd_status) + + uninst_p = sub.add_parser("uninstall", help="Stop and remove the daemon") + uninst_p.set_defaults(func=cmd_uninstall) + + args = parser.parse_args() + if not hasattr(args, "func"): + parser.print_help() + sys.exit(1) + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/hindsight-integrations/zed/hindsight_zed/client.py b/hindsight-integrations/zed/hindsight_zed/client.py new file mode 100644 index 000000000..ed64e322c --- /dev/null +++ b/hindsight-integrations/zed/hindsight_zed/client.py @@ -0,0 +1,117 @@ +"""Hindsight REST API client (stdlib HTTP). + +A thin client over the Hindsight HTTP API — recall, retain, and bank mission — +sharing the shape used by the other editor integrations. +""" + +import json +import time +import urllib.error +import urllib.parse +import urllib.request +from typing import Optional + +DEFAULT_TIMEOUT = 15 # seconds +HEALTH_CHECK_RETRIES = 3 +HEALTH_CHECK_DELAY = 2 # seconds + + +def _validate_api_url(url: str) -> str: + """Validate and normalize the API URL. Reject non-HTTP schemes.""" + parsed = urllib.parse.urlparse(url) + if parsed.scheme not in ("http", "https"): + raise ValueError(f"Hindsight API URL must use http or https, got: {parsed.scheme!r}") + if not parsed.hostname: + raise ValueError(f"Hindsight API URL has no hostname: {url!r}") + return url.rstrip("/") + + +class HindsightClient: + """HTTP client for the Hindsight API.""" + + def __init__(self, api_url: str, api_token: Optional[str] = None): + self.api_url = _validate_api_url(api_url) + self.api_token = api_token + + def _headers(self) -> dict: + headers = {"Content-Type": "application/json"} + if self.api_token: + headers["Authorization"] = f"Bearer {self.api_token}" + return headers + + def _request(self, method: str, path: str, body: Optional[dict] = None, timeout: int = DEFAULT_TIMEOUT) -> dict: + url = f"{self.api_url}{path}" + data = json.dumps(body).encode() if body else None + req = urllib.request.Request(url, data=data, headers=self._headers(), method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read().decode()) + except urllib.error.HTTPError as e: + body_text = "" + try: + body_text = e.read().decode() + except Exception: + pass + raise RuntimeError(f"HTTP {e.code} from {url}: {body_text}") from e + + def health_check(self, timeout: int = 5) -> bool: + """Return True if the Hindsight server is reachable (retries a few times).""" + for attempt in range(1, HEALTH_CHECK_RETRIES + 1): + try: + url = f"{self.api_url}/health" + req = urllib.request.Request(url, headers=self._headers(), method="GET") + with urllib.request.urlopen(req, timeout=timeout) as resp: + if resp.status == 200: + return True + except Exception: + pass + if attempt < HEALTH_CHECK_RETRIES: + time.sleep(HEALTH_CHECK_DELAY) + return False + + def recall( + self, + bank_id: str, + query: str, + max_tokens: int = 1024, + budget: str = "mid", + types: Optional[list] = None, + timeout: int = 10, + ) -> dict: + """Recall memories from a bank. Returns the raw API response dict.""" + path = f"/v1/default/banks/{urllib.parse.quote(bank_id, safe='')}/memories/recall" + body: dict = {"query": query, "max_tokens": max_tokens} + if budget: + body["budget"] = budget + if types: + body["types"] = types + return self._request("POST", path, body, timeout=timeout) + + def retain( + self, + bank_id: str, + content: str, + document_id: str = "conversation", + context: Optional[str] = None, + metadata: Optional[dict] = None, + tags: Optional[list] = None, + timeout: int = 15, + ) -> dict: + """Retain content into a bank's memory (async server-side processing).""" + path = f"/v1/default/banks/{urllib.parse.quote(bank_id, safe='')}/memories" + item: dict = {"content": content, "document_id": document_id, "metadata": metadata or {}} + if context: + item["context"] = context + if tags: + item["tags"] = tags + return self._request("POST", path, {"items": [item], "async": True}, timeout=timeout) + + def set_bank_mission( + self, bank_id: str, mission: str, retain_mission: Optional[str] = None, timeout: int = 15 + ) -> dict: + """Set the reflect/retain mission for a bank.""" + path = f"/v1/default/banks/{urllib.parse.quote(bank_id, safe='')}/config" + updates: dict = {"reflect_mission": mission} + if retain_mission: + updates["retain_mission"] = retain_mission + return self._request("PATCH", path, {"updates": updates}, timeout=timeout) diff --git a/hindsight-integrations/zed/hindsight_zed/config.py b/hindsight-integrations/zed/hindsight_zed/config.py new file mode 100644 index 000000000..298053579 --- /dev/null +++ b/hindsight-integrations/zed/hindsight_zed/config.py @@ -0,0 +1,137 @@ +"""Configuration for the Hindsight Zed daemon. + +Settings layer (later wins): built-in defaults → ``~/.hindsight/zed.json`` → +environment variables. Resolved into a typed :class:`ZedConfig`. +""" + +import json +import os +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +# Cross-integration cloud-default convention. +DEFAULT_HINDSIGHT_API_URL = "https://api.hindsight.vectorize.io" + +USER_CONFIG_FILE = Path.home() / ".hindsight" / "zed.json" + + +@dataclass +class ZedConfig: + """Resolved daemon configuration.""" + + # Connection + hindsight_api_url: str = DEFAULT_HINDSIGHT_API_URL + hindsight_api_token: Optional[str] = None + + # Bank scoping (per-project by default — one bank per git repo / folder). + bank_prefix: str = "zed" + fixed_bank_id: Optional[str] = None # if set, all projects share this bank + bank_mission: str = "" + + # Recall (passive injection) + auto_recall: bool = True + recall_budget: str = "mid" + recall_max_tokens: int = 1024 + recall_types: list = field(default_factory=lambda: ["world", "experience"]) + recall_max_query_chars: int = 800 + recall_preamble: str = ( + "Relevant memory from past sessions on this project (recalled automatically; " + "use what's relevant, ignore the rest):" + ) + + # Retain (passive capture) + auto_retain: bool = True + retain_context: str = "zed" + retain_tags: list = field(default_factory=list) + + # Daemon + poll_interval: float = 5.0 # seconds between threads.db polls + debug: bool = False + + +# settings.json / user-config key -> (attribute, caster) +_FILE_KEYS = { + "hindsightApiUrl": ("hindsight_api_url", str), + "hindsightApiToken": ("hindsight_api_token", str), + "bankPrefix": ("bank_prefix", str), + "fixedBankId": ("fixed_bank_id", str), + "bankMission": ("bank_mission", str), + "autoRecall": ("auto_recall", bool), + "recallBudget": ("recall_budget", str), + "recallMaxTokens": ("recall_max_tokens", int), + "recallTypes": ("recall_types", list), + "recallMaxQueryChars": ("recall_max_query_chars", int), + "recallPreamble": ("recall_preamble", str), + "autoRetain": ("auto_retain", bool), + "retainContext": ("retain_context", str), + "retainTags": ("retain_tags", list), + "pollInterval": ("poll_interval", float), + "debug": ("debug", bool), +} + +# env var -> (attribute, caster) +_ENV_KEYS = { + "HINDSIGHT_API_URL": ("hindsight_api_url", str), + "HINDSIGHT_API_TOKEN": ("hindsight_api_token", str), + "HINDSIGHT_ZED_BANK_PREFIX": ("bank_prefix", str), + "HINDSIGHT_ZED_FIXED_BANK_ID": ("fixed_bank_id", str), + "HINDSIGHT_ZED_BANK_MISSION": ("bank_mission", str), + "HINDSIGHT_ZED_AUTO_RECALL": ("auto_recall", bool), + "HINDSIGHT_ZED_AUTO_RETAIN": ("auto_retain", bool), + "HINDSIGHT_ZED_RECALL_BUDGET": ("recall_budget", str), + "HINDSIGHT_ZED_RECALL_MAX_TOKENS": ("recall_max_tokens", int), + "HINDSIGHT_ZED_RETAIN_CONTEXT": ("retain_context", str), + "HINDSIGHT_ZED_POLL_INTERVAL": ("poll_interval", float), + "HINDSIGHT_ZED_DEBUG": ("debug", bool), +} + + +def _cast(value, typ): + """Cast a value (str from env, or JSON value from file) to ``typ``.""" + try: + if typ is bool: + if isinstance(value, bool): + return value + return str(value).lower() in ("true", "1", "yes") + if typ is int: + return int(value) + if typ is float: + return float(value) + if typ is list: + return list(value) if isinstance(value, (list, tuple)) else [value] + return str(value) + except (ValueError, TypeError): + return None + + +def load_config(config_file: Optional[Path] = None, env: Optional[dict] = None) -> ZedConfig: + """Load and resolve daemon configuration.""" + cfg = ZedConfig() + env = os.environ if env is None else env + + # 1. user config file + path = config_file if config_file is not None else USER_CONFIG_FILE + if path.is_file(): + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + data = {} + for key, (attr, typ) in _FILE_KEYS.items(): + if key in data and data[key] is not None: + cast = _cast(data[key], typ) + if cast is not None: + setattr(cfg, attr, cast) + + # 2. environment overrides + for key, (attr, typ) in _ENV_KEYS.items(): + if key in env and env[key] != "": + cast = _cast(env[key], typ) + if cast is not None: + setattr(cfg, attr, cast) + + # Empty URL falls back to the cloud default. + if not cfg.hindsight_api_url: + cfg.hindsight_api_url = DEFAULT_HINDSIGHT_API_URL + + return cfg diff --git a/hindsight-integrations/zed/hindsight_zed/content.py b/hindsight-integrations/zed/hindsight_zed/content.py new file mode 100644 index 000000000..cb3a80fa4 --- /dev/null +++ b/hindsight-integrations/zed/hindsight_zed/content.py @@ -0,0 +1,53 @@ +"""Format Zed threads for retain, and recall results for injection.""" + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .threads_db import ThreadMessage, ZedThread + + +def compose_recall_query(messages: "list[ThreadMessage]", max_chars: int = 800) -> str: + """Build a recall query from the most recent user message(s). + + Recall is keyed on what the user is currently asking, so we use the last + user turn (truncated). Falls back to the last message of any role. + """ + last_user = next((m.text for m in reversed(messages) if m.role == "user" and m.text.strip()), "") + query = last_user or (messages[-1].text if messages else "") + query = query.strip() + if len(query) > max_chars: + query = query[:max_chars] + return query + + +def format_memory_block(results: list) -> str: + """Format recall results into the memory block body (markdown list). + + ``results`` is the ``results`` array from the recall API response. + """ + lines = [] + for r in results or []: + if not isinstance(r, dict): + continue + text = (r.get("text") or "").strip() + if not text: + continue + mem_type = r.get("type") or "" + when = r.get("mentioned_at") or "" + suffix = "" + if mem_type: + suffix += f" [{mem_type}]" + if when: + suffix += f" ({when})" + lines.append(f"- {text}{suffix}") + return "\n".join(lines) + + +def format_transcript(thread: "ZedThread") -> str: + """Render a thread's messages as a ``[role]\\ntext`` transcript for retain.""" + blocks = [] + for m in thread.messages: + text = m.text.strip() + if text: + blocks.append(f"[{m.role}]\n{text}") + return "\n\n".join(blocks) diff --git a/hindsight-integrations/zed/hindsight_zed/daemon.py b/hindsight-integrations/zed/hindsight_zed/daemon.py new file mode 100644 index 000000000..b5c16a98b --- /dev/null +++ b/hindsight-integrations/zed/hindsight_zed/daemon.py @@ -0,0 +1,134 @@ +"""The Hindsight Zed daemon. + +Zed exposes no AI-conversation hook, so a small background process supplies the +automation: + + - **Auto-recall (passive injection):** when a thread is updated, recall memory + against its latest user message and rewrite the fenced ```` + block in that project's instruction file. Zed always includes that file in + the agent's context, so memory "just shows up" on the next turn. + - **Auto-retain (passive capture):** when a thread advances, store its + transcript into the project's Hindsight bank. + +Both sides poll the same ``threads.db`` so a single pass over new/changed +threads drives recall and retain together. +""" + +import logging +import time +from pathlib import Path +from typing import Optional + +from .bank import bank_id_for_thread_paths +from .client import HindsightClient +from .config import ZedConfig, load_config +from .content import compose_recall_query, format_memory_block, format_transcript +from .rules_file import write_memory_block +from .state import DaemonState +from .threads_db import ZedThread, default_threads_db_path, read_threads + +logger = logging.getLogger("hindsight_zed.daemon") + + +def _project_dir(thread: ZedThread) -> Optional[Path]: + """Return the first existing project folder for a thread, if any.""" + for path in thread.folder_paths: + p = Path(path) + if p.is_dir(): + return p + return None + + +def process_thread( + thread: ZedThread, + client: HindsightClient, + config: ZedConfig, + state: DaemonState, +) -> None: + """Run recall (inject) and retain (capture) for a single updated thread.""" + project = _project_dir(thread) + bank_id = bank_id_for_thread_paths(thread.folder_paths, config) + if not bank_id or project is None: + # No on-disk project to scope a bank or write a rules file to — skip. + return + + # ── Auto-recall → rewrite the project's memory block ────────────────────── + if config.auto_recall and thread.messages: + query = compose_recall_query(thread.messages, config.recall_max_query_chars) + if query: + try: + resp = client.recall( + bank_id, + query, + max_tokens=config.recall_max_tokens, + budget=config.recall_budget, + types=config.recall_types, + ) + block = format_memory_block(resp.get("results", [])) + write_memory_block(project, block, preamble=config.recall_preamble) + except Exception as e: + logger.debug("recall failed for bank %s: %s", bank_id, e) + + # ── Auto-retain → store the transcript ──────────────────────────────────── + if config.auto_retain and state.needs_retain(thread.id, thread.updated_at): + transcript = format_transcript(thread) + if transcript.strip(): + try: + client.retain( + bank_id, + transcript, + document_id=f"zed-thread-{thread.id}", + context=config.retain_context, + tags=config.retain_tags, + metadata={"source": "zed", "thread_id": thread.id}, + ) + state.mark_retained(thread.id, thread.updated_at) + except Exception as e: + logger.debug("retain failed for bank %s: %s", bank_id, e) + + +def poll_once( + db_path: Path, + client: HindsightClient, + config: ZedConfig, + state: DaemonState, + since: Optional[str], +) -> Optional[str]: + """Process all threads updated since ``since``. Returns the new high-water mark.""" + threads = read_threads(db_path, since=since) + if not threads: + return since + threads.sort(key=lambda t: t.updated_at) + high = since + for thread in threads: + process_thread(thread, client, config, state) + if high is None or thread.updated_at > high: + high = thread.updated_at + state.save() + return high + + +def run(db_path: Optional[Path] = None, config: Optional[ZedConfig] = None) -> None: + """Run the daemon poll loop forever.""" + config = config or load_config() + db_path = db_path or default_threads_db_path() + if config.debug: + logging.basicConfig(level=logging.DEBUG) + + client = HindsightClient(config.hindsight_api_url, config.hindsight_api_token) + state = DaemonState.load() + # Start from the newest already-retained revision so we don't reprocess the + # entire backlog on first run (recall would still refresh on the next turn). + since: Optional[str] = max(state.retained.values(), default=None) + + logger.info("hindsight-zed daemon started (db=%s, api=%s)", db_path, config.hindsight_api_url) + while True: + try: + since = poll_once(db_path, client, config, state, since) + except Exception as e: # never let one bad poll kill the daemon + logger.debug("poll error: %s", e) + time.sleep(config.poll_interval) + + +if __name__ == "__main__": + run() diff --git a/hindsight-integrations/zed/hindsight_zed/state.py b/hindsight-integrations/zed/hindsight_zed/state.py new file mode 100644 index 000000000..11e4c82b0 --- /dev/null +++ b/hindsight-integrations/zed/hindsight_zed/state.py @@ -0,0 +1,43 @@ +"""Persistent daemon state: which threads have been retained, and at what revision. + +A thread is retained again only when its ``updated_at`` advances past what we +last stored, so a long-running conversation is re-captured as it grows without +duplicating unchanged threads. +""" + +import json +from dataclasses import dataclass, field +from pathlib import Path + +DEFAULT_STATE_FILE = Path.home() / ".hindsight" / "zed-state.json" + + +@dataclass +class DaemonState: + """Tracks the last-retained ``updated_at`` per thread id.""" + + retained: dict = field(default_factory=dict) # thread_id -> updated_at string + path: Path = DEFAULT_STATE_FILE + + @classmethod + def load(cls, path: Path = DEFAULT_STATE_FILE) -> "DaemonState": + if path.is_file(): + try: + data = json.loads(path.read_text(encoding="utf-8")) + retained = data.get("retained", {}) + if isinstance(retained, dict): + return cls(retained={str(k): str(v) for k, v in retained.items()}, path=path) + except (json.JSONDecodeError, OSError): + pass + return cls(path=path) + + def save(self) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.write_text(json.dumps({"retained": self.retained}, indent=2) + "\n", encoding="utf-8") + + def needs_retain(self, thread_id: str, updated_at: str) -> bool: + """True if this thread is new or has advanced since the last retain.""" + return self.retained.get(thread_id) != updated_at + + def mark_retained(self, thread_id: str, updated_at: str) -> None: + self.retained[thread_id] = updated_at diff --git a/hindsight-integrations/zed/pyproject.toml b/hindsight-integrations/zed/pyproject.toml new file mode 100644 index 000000000..0766c0631 --- /dev/null +++ b/hindsight-integrations/zed/pyproject.toml @@ -0,0 +1,61 @@ +[project] +name = "hindsight-zed" +version = "0.1.0" +description = "Automatic long-term memory for the Zed editor's AI assistant via Hindsight" +readme = "README.md" +requires-python = ">=3.10" +license = { text = "MIT" } +authors = [ + { name = "Vectorize", email = "support@vectorize.io" } +] +keywords = [ + "ai", + "memory", + "zed", + "agents", + "hindsight", + "coding", +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] + +# Zed compresses its thread database with zstd, so a zstd codec is required. +dependencies = [ + "zstandard>=0.21.0", +] + +[project.scripts] +hindsight-zed = "hindsight_zed.cli:main" + +[project.urls] +Homepage = "https://github.com/vectorize-io/hindsight" +Documentation = "https://github.com/vectorize-io/hindsight/tree/main/hindsight-integrations/zed" +Repository = "https://github.com/vectorize-io/hindsight" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["hindsight_zed"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +markers = [ + "requires_real_llm: end-to-end test that needs live external services (a running Hindsight server and/or real LLM provider keys). Excluded from the deterministic PR-CI bucket via -m 'not requires_real_llm'; run on its own via -m requires_real_llm.", +] + +[dependency-groups] +dev = [ + "pytest>=9.0.2", + "ruff>=0.8.0", + "zstandard>=0.21.0", +] diff --git a/hindsight-integrations/zed/settings.json b/hindsight-integrations/zed/settings.json new file mode 100644 index 000000000..46454e73b --- /dev/null +++ b/hindsight-integrations/zed/settings.json @@ -0,0 +1,13 @@ +{ + "hindsightApiUrl": "https://api.hindsight.vectorize.io", + "hindsightApiToken": null, + "bankPrefix": "zed", + "fixedBankId": null, + "autoRecall": true, + "recallBudget": "mid", + "recallMaxTokens": 1024, + "autoRetain": true, + "retainContext": "zed", + "pollInterval": 5.0, + "debug": false +} diff --git a/hindsight-integrations/zed/tests/test_bank.py b/hindsight-integrations/zed/tests/test_bank.py new file mode 100644 index 000000000..3901ecc5c --- /dev/null +++ b/hindsight-integrations/zed/tests/test_bank.py @@ -0,0 +1,52 @@ +"""Tests for per-project bank derivation.""" + +import subprocess + +from hindsight_zed.bank import bank_id_for_project, bank_id_for_thread_paths, project_name +from hindsight_zed.config import ZedConfig + + +def _git_init(path): + subprocess.run(["git", "init", "-q"], cwd=path, check=True) + + +def test_project_name_from_git_root(tmp_path): + repo = tmp_path / "My_Cool.Repo" + repo.mkdir() + _git_init(repo) + sub = repo / "src" / "deep" + sub.mkdir(parents=True) + # A nested dir resolves to the repo root basename, slugified. + assert project_name(str(sub)) == "my_cool.repo" + + +def test_project_name_non_git_uses_basename(tmp_path): + d = tmp_path / "Plain Folder" + d.mkdir() + assert project_name(str(d)) == "plain-folder" + + +def test_bank_id_per_project(tmp_path): + d = tmp_path / "acme" + d.mkdir() + cfg = ZedConfig(bank_prefix="zed") + assert bank_id_for_project(str(d), cfg) == "zed-acme" + + +def test_bank_id_fixed_overrides(tmp_path): + d = tmp_path / "acme" + d.mkdir() + cfg = ZedConfig(fixed_bank_id="shared") + assert bank_id_for_project(str(d), cfg) == "shared" + + +def test_bank_id_for_thread_paths_uses_first_existing(tmp_path): + real = tmp_path / "real-proj" + real.mkdir() + cfg = ZedConfig(bank_prefix="zed") + bid = bank_id_for_thread_paths(["/does/not/exist", str(real)], cfg) + assert bid == "zed-real-proj" + + +def test_bank_id_for_thread_paths_none_when_empty(): + assert bank_id_for_thread_paths([], ZedConfig()) is None diff --git a/hindsight-integrations/zed/tests/test_cli.py b/hindsight-integrations/zed/tests/test_cli.py new file mode 100644 index 000000000..f5a4d3201 --- /dev/null +++ b/hindsight-integrations/zed/tests/test_cli.py @@ -0,0 +1,46 @@ +"""Tests for the CLI (config scaffolding; daemon install mocked).""" + +import json + +import pytest + +from hindsight_zed import cli + + +@pytest.fixture +def fake_home(tmp_path, monkeypatch): + monkeypatch.setattr(cli, "USER_CONFIG_FILE", tmp_path / ".hindsight" / "zed.json") + # Don't actually touch launchd/systemd or the network during init. + monkeypatch.setattr(cli, "_install_daemon", lambda: None) + monkeypatch.setattr(cli, "load_config", lambda: type("C", (), {"hindsight_api_url": "x", "hindsight_api_token": None})()) + return tmp_path + + +def test_init_writes_config(fake_home, monkeypatch, capsys): + monkeypatch.setattr("sys.argv", ["hindsight-zed", "init", "--api-token", "tok", "--api-url", "http://localhost:8888"]) + cli.main() + written = json.loads(cli.USER_CONFIG_FILE.read_text()) + assert written["hindsightApiToken"] == "tok" + assert written["hindsightApiUrl"] == "http://localhost:8888" + + +def test_init_fixed_bank(fake_home, monkeypatch): + monkeypatch.setattr("sys.argv", ["hindsight-zed", "init", "--fixed-bank-id", "shared", "--no-daemon"]) + cli.main() + written = json.loads(cli.USER_CONFIG_FILE.read_text()) + assert written["fixedBankId"] == "shared" + + +def test_init_does_not_clobber_existing_config(fake_home, monkeypatch): + cli.USER_CONFIG_FILE.parent.mkdir(parents=True) + cli.USER_CONFIG_FILE.write_text(json.dumps({"hindsightApiToken": "original"})) + monkeypatch.setattr("sys.argv", ["hindsight-zed", "init", "--api-token", "new", "--no-daemon"]) + cli.main() + assert json.loads(cli.USER_CONFIG_FILE.read_text())["hindsightApiToken"] == "original" + + +def test_version(monkeypatch, capsys): + monkeypatch.setattr("sys.argv", ["hindsight-zed", "--version"]) + with pytest.raises(SystemExit): + cli.main() + assert "hindsight-zed" in capsys.readouterr().out diff --git a/hindsight-integrations/zed/tests/test_config.py b/hindsight-integrations/zed/tests/test_config.py new file mode 100644 index 000000000..7fb24621b --- /dev/null +++ b/hindsight-integrations/zed/tests/test_config.py @@ -0,0 +1,45 @@ +"""Tests for config loading/merging.""" + +import json + +from hindsight_zed.config import DEFAULT_HINDSIGHT_API_URL, load_config + + +def test_defaults(tmp_path): + cfg = load_config(config_file=tmp_path / "nope.json", env={}) + assert cfg.hindsight_api_url == DEFAULT_HINDSIGHT_API_URL + assert cfg.auto_recall is True + assert cfg.auto_retain is True + assert cfg.bank_prefix == "zed" + assert cfg.fixed_bank_id is None + + +def test_file_overrides(tmp_path): + f = tmp_path / "zed.json" + f.write_text(json.dumps({"hindsightApiUrl": "http://localhost:8888", "fixedBankId": "shared", "autoRecall": False})) + cfg = load_config(config_file=f, env={}) + assert cfg.hindsight_api_url == "http://localhost:8888" + assert cfg.fixed_bank_id == "shared" + assert cfg.auto_recall is False + + +def test_env_overrides_file(tmp_path): + f = tmp_path / "zed.json" + f.write_text(json.dumps({"hindsightApiUrl": "http://from-file:8888"})) + cfg = load_config(config_file=f, env={"HINDSIGHT_API_URL": "http://from-env:9999", "HINDSIGHT_ZED_DEBUG": "true"}) + assert cfg.hindsight_api_url == "http://from-env:9999" + assert cfg.debug is True + + +def test_empty_url_falls_back_to_cloud(tmp_path): + f = tmp_path / "zed.json" + f.write_text(json.dumps({"hindsightApiUrl": ""})) + cfg = load_config(config_file=f, env={}) + assert cfg.hindsight_api_url == DEFAULT_HINDSIGHT_API_URL + + +def test_bad_types_ignored(tmp_path): + f = tmp_path / "zed.json" + f.write_text(json.dumps({"recallMaxTokens": "not-an-int"})) + cfg = load_config(config_file=f, env={}) + assert cfg.recall_max_tokens == 1024 # default kept diff --git a/hindsight-integrations/zed/tests/test_content.py b/hindsight-integrations/zed/tests/test_content.py new file mode 100644 index 000000000..247ae57cd --- /dev/null +++ b/hindsight-integrations/zed/tests/test_content.py @@ -0,0 +1,51 @@ +"""Tests for content formatting.""" + +from hindsight_zed.content import compose_recall_query, format_memory_block, format_transcript +from hindsight_zed.threads_db import ThreadMessage, ZedThread + + +def test_compose_recall_query_uses_last_user_message(): + msgs = [ + ThreadMessage("user", "first question"), + ThreadMessage("assistant", "an answer"), + ThreadMessage("user", "the latest question"), + ] + assert compose_recall_query(msgs) == "the latest question" + + +def test_compose_recall_query_truncates(): + long = "x" * 2000 + assert len(compose_recall_query([ThreadMessage("user", long)], max_chars=100)) == 100 + + +def test_compose_recall_query_empty(): + assert compose_recall_query([]) == "" + + +def test_format_memory_block(): + results = [ + {"text": "User prefers pytest", "type": "world", "mentioned_at": "2026-06-01"}, + {"text": "Building a parser", "type": "experience"}, + {"text": " "}, # blank — skipped + "not-a-dict", # ignored + ] + block = format_memory_block(results) + assert "- User prefers pytest [world] (2026-06-01)" in block + assert "- Building a parser [experience]" in block + assert block.count("\n- ") == 1 # two items -> one joiner; blank/invalid excluded + + +def test_format_memory_block_empty(): + assert format_memory_block([]) == "" + + +def test_format_transcript(): + thread = ZedThread( + id="t1", + title="x", + updated_at="2026-06-10T00:00:00Z", + messages=[ThreadMessage("user", "hello"), ThreadMessage("assistant", "hi there")], + ) + out = format_transcript(thread) + assert "[user]\nhello" in out + assert "[assistant]\nhi there" in out diff --git a/hindsight-integrations/zed/tests/test_daemon.py b/hindsight-integrations/zed/tests/test_daemon.py new file mode 100644 index 000000000..04734dccf --- /dev/null +++ b/hindsight-integrations/zed/tests/test_daemon.py @@ -0,0 +1,164 @@ +"""Daemon flow tests. + +Exercise the recall→inject and capture→retain pipeline against a synthetic +threads.db and a fake Hindsight client (no network, no real Zed). +""" + +import json +import sqlite3 + +import zstandard + +from hindsight_zed.config import ZedConfig +from hindsight_zed.daemon import poll_once, process_thread +from hindsight_zed.rules_file import BEGIN_MARKER +from hindsight_zed.state import DaemonState +from hindsight_zed.threads_db import ThreadMessage, ZedThread + + +class FakeClient: + """Records recall/retain calls; returns canned recall results.""" + + def __init__(self, recall_results=None): + self.recall_results = recall_results or [] + self.recall_calls = [] + self.retain_calls = [] + + def recall(self, bank_id, query, **kw): + self.recall_calls.append((bank_id, query)) + return {"results": self.recall_results} + + def retain(self, bank_id, content, **kw): + self.retain_calls.append((bank_id, content, kw)) + return {"ok": True} + + +def _thread(project, *, id="t1", updated="2026-06-10T10:00:00Z"): + return ZedThread( + id=id, + title="Session", + updated_at=updated, + messages=[ThreadMessage("user", "How do I fix the parser?"), ThreadMessage("assistant", "Guard for empty.")], + folder_paths=[str(project)], + ) + + +def test_process_thread_recalls_and_writes_block(tmp_path): + project = tmp_path / "proj" + project.mkdir() + client = FakeClient(recall_results=[{"text": "The user prefers pytest", "type": "world"}]) + cfg = ZedConfig(bank_prefix="zed") + state = DaemonState(path=tmp_path / "s.json") + + process_thread(_thread(project), client, cfg, state) + + # Recalled against the latest user message, scoped to the project's bank. + assert client.recall_calls + bank, query = client.recall_calls[0] + assert bank == "zed-proj" + assert "parser" in query + # Memory block written into the project's instruction file (.rules here). + rules = (project / ".rules").read_text() + assert BEGIN_MARKER in rules + assert "The user prefers pytest" in rules + # Retained the transcript. + assert client.retain_calls + assert client.retain_calls[0][0] == "zed-proj" + assert "[user]" in client.retain_calls[0][1] + + +def test_process_thread_retain_dedup(tmp_path): + project = tmp_path / "proj" + project.mkdir() + client = FakeClient() + cfg = ZedConfig() + state = DaemonState(path=tmp_path / "s.json") + + process_thread(_thread(project), client, cfg, state) + process_thread(_thread(project), client, cfg, state) # same updated_at → skip retain + assert len(client.retain_calls) == 1 + # But an advanced thread retains again. + process_thread(_thread(project, updated="2026-06-10T11:00:00Z"), client, cfg, state) + assert len(client.retain_calls) == 2 + + +def test_process_thread_skips_when_no_project(tmp_path): + client = FakeClient() + cfg = ZedConfig() + state = DaemonState(path=tmp_path / "s.json") + thread = ZedThread(id="t", title="x", updated_at="2026-06-10T10:00:00Z", + messages=[ThreadMessage("user", "hi")], folder_paths=[]) + process_thread(thread, client, cfg, state) + assert not client.recall_calls and not client.retain_calls + + +def test_auto_recall_off(tmp_path): + project = tmp_path / "proj" + project.mkdir() + client = FakeClient() + cfg = ZedConfig(auto_recall=False) + process_thread(_thread(project), client, cfg, DaemonState(path=tmp_path / "s.json")) + assert not client.recall_calls + assert not (project / ".rules").exists() + assert client.retain_calls # retain still happens + + +def test_recall_failure_does_not_block_retain(tmp_path): + project = tmp_path / "proj" + project.mkdir() + + class Boom(FakeClient): + def recall(self, *a, **k): + raise RuntimeError("server down") + + client = Boom() + process_thread(_thread(project), client, ZedConfig(), DaemonState(path=tmp_path / "s.json")) + assert client.retain_calls # retain still ran despite recall error + + +# ── poll_once against a real synthetic threads.db ───────────────────────────── + + +def _make_threads_db(tmp_path, project): + db = tmp_path / "threads.db" + conn = sqlite3.connect(db) + conn.execute( + "CREATE TABLE threads (id TEXT PRIMARY KEY, summary TEXT, updated_at TEXT, " + "data_type TEXT, data BLOB, folder_paths TEXT)" + ) + doc = { + "version": "0.3.0", + "title": "Session", + "messages": [ + {"User": {"id": "u", "content": [{"Text": "fix the parser"}]}}, + {"Agent": {"content": [{"Text": "guard empties"}], "tool_results": {}}}, + ], + } + obj = zstandard.ZstdCompressor(level=3).compressobj() + data = obj.compress(json.dumps(doc).encode()) + obj.flush() + conn.execute( + "INSERT INTO threads VALUES (?,?,?,?,?,?)", + ("t1", "Session", "2026-06-10T10:00:00Z", "zstd", data, json.dumps([str(project)])), + ) + conn.commit() + conn.close() + return db + + +def test_poll_once_end_to_end(tmp_path): + project = tmp_path / "proj" + project.mkdir() + db = _make_threads_db(tmp_path, project) + client = FakeClient(recall_results=[{"text": "prefers pytest", "type": "world"}]) + cfg = ZedConfig(bank_prefix="zed") + state = DaemonState(path=tmp_path / "s.json") + + high = poll_once(db, client, cfg, state, since=None) + + assert high == "2026-06-10T10:00:00Z" + assert client.recall_calls and client.retain_calls + assert "prefers pytest" in (project / ".rules").read_text() + # Polling again with the high-water mark yields no reprocessing. + client.recall_calls.clear() + poll_once(db, client, cfg, state, since=high) + assert not client.recall_calls diff --git a/hindsight-integrations/zed/tests/test_e2e.py b/hindsight-integrations/zed/tests/test_e2e.py new file mode 100644 index 000000000..0a8572145 --- /dev/null +++ b/hindsight-integrations/zed/tests/test_e2e.py @@ -0,0 +1,66 @@ +"""End-to-end: drive the daemon pipeline against a live Hindsight server. + +Exercises the real recall + retain HTTP path (no real Zed needed — we build a +thread in memory). Gated behind HINDSIGHT_API_URL and marked requires_real_llm, +so it is excluded from the deterministic PR-CI bucket. +""" + +import os +import time +import uuid + +import pytest + +from hindsight_zed.client import HindsightClient +from hindsight_zed.config import ZedConfig +from hindsight_zed.daemon import process_thread +from hindsight_zed.rules_file import BEGIN_MARKER +from hindsight_zed.state import DaemonState +from hindsight_zed.threads_db import ThreadMessage, ZedThread + +pytestmark = pytest.mark.requires_real_llm + +API_URL = os.environ.get("HINDSIGHT_API_URL") + + +@pytest.mark.skipif(not API_URL, reason="HINDSIGHT_API_URL not set") +def test_retain_then_recall_roundtrip(tmp_path): + project = tmp_path / f"zed-e2e-{uuid.uuid4().hex[:8]}" + project.mkdir() + client = HindsightClient(API_URL, os.environ.get("HINDSIGHT_API_TOKEN")) + cfg = ZedConfig(bank_prefix=f"zed-e2e-{uuid.uuid4().hex[:6]}") + state = DaemonState(path=tmp_path / "s.json") + + # Turn 1 — retain a clear, recallable fact (recall has nothing yet). + t1 = ZedThread( + id="e2e-1", + title="prefs", + updated_at="2026-06-10T10:00:00Z", + messages=[ + ThreadMessage("user", "Remember that my favorite language is Haskell."), + ThreadMessage("assistant", "Got it — Haskell."), + ], + folder_paths=[str(project)], + ) + process_thread(t1, client, cfg, state) + + # Turn 2 — after extraction settles, a new thread's recall should surface it + # into the project's instruction file. + found = False + for _ in range(20): + time.sleep(3) + t2 = ZedThread( + id="e2e-2", + title="q", + updated_at="2026-06-10T10:05:00Z", + messages=[ThreadMessage("user", "What is my favorite language?")], + folder_paths=[str(project)], + ) + process_thread(t2, client, cfg, state) + rules = project / ".rules" + if rules.is_file() and "haskell" in rules.read_text().lower(): + assert BEGIN_MARKER in rules.read_text() + found = True + break + + assert found, "retained memory was not recalled into the project's .rules within the timeout" diff --git a/hindsight-integrations/zed/tests/test_state.py b/hindsight-integrations/zed/tests/test_state.py new file mode 100644 index 000000000..41b7ca6d3 --- /dev/null +++ b/hindsight-integrations/zed/tests/test_state.py @@ -0,0 +1,30 @@ +"""Tests for daemon state persistence.""" + +from hindsight_zed.state import DaemonState + + +def test_needs_retain_new_thread(tmp_path): + st = DaemonState(path=tmp_path / "s.json") + assert st.needs_retain("t1", "2026-06-10T00:00:00Z") is True + + +def test_mark_and_skip_unchanged(tmp_path): + st = DaemonState(path=tmp_path / "s.json") + st.mark_retained("t1", "2026-06-10T00:00:00Z") + assert st.needs_retain("t1", "2026-06-10T00:00:00Z") is False + # advanced timestamp → retain again + assert st.needs_retain("t1", "2026-06-10T01:00:00Z") is True + + +def test_persistence_roundtrip(tmp_path): + p = tmp_path / "s.json" + st = DaemonState(path=p) + st.mark_retained("t1", "2026-06-10T00:00:00Z") + st.save() + reloaded = DaemonState.load(p) + assert reloaded.needs_retain("t1", "2026-06-10T00:00:00Z") is False + + +def test_load_missing_file(tmp_path): + st = DaemonState.load(tmp_path / "nope.json") + assert st.retained == {} diff --git a/scripts/release-integration.sh b/scripts/release-integration.sh index 97c1b7c77..e747aa7f3 100755 --- a/scripts/release-integration.sh +++ b/scripts/release-integration.sh @@ -13,7 +13,7 @@ print_info() { echo -e "${GREEN}[INFO]${NC} $1"; } print_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } print_error() { echo -e "${RED}[ERROR]${NC} $1"; } -VALID_INTEGRATIONS=("ag2" "agentcore" "agno" "ai-sdk" "autogen" "chat" "claude-agent-sdk" "claude-code" "cline" "cloudflare-oauth-proxy" "codex" "crewai" "cursor" "cursor-cli" "dify" "flowise" "gemini-spark" "google-adk" "haystack" "langgraph" "litellm" "llamaindex" "n8n" "nemoclaw" "obsidian" "omo" "openai-agents" "openclaw" "opencode" "paperclip" "pipecat" "pydantic-ai" "roo-code" "smolagents" "strands" "superagent" "vapi") +VALID_INTEGRATIONS=("ag2" "agentcore" "agno" "ai-sdk" "autogen" "chat" "claude-agent-sdk" "claude-code" "cline" "cloudflare-oauth-proxy" "codex" "crewai" "cursor" "cursor-cli" "dify" "flowise" "gemini-spark" "google-adk" "haystack" "langgraph" "litellm" "llamaindex" "n8n" "nemoclaw" "obsidian" "omo" "openai-agents" "openclaw" "opencode" "paperclip" "pipecat" "pydantic-ai" "roo-code" "smolagents" "strands" "superagent" "vapi" "zed") usage() { print_error "Usage: $0 " From 7a6107015c5de9e7e1de440d1f1ad93e75b78f6b Mon Sep 17 00:00:00 2001 From: DK09876 Date: Thu, 11 Jun 2026 12:28:44 -0700 Subject: [PATCH 3/7] feat(zed): use official Zed brand logo for the gallery icon Replace the placeholder SVG with Zed's official logo (assets/images/zed_logo.svg from zed-industries/zed). --- hindsight-docs/static/img/icons/zed.svg | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/hindsight-docs/static/img/icons/zed.svg b/hindsight-docs/static/img/icons/zed.svg index 589bdafb1..d1769449c 100644 --- a/hindsight-docs/static/img/icons/zed.svg +++ b/hindsight-docs/static/img/icons/zed.svg @@ -1 +1,10 @@ - + + + + + + + + + + From cac15705892e05409b68ed7e14653cae62070dd3 Mon Sep 17 00:00:00 2001 From: DK09876 Date: Thu, 11 Jun 2026 12:48:39 -0700 Subject: [PATCH 4/7] fix(zed): drop changelog link (no changelog page until first release) The /changelog/integrations/zed page is generated by the release script when a version is cut; linking it before release broke the docs build (broken link). --- hindsight-docs/docs-integrations/zed.md | 1 - 1 file changed, 1 deletion(-) diff --git a/hindsight-docs/docs-integrations/zed.md b/hindsight-docs/docs-integrations/zed.md index 57a1dc970..82bb3f905 100644 --- a/hindsight-docs/docs-integrations/zed.md +++ b/hindsight-docs/docs-integrations/zed.md @@ -8,7 +8,6 @@ description: "Add automatic long-term memory to the Zed editor's AI assistant wi Automatic, always-on long-term memory for the [Zed](https://zed.dev) editor's AI assistant, powered by [Hindsight](https://vectorize.io/hindsight). When you chat with Zed's Agent Panel, relevant memory from past sessions on that project is injected automatically — no manual tool calls — and your conversations are retained so the next session builds on them. -[View Changelog →](/changelog/integrations/zed) ## How It Works From 21eb1e262d8b6baf1b04f98bcde2e85c6c8fb170 Mon Sep 17 00:00:00 2001 From: DK09876 Date: Thu, 11 Jun 2026 12:58:53 -0700 Subject: [PATCH 5/7] fix(zed): surface auth/connection failures at WARNING (review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recall/retain errors were swallowed at DEBUG, so a wrong API token or an unreachable server failed silently — the user just saw no memory in Zed with no hint why. Now auth (401/403) and connection/5xx errors escalate to WARNING in the daemon log, deduped per bank (the 5s poll loop won't spam) and re-armed on a later success. Client raises a typed HindsightHTTPError carrying the status code. --- .../zed/hindsight_zed/client.py | 14 ++- .../zed/hindsight_zed/daemon.py | 52 ++++++++- .../zed/hindsight_zed/state.py | 3 + .../zed/tests/test_daemon.py | 110 ++++++++++++++++++ 4 files changed, 175 insertions(+), 4 deletions(-) diff --git a/hindsight-integrations/zed/hindsight_zed/client.py b/hindsight-integrations/zed/hindsight_zed/client.py index ed64e322c..03accb517 100644 --- a/hindsight-integrations/zed/hindsight_zed/client.py +++ b/hindsight-integrations/zed/hindsight_zed/client.py @@ -16,6 +16,18 @@ HEALTH_CHECK_DELAY = 2 # seconds +class HindsightHTTPError(RuntimeError): + """An HTTP error response from the Hindsight API, carrying the status code. + + Lets callers distinguish auth failures (401/403) from transient errors so + they can be surfaced more loudly. + """ + + def __init__(self, status_code: int, url: str, body: str): + self.status_code = status_code + super().__init__(f"HTTP {status_code} from {url}: {body}") + + def _validate_api_url(url: str) -> str: """Validate and normalize the API URL. Reject non-HTTP schemes.""" parsed = urllib.parse.urlparse(url) @@ -52,7 +64,7 @@ def _request(self, method: str, path: str, body: Optional[dict] = None, timeout: body_text = e.read().decode() except Exception: pass - raise RuntimeError(f"HTTP {e.code} from {url}: {body_text}") from e + raise HindsightHTTPError(e.code, url, body_text) from e def health_check(self, timeout: int = 5) -> bool: """Return True if the Hindsight server is reachable (retries a few times).""" diff --git a/hindsight-integrations/zed/hindsight_zed/daemon.py b/hindsight-integrations/zed/hindsight_zed/daemon.py index b5c16a98b..a5f23f883 100644 --- a/hindsight-integrations/zed/hindsight_zed/daemon.py +++ b/hindsight-integrations/zed/hindsight_zed/daemon.py @@ -16,11 +16,12 @@ import logging import time +import urllib.error from pathlib import Path from typing import Optional from .bank import bank_id_for_thread_paths -from .client import HindsightClient +from .client import HindsightClient, HindsightHTTPError from .config import ZedConfig, load_config from .content import compose_recall_query, format_memory_block, format_transcript from .rules_file import write_memory_block @@ -29,6 +30,49 @@ logger = logging.getLogger("hindsight_zed.daemon") +# Connection-level failures that mean "can't reach the server". +_CONNECTION_ERRORS = (urllib.error.URLError, ConnectionError, TimeoutError, OSError) + + +def _log_api_error(op: str, bank_id: str, exc: Exception, state: DaemonState) -> None: + """Log an API failure, escalating auth/connection errors to WARNING. + + A wrong token or unreachable server otherwise fails silently — the user just + sees no memory in Zed. Those two cases are surfaced at WARNING (once per + bank, until a later success clears it, so the poll loop doesn't spam the + log); everything else stays at DEBUG to avoid noise from transient blips. + """ + if isinstance(exc, HindsightHTTPError) and exc.status_code in (401, 403): + key = ("auth", bank_id) + if key not in state.warned: + logger.warning( + "Hindsight rejected %s for bank %s (HTTP %s) — check your " + "HINDSIGHT_API_TOKEN (~/.hindsight/zed.json). Memory will not " + "work until this is fixed.", + op, bank_id, exc.status_code, + ) + state.warned.add(key) + return + if isinstance(exc, _CONNECTION_ERRORS) or ( + isinstance(exc, HindsightHTTPError) and exc.status_code >= 500 + ): + key = ("conn", bank_id) + if key not in state.warned: + logger.warning( + "Hindsight %s could not reach the server for bank %s: %s — " + "check the API URL and your network.", + op, bank_id, exc, + ) + state.warned.add(key) + return + logger.debug("%s failed for bank %s: %s", op, bank_id, exc) + + +def _clear_warnings(bank_id: str, state: DaemonState) -> None: + """A successful call clears prior warnings so a later failure re-surfaces.""" + state.warned.discard(("auth", bank_id)) + state.warned.discard(("conn", bank_id)) + def _project_dir(thread: ZedThread) -> Optional[Path]: """Return the first existing project folder for a thread, if any.""" @@ -66,8 +110,9 @@ def process_thread( ) block = format_memory_block(resp.get("results", [])) write_memory_block(project, block, preamble=config.recall_preamble) + _clear_warnings(bank_id, state) except Exception as e: - logger.debug("recall failed for bank %s: %s", bank_id, e) + _log_api_error("recall", bank_id, e, state) # ── Auto-retain → store the transcript ──────────────────────────────────── if config.auto_retain and state.needs_retain(thread.id, thread.updated_at): @@ -83,8 +128,9 @@ def process_thread( metadata={"source": "zed", "thread_id": thread.id}, ) state.mark_retained(thread.id, thread.updated_at) + _clear_warnings(bank_id, state) except Exception as e: - logger.debug("retain failed for bank %s: %s", bank_id, e) + _log_api_error("retain", bank_id, e, state) def poll_once( diff --git a/hindsight-integrations/zed/hindsight_zed/state.py b/hindsight-integrations/zed/hindsight_zed/state.py index 11e4c82b0..93eaa8bd6 100644 --- a/hindsight-integrations/zed/hindsight_zed/state.py +++ b/hindsight-integrations/zed/hindsight_zed/state.py @@ -18,6 +18,9 @@ class DaemonState: retained: dict = field(default_factory=dict) # thread_id -> updated_at string path: Path = DEFAULT_STATE_FILE + # Runtime-only (not persisted): which (kind, bank) error warnings we've + # already emitted, so a persistent failure isn't logged every poll. + warned: set = field(default_factory=set) @classmethod def load(cls, path: Path = DEFAULT_STATE_FILE) -> "DaemonState": diff --git a/hindsight-integrations/zed/tests/test_daemon.py b/hindsight-integrations/zed/tests/test_daemon.py index 04734dccf..52123180c 100644 --- a/hindsight-integrations/zed/tests/test_daemon.py +++ b/hindsight-integrations/zed/tests/test_daemon.py @@ -162,3 +162,113 @@ def test_poll_once_end_to_end(tmp_path): client.recall_calls.clear() poll_once(db, client, cfg, state, since=high) assert not client.recall_calls + +# ── Auth/connection error escalation ────────────────────────────────────────── + +import logging # noqa: E402 + +from hindsight_zed.client import HindsightHTTPError # noqa: E402 + + +class AuthFailClient(FakeClient): + """A bad token fails *every* call (401/403), like the real server.""" + + def __init__(self, status=401): + super().__init__() + self.status = status + + def recall(self, *a, **k): + raise HindsightHTTPError(self.status, "http://x/recall", "denied") + + def retain(self, *a, **k): + raise HindsightHTTPError(self.status, "http://x/retain", "denied") + + +def test_auth_error_escalated_to_warning(tmp_path, caplog): + project = tmp_path / "proj" + project.mkdir() + state = DaemonState(path=tmp_path / "s.json") + with caplog.at_level(logging.WARNING, logger="hindsight_zed.daemon"): + process_thread(_thread(project), AuthFailClient(401), ZedConfig(), state) + assert any("HINDSIGHT_API_TOKEN" in r.getMessage() for r in caplog.records) + assert ("auth", "zed-proj") in state.warned + + +def test_auth_warning_deduped_across_polls(tmp_path, caplog): + project = tmp_path / "proj" + project.mkdir() + client = AuthFailClient(403) + state = DaemonState(path=tmp_path / "s.json") + with caplog.at_level(logging.WARNING, logger="hindsight_zed.daemon"): + process_thread(_thread(project), client, ZedConfig(), state) + process_thread(_thread(project, id="t2"), client, ZedConfig(), state) + # One WARNING total despite recall+retain failing on two polls (4 failures). + warns = [r for r in caplog.records if r.levelno >= logging.WARNING] + assert len(warns) == 1 + + +def test_connection_error_escalated(tmp_path, caplog): + import urllib.error + + project = tmp_path / "proj" + project.mkdir() + + class Down(FakeClient): + def recall(self, *a, **k): + raise urllib.error.URLError("connection refused") + + def retain(self, *a, **k): + raise urllib.error.URLError("connection refused") + + state = DaemonState(path=tmp_path / "s.json") + with caplog.at_level(logging.WARNING, logger="hindsight_zed.daemon"): + process_thread(_thread(project), Down(), ZedConfig(), state) + assert any("could not reach" in r.getMessage() for r in caplog.records) + assert ("conn", "zed-proj") in state.warned + + +def test_transient_error_stays_debug(tmp_path, caplog): + project = tmp_path / "proj" + project.mkdir() + + class Boom(FakeClient): + def recall(self, *a, **k): + raise RuntimeError("weird transient") + + def retain(self, *a, **k): + raise RuntimeError("weird transient") + + state = DaemonState(path=tmp_path / "s.json") + with caplog.at_level(logging.WARNING, logger="hindsight_zed.daemon"): + process_thread(_thread(project), Boom(), ZedConfig(), state) + assert not any(r.levelno >= logging.WARNING for r in caplog.records) + assert not state.warned + + +def test_warning_cleared_on_recovery(tmp_path): + project = tmp_path / "proj" + project.mkdir() + state = DaemonState(path=tmp_path / "s.json") + + class Flappy(FakeClient): + def __init__(self): + super().__init__(recall_results=[{"text": "ok", "type": "world"}]) + self.fail = True + + def recall(self, *a, **k): + if self.fail: + raise HindsightHTTPError(401, "http://x", "no") + return super().recall(*a, **k) + + def retain(self, *a, **k): + if self.fail: + raise HindsightHTTPError(401, "http://x", "no") + return super().retain(*a, **k) + + client = Flappy() + process_thread(_thread(project), client, ZedConfig(), state) + assert ("auth", "zed-proj") in state.warned + # User fixes the token → both calls succeed → warning state cleared. + client.fail = False + process_thread(_thread(project, id="t2"), client, ZedConfig(), state) + assert ("auth", "zed-proj") not in state.warned From c36a9ebbe9359942d3774277a729bee264eed3f7 Mon Sep 17 00:00:00 2001 From: DK09876 Date: Thu, 11 Jun 2026 13:15:02 -0700 Subject: [PATCH 6/7] style(zed): apply ruff format (repo-wide formatter) verify-generated-files runs 'ruff format' across the repo (separate from 'ruff check'); reformat the 5 files it rewrote so the no-uncommitted-changes gate passes. --- .../zed/hindsight_zed/daemon.py | 15 ++++++++------- .../zed/hindsight_zed/threads_db.py | 12 ++++++------ hindsight-integrations/zed/tests/test_cli.py | 8 ++++++-- hindsight-integrations/zed/tests/test_daemon.py | 6 ++++-- .../zed/tests/test_threads_db.py | 11 +++++------ 5 files changed, 29 insertions(+), 23 deletions(-) diff --git a/hindsight-integrations/zed/hindsight_zed/daemon.py b/hindsight-integrations/zed/hindsight_zed/daemon.py index a5f23f883..ca92209a0 100644 --- a/hindsight-integrations/zed/hindsight_zed/daemon.py +++ b/hindsight-integrations/zed/hindsight_zed/daemon.py @@ -49,19 +49,20 @@ def _log_api_error(op: str, bank_id: str, exc: Exception, state: DaemonState) -> "Hindsight rejected %s for bank %s (HTTP %s) — check your " "HINDSIGHT_API_TOKEN (~/.hindsight/zed.json). Memory will not " "work until this is fixed.", - op, bank_id, exc.status_code, + op, + bank_id, + exc.status_code, ) state.warned.add(key) return - if isinstance(exc, _CONNECTION_ERRORS) or ( - isinstance(exc, HindsightHTTPError) and exc.status_code >= 500 - ): + if isinstance(exc, _CONNECTION_ERRORS) or (isinstance(exc, HindsightHTTPError) and exc.status_code >= 500): key = ("conn", bank_id) if key not in state.warned: logger.warning( - "Hindsight %s could not reach the server for bank %s: %s — " - "check the API URL and your network.", - op, bank_id, exc, + "Hindsight %s could not reach the server for bank %s: %s — check the API URL and your network.", + op, + bank_id, + exc, ) state.warned.add(key) return diff --git a/hindsight-integrations/zed/hindsight_zed/threads_db.py b/hindsight-integrations/zed/hindsight_zed/threads_db.py index 84417ad5e..4774a20d1 100644 --- a/hindsight-integrations/zed/hindsight_zed/threads_db.py +++ b/hindsight-integrations/zed/hindsight_zed/threads_db.py @@ -112,9 +112,7 @@ def _decompress(data: bytes, data_type: str) -> str: # Fallback so the package has no hard runtime dep: a zstd frame can be # decoded by zlib only if it is not actually zstd. We never expect this # path in practice, but raise a clear error rather than silently fail. - raise RuntimeError( - "thread is zstd-compressed but the 'zstandard' package is not installed" - ) + raise RuntimeError("thread is zstd-compressed but the 'zstandard' package is not installed") raise ValueError(f"unknown thread data_type: {data_type!r}") @@ -268,9 +266,11 @@ def read_threads(db_path: Path, since: Optional[str] = None) -> list[ZedThread]: try: cols = {row[1] for row in conn.execute("PRAGMA table_info(threads)")} has_folders = "folder_paths" in cols - select = "SELECT id, summary, updated_at, data_type, data" + ( - ", folder_paths" if has_folders else "" - ) + " FROM threads" + select = ( + "SELECT id, summary, updated_at, data_type, data" + + (", folder_paths" if has_folders else "") + + " FROM threads" + ) params: tuple = () if since is not None: select += " WHERE updated_at > ?" diff --git a/hindsight-integrations/zed/tests/test_cli.py b/hindsight-integrations/zed/tests/test_cli.py index f5a4d3201..043be5bc9 100644 --- a/hindsight-integrations/zed/tests/test_cli.py +++ b/hindsight-integrations/zed/tests/test_cli.py @@ -12,12 +12,16 @@ def fake_home(tmp_path, monkeypatch): monkeypatch.setattr(cli, "USER_CONFIG_FILE", tmp_path / ".hindsight" / "zed.json") # Don't actually touch launchd/systemd or the network during init. monkeypatch.setattr(cli, "_install_daemon", lambda: None) - monkeypatch.setattr(cli, "load_config", lambda: type("C", (), {"hindsight_api_url": "x", "hindsight_api_token": None})()) + monkeypatch.setattr( + cli, "load_config", lambda: type("C", (), {"hindsight_api_url": "x", "hindsight_api_token": None})() + ) return tmp_path def test_init_writes_config(fake_home, monkeypatch, capsys): - monkeypatch.setattr("sys.argv", ["hindsight-zed", "init", "--api-token", "tok", "--api-url", "http://localhost:8888"]) + monkeypatch.setattr( + "sys.argv", ["hindsight-zed", "init", "--api-token", "tok", "--api-url", "http://localhost:8888"] + ) cli.main() written = json.loads(cli.USER_CONFIG_FILE.read_text()) assert written["hindsightApiToken"] == "tok" diff --git a/hindsight-integrations/zed/tests/test_daemon.py b/hindsight-integrations/zed/tests/test_daemon.py index 52123180c..ec209cf0f 100644 --- a/hindsight-integrations/zed/tests/test_daemon.py +++ b/hindsight-integrations/zed/tests/test_daemon.py @@ -86,8 +86,9 @@ def test_process_thread_skips_when_no_project(tmp_path): client = FakeClient() cfg = ZedConfig() state = DaemonState(path=tmp_path / "s.json") - thread = ZedThread(id="t", title="x", updated_at="2026-06-10T10:00:00Z", - messages=[ThreadMessage("user", "hi")], folder_paths=[]) + thread = ZedThread( + id="t", title="x", updated_at="2026-06-10T10:00:00Z", messages=[ThreadMessage("user", "hi")], folder_paths=[] + ) process_thread(thread, client, cfg, state) assert not client.recall_calls and not client.retain_calls @@ -163,6 +164,7 @@ def test_poll_once_end_to_end(tmp_path): poll_once(db, client, cfg, state, since=high) assert not client.recall_calls + # ── Auth/connection error escalation ────────────────────────────────────────── import logging # noqa: E402 diff --git a/hindsight-integrations/zed/tests/test_threads_db.py b/hindsight-integrations/zed/tests/test_threads_db.py index d6fa14c48..4bf8ab968 100644 --- a/hindsight-integrations/zed/tests/test_threads_db.py +++ b/hindsight-integrations/zed/tests/test_threads_db.py @@ -60,8 +60,7 @@ def _insert(db: Path, *, id, summary, updated_at, doc, compress=True, folder_pat data, data_type = raw, "json" conn = sqlite3.connect(db) conn.execute( - "INSERT INTO threads (id, summary, updated_at, data_type, data, folder_paths) " - "VALUES (?, ?, ?, ?, ?, ?)", + "INSERT INTO threads (id, summary, updated_at, data_type, data, folder_paths) VALUES (?, ?, ?, ?, ?, ?)", (id, summary, updated_at, data_type, data, json.dumps(folder_paths) if folder_paths else None), ) conn.commit() @@ -158,8 +157,9 @@ def test_malformed_json_returns_none(): def test_read_threads_mixed_versions(tmp_path): db = _make_db(tmp_path) - _insert(db, id="a", summary="cur", updated_at="2026-06-10T10:00:00Z", doc=CURRENT_DOC, - folder_paths=["/Users/me/proj-a"]) + _insert( + db, id="a", summary="cur", updated_at="2026-06-10T10:00:00Z", doc=CURRENT_DOC, folder_paths=["/Users/me/proj-a"] + ) _insert(db, id="b", summary="leg", updated_at="2026-06-10T09:00:00Z", doc=LEGACY_020_DOC) _insert(db, id="c", summary="old", updated_at="2026-06-10T08:00:00Z", doc=OLDEST_DOC) @@ -194,8 +194,7 @@ def test_read_threads_missing_db_returns_empty(tmp_path): def test_read_threads_unknown_version_skipped_at_db_level(tmp_path): db = _make_db(tmp_path) - _insert(db, id="future", summary="f", updated_at="2026-06-10T10:00:00Z", - doc={"version": "9.9.9", "messages": []}) + _insert(db, id="future", summary="f", updated_at="2026-06-10T10:00:00Z", doc={"version": "9.9.9", "messages": []}) _insert(db, id="ok", summary="g", updated_at="2026-06-10T11:00:00Z", doc=CURRENT_DOC) threads = read_threads(db) assert [t.id for t in threads] == ["ok"] From b1d44ee6c3651981c64490f538a4226ad735c3fb Mon Sep 17 00:00:00 2001 From: DK09876 Date: Thu, 11 Jun 2026 16:27:27 -0700 Subject: [PATCH 7/7] feat(zed): debounce retain until a conversation goes idle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Zed has no 'thread finished' event, so the daemon previously retained on every threads.db update — re-retaining the full transcript each turn and risking mid-stream snapshots. Add RetainDebouncer: a thread is retained once its updated_at has been stable for retain_idle_seconds (default 45s), collapsing a multi-turn conversation into a single retain of its settled state. Recall stays eager (memory should be as fresh as Zed allows); only retain is debounced. +3 tests (idle hold, timer reset on new revision, poll-level debounce). --- .../zed/hindsight_zed/config.py | 7 + .../zed/hindsight_zed/daemon.py | 151 ++++++++++++------ hindsight-integrations/zed/settings.json | 1 + .../zed/tests/test_daemon.py | 74 ++++++++- 4 files changed, 181 insertions(+), 52 deletions(-) diff --git a/hindsight-integrations/zed/hindsight_zed/config.py b/hindsight-integrations/zed/hindsight_zed/config.py index 298053579..6c1926dc8 100644 --- a/hindsight-integrations/zed/hindsight_zed/config.py +++ b/hindsight-integrations/zed/hindsight_zed/config.py @@ -44,6 +44,11 @@ class ZedConfig: auto_retain: bool = True retain_context: str = "zed" retain_tags: list = field(default_factory=list) + # A thread is retained once its updated_at has been stable for this many + # seconds — Zed has no "conversation finished" signal, so we approximate it + # with "the exchange has gone idle". Avoids re-retaining every turn and + # capturing mid-stream snapshots. + retain_idle_seconds: float = 45.0 # Daemon poll_interval: float = 5.0 # seconds between threads.db polls @@ -66,6 +71,7 @@ class ZedConfig: "autoRetain": ("auto_retain", bool), "retainContext": ("retain_context", str), "retainTags": ("retain_tags", list), + "retainIdleSeconds": ("retain_idle_seconds", float), "pollInterval": ("poll_interval", float), "debug": ("debug", bool), } @@ -82,6 +88,7 @@ class ZedConfig: "HINDSIGHT_ZED_RECALL_BUDGET": ("recall_budget", str), "HINDSIGHT_ZED_RECALL_MAX_TOKENS": ("recall_max_tokens", int), "HINDSIGHT_ZED_RETAIN_CONTEXT": ("retain_context", str), + "HINDSIGHT_ZED_RETAIN_IDLE_SECONDS": ("retain_idle_seconds", float), "HINDSIGHT_ZED_POLL_INTERVAL": ("poll_interval", float), "HINDSIGHT_ZED_DEBUG": ("debug", bool), } diff --git a/hindsight-integrations/zed/hindsight_zed/daemon.py b/hindsight-integrations/zed/hindsight_zed/daemon.py index ca92209a0..1f25a90f8 100644 --- a/hindsight-integrations/zed/hindsight_zed/daemon.py +++ b/hindsight-integrations/zed/hindsight_zed/daemon.py @@ -84,54 +84,99 @@ def _project_dir(thread: ZedThread) -> Optional[Path]: return None -def process_thread( - thread: ZedThread, - client: HindsightClient, - config: ZedConfig, - state: DaemonState, -) -> None: - """Run recall (inject) and retain (capture) for a single updated thread.""" +def do_recall(thread: ZedThread, client: HindsightClient, config: ZedConfig, state: DaemonState) -> None: + """Recall memory for a thread and rewrite its project's instruction block. + + Runs eagerly on every change (we want injected memory as fresh as Zed lets + us), unlike retain which is debounced until the conversation goes idle. + """ + if not (config.auto_recall and thread.messages): + return project = _project_dir(thread) bank_id = bank_id_for_thread_paths(thread.folder_paths, config) if not bank_id or project is None: - # No on-disk project to scope a bank or write a rules file to — skip. return + query = compose_recall_query(thread.messages, config.recall_max_query_chars) + if not query: + return + try: + resp = client.recall( + bank_id, + query, + max_tokens=config.recall_max_tokens, + budget=config.recall_budget, + types=config.recall_types, + ) + block = format_memory_block(resp.get("results", [])) + write_memory_block(project, block, preamble=config.recall_preamble) + _clear_warnings(bank_id, state) + except Exception as e: + _log_api_error("recall", bank_id, e, state) + + +def do_retain(thread: ZedThread, client: HindsightClient, config: ZedConfig, state: DaemonState) -> None: + """Retain a thread's transcript (idempotent — skips if already at this revision).""" + if not (config.auto_retain and state.needs_retain(thread.id, thread.updated_at)): + return + bank_id = bank_id_for_thread_paths(thread.folder_paths, config) + if not bank_id: + return + transcript = format_transcript(thread) + if not transcript.strip(): + return + try: + client.retain( + bank_id, + transcript, + document_id=f"zed-thread-{thread.id}", + context=config.retain_context, + tags=config.retain_tags, + metadata={"source": "zed", "thread_id": thread.id}, + ) + state.mark_retained(thread.id, thread.updated_at) + _clear_warnings(bank_id, state) + except Exception as e: + _log_api_error("retain", bank_id, e, state) + + +def process_thread(thread: ZedThread, client: HindsightClient, config: ZedConfig, state: DaemonState) -> None: + """Recall then retain a thread immediately (no debounce). + + Convenience for one-shot use; the live daemon debounces retain via + :class:`RetainDebouncer` so it captures a settled conversation once. + """ + do_recall(thread, client, config, state) + do_retain(thread, client, config, state) + + +class RetainDebouncer: + """Defers a thread's retain until its ``updated_at`` has been idle. + + Zed has no "conversation finished" event, so we approximate it: each time a + thread changes we (re)start its timer; once it's been quiet for + ``idle_seconds`` it becomes due for retain. This collapses a multi-turn + conversation into one retain and avoids capturing mid-stream snapshots. + """ + + def __init__(self, idle_seconds: float, clock=time.monotonic): + self.idle_seconds = idle_seconds + self._clock = clock + # thread_id -> (latest thread object, updated_at, last_change_time) + self._pending: dict[str, tuple[ZedThread, str, float]] = {} + + def note(self, thread: ZedThread) -> None: + """Record a thread sighting, resetting its idle timer if it advanced.""" + prev = self._pending.get(thread.id) + if prev is None or prev[1] != thread.updated_at: + self._pending[thread.id] = (thread, thread.updated_at, self._clock()) - # ── Auto-recall → rewrite the project's memory block ────────────────────── - if config.auto_recall and thread.messages: - query = compose_recall_query(thread.messages, config.recall_max_query_chars) - if query: - try: - resp = client.recall( - bank_id, - query, - max_tokens=config.recall_max_tokens, - budget=config.recall_budget, - types=config.recall_types, - ) - block = format_memory_block(resp.get("results", [])) - write_memory_block(project, block, preamble=config.recall_preamble) - _clear_warnings(bank_id, state) - except Exception as e: - _log_api_error("recall", bank_id, e, state) - - # ── Auto-retain → store the transcript ──────────────────────────────────── - if config.auto_retain and state.needs_retain(thread.id, thread.updated_at): - transcript = format_transcript(thread) - if transcript.strip(): - try: - client.retain( - bank_id, - transcript, - document_id=f"zed-thread-{thread.id}", - context=config.retain_context, - tags=config.retain_tags, - metadata={"source": "zed", "thread_id": thread.id}, - ) - state.mark_retained(thread.id, thread.updated_at) - _clear_warnings(bank_id, state) - except Exception as e: - _log_api_error("retain", bank_id, e, state) + def due(self) -> list[ZedThread]: + """Return (and drop) threads that have been idle ≥ ``idle_seconds``.""" + now = self._clock() + ready = [(tid, th) for tid, (th, _, t) in self._pending.items() if now - t >= self.idle_seconds] + for tid, _ in ready: + del self._pending[tid] + return [th for _, th in ready] def poll_once( @@ -140,17 +185,22 @@ def poll_once( config: ZedConfig, state: DaemonState, since: Optional[str], + debouncer: RetainDebouncer, ) -> Optional[str]: - """Process all threads updated since ``since``. Returns the new high-water mark.""" + """One poll: recall changed threads eagerly, retain ones that have gone idle. + + Returns the new high-water mark. + """ threads = read_threads(db_path, since=since) - if not threads: - return since - threads.sort(key=lambda t: t.updated_at) high = since - for thread in threads: - process_thread(thread, client, config, state) + for thread in sorted(threads, key=lambda t: t.updated_at): + do_recall(thread, client, config, state) + debouncer.note(thread) if high is None or thread.updated_at > high: high = thread.updated_at + # Retain any conversation that has settled (idle past the window). + for thread in debouncer.due(): + do_retain(thread, client, config, state) state.save() return high @@ -164,6 +214,7 @@ def run(db_path: Optional[Path] = None, config: Optional[ZedConfig] = None) -> N client = HindsightClient(config.hindsight_api_url, config.hindsight_api_token) state = DaemonState.load() + debouncer = RetainDebouncer(config.retain_idle_seconds) # Start from the newest already-retained revision so we don't reprocess the # entire backlog on first run (recall would still refresh on the next turn). since: Optional[str] = max(state.retained.values(), default=None) @@ -171,7 +222,7 @@ def run(db_path: Optional[Path] = None, config: Optional[ZedConfig] = None) -> N logger.info("hindsight-zed daemon started (db=%s, api=%s)", db_path, config.hindsight_api_url) while True: try: - since = poll_once(db_path, client, config, state, since) + since = poll_once(db_path, client, config, state, since, debouncer) except Exception as e: # never let one bad poll kill the daemon logger.debug("poll error: %s", e) time.sleep(config.poll_interval) diff --git a/hindsight-integrations/zed/settings.json b/hindsight-integrations/zed/settings.json index 46454e73b..fcf2bf0b6 100644 --- a/hindsight-integrations/zed/settings.json +++ b/hindsight-integrations/zed/settings.json @@ -8,6 +8,7 @@ "recallMaxTokens": 1024, "autoRetain": true, "retainContext": "zed", + "retainIdleSeconds": 45.0, "pollInterval": 5.0, "debug": false } diff --git a/hindsight-integrations/zed/tests/test_daemon.py b/hindsight-integrations/zed/tests/test_daemon.py index ec209cf0f..b03106fc3 100644 --- a/hindsight-integrations/zed/tests/test_daemon.py +++ b/hindsight-integrations/zed/tests/test_daemon.py @@ -147,21 +147,25 @@ def _make_threads_db(tmp_path, project): def test_poll_once_end_to_end(tmp_path): + from hindsight_zed.daemon import RetainDebouncer + project = tmp_path / "proj" project.mkdir() db = _make_threads_db(tmp_path, project) client = FakeClient(recall_results=[{"text": "prefers pytest", "type": "world"}]) cfg = ZedConfig(bank_prefix="zed") state = DaemonState(path=tmp_path / "s.json") + # idle=0 → retain becomes due immediately, so this exercises the full path. + deb = RetainDebouncer(idle_seconds=0) - high = poll_once(db, client, cfg, state, since=None) + high = poll_once(db, client, cfg, state, since=None, debouncer=deb) assert high == "2026-06-10T10:00:00Z" assert client.recall_calls and client.retain_calls assert "prefers pytest" in (project / ".rules").read_text() # Polling again with the high-water mark yields no reprocessing. client.recall_calls.clear() - poll_once(db, client, cfg, state, since=high) + poll_once(db, client, cfg, state, since=high, debouncer=deb) assert not client.recall_calls @@ -274,3 +278,69 @@ def retain(self, *a, **k): client.fail = False process_thread(_thread(project, id="t2"), client, ZedConfig(), state) assert ("auth", "zed-proj") not in state.warned + + +# ── Retain debounce (idle detection) ────────────────────────────────────────── + +from hindsight_zed.daemon import RetainDebouncer, do_recall # noqa: E402 + + +class FakeClock: + def __init__(self): + self.t = 0.0 + + def __call__(self): + return self.t + + def advance(self, dt): + self.t += dt + + +def test_debouncer_holds_until_idle(): + clk = FakeClock() + deb = RetainDebouncer(idle_seconds=45, clock=clk) + t = ZedThread(id="t1", title="x", updated_at="A", messages=[], folder_paths=[]) + deb.note(t) + assert deb.due() == [] # just changed → not due + clk.advance(30) + assert deb.due() == [] # still within window + clk.advance(20) # now 50s since change + assert [x.id for x in deb.due()] == ["t1"] + assert deb.due() == [] # consumed + + +def test_debouncer_resets_timer_on_new_update(): + clk = FakeClock() + deb = RetainDebouncer(idle_seconds=45, clock=clk) + deb.note(ZedThread(id="t1", title="x", updated_at="A", messages=[], folder_paths=[])) + clk.advance(40) + # A new revision arrives before the window elapses → timer resets. + deb.note(ZedThread(id="t1", title="x", updated_at="B", messages=[], folder_paths=[])) + clk.advance(40) # 40s since the *latest* change → still not due + assert deb.due() == [] + clk.advance(10) + due = deb.due() + assert [x.id for x in due] == ["t1"] + assert due[0].updated_at == "B" # retains the latest revision + + +def test_poll_debounces_retain(tmp_path): + # A live conversation should NOT be retained mid-flight, only once idle. + project = tmp_path / "proj" + project.mkdir() + db = _make_threads_db(tmp_path, project) + client = FakeClient(recall_results=[{"text": "x", "type": "world"}]) + cfg = ZedConfig(bank_prefix="zed") + state = DaemonState(path=tmp_path / "s.json") + clk = FakeClock() + deb = RetainDebouncer(idle_seconds=45, clock=clk) + + # First poll: recall happens, but retain is deferred (just changed). + poll_once(db, client, cfg, state, since=None, debouncer=deb) + assert client.recall_calls + assert not client.retain_calls # not idle yet + + # Poll again after the idle window with no new changes → now retained once. + clk.advance(60) + poll_once(db, client, cfg, state, since="2026-06-10T10:00:00Z", debouncer=deb) + assert len(client.retain_calls) == 1