Implement DiffKit repo search MVP on Cloudflare + Livegrep

.dockerignore

@@ -0,0 +1,11 @@
+node_modules
+.git
+.turbo
+**/node_modules
+**/.next
+**/dist
+**/coverage
+**/.env
+**/.env.*
+apps/dashboard/.wrangler
+apps/dashboard/.dev.vars*

README.md

@@ -212,6 +212,27 @@ The dev server uses `DEV_TUNNEL_URL` to allow the tunnel host and configure HMR
 | `pnpm --filter dashboard test` | Run tests |
 | `pnpm --filter dashboard deploy` | Build and deploy to Cloudflare Workers |
 
+### Search-service scripts
+
+| Command | Description |
+|---------|------------|
+| `pnpm --filter @diffkit/search dev` | Run self-hosted search service locally |
+| `pnpm --filter @diffkit/search start` | Start self-hosted search service |
+| `pnpm --filter @diffkit/search check-types` | Type-check search service |
+| `pnpm --filter @diffkit/search check` | Lint/format-check search service |
+
+## Self-hosted repo search service (Livegrep + apps/search)
+
+DiffKit's repo-search MVP is designed as:
+
+- Cloudflare Worker (`apps/dashboard`) for public API/control-plane orchestration.
+- Self-hosted `apps/search` service for mirror sync/index workflow + search proxying.
+- Livegrep backend for fast code search.
+
+For complete setup and deployment instructions (local + VPS), see:
+
+- `apps/search/README.md`
+
 ## GitHub App Permissions Reference
 
 Expanding permissions after users have installed the app will require those installations to approve the new permission set.

apps/dashboard/.dev.vars.example

@@ -114,3 +114,17 @@ DEV_TUNNEL_URL=
 # real bucket named there (see bucket_name). Your pub URL must be for that bucket — not only
 # preview_bucket_name (that name is for local simulation naming when not remote).
 R2_PUBLIC_BASE_URL=
+
+# -----------------------------------------------------------------------------
+# 7. Search service (apps/search + livegrep)
+# -----------------------------------------------------------------------------
+# For local development with apps/search running on localhost:8910.
+# The Worker fetches this endpoint for both query and control endpoints.
+LIVEGREP_BASE_URL=http://127.0.0.1:8910
+SEARCH_CONTROL_BASE_URL=http://127.0.0.1:8910
+
+# Must match SEARCH_CONTROL_TOKEN in apps/search environment (optional but recommended).
+SEARCH_CONTROL_TOKEN=change-me
+
+# Optional: only needed if your apps/search endpoint requires bearer auth for /api/v1/search.
+LIVEGREP_API_TOKEN=

apps/dashboard/drizzle/0004_diffkit_search_mvp.sql

@@ -0,0 +1,57 @@
+CREATE TABLE `search_repo_registry` (
+	`id` text PRIMARY KEY NOT NULL,
+	`provider` text NOT NULL,
+	`owner` text NOT NULL,
+	`name` text NOT NULL,
+	`default_branch` text NOT NULL,
+	`is_enabled` integer NOT NULL,
+	`tier` text NOT NULL,
+	`last_seen_head_sha` text,
+	`last_indexed_head_sha` text,
+	`last_synced_at` integer,
+	`last_indexed_at` integer,
+	`status` text NOT NULL,
+	`is_private` integer NOT NULL,
+	`last_error` text,
+	`created_at` integer NOT NULL,
+	`updated_at` integer NOT NULL
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX `search_repo_registry_provider_owner_name_uidx` ON `search_repo_registry` (`provider`,`owner`,`name`);
+--> statement-breakpoint
+CREATE INDEX `search_repo_registry_status_idx` ON `search_repo_registry` (`status`);
+--> statement-breakpoint
+CREATE INDEX `search_repo_registry_tier_status_idx` ON `search_repo_registry` (`tier`,`status`);
+--> statement-breakpoint
+CREATE INDEX `search_repo_registry_enabled_tier_idx` ON `search_repo_registry` (`is_enabled`,`tier`);
+--> statement-breakpoint
+CREATE TABLE `search_jobs` (
+	`id` text PRIMARY KEY NOT NULL,
+	`repo_id` text NOT NULL,
+	`job_type` text NOT NULL,
+	`priority` text NOT NULL,
+	`status` text NOT NULL,
+	`attempt` integer NOT NULL,
+	`error` text,
+	`created_at` integer NOT NULL,
+	`updated_at` integer NOT NULL,
+	FOREIGN KEY (`repo_id`) REFERENCES `search_repo_registry`(`id`) ON UPDATE no action ON DELETE cascade
+);
+--> statement-breakpoint
+CREATE INDEX `search_jobs_repo_type_status_idx` ON `search_jobs` (`repo_id`,`job_type`,`status`);
+--> statement-breakpoint
+CREATE INDEX `search_jobs_status_created_idx` ON `search_jobs` (`status`,`created_at`);
+--> statement-breakpoint
+CREATE TABLE `search_index_builds` (
+	`id` text PRIMARY KEY NOT NULL,
+	`build_version` text NOT NULL,
+	`repo_count` integer NOT NULL,
+	`started_at` integer NOT NULL,
+	`finished_at` integer,
+	`status` text NOT NULL,
+	`manifest_r2_key` text
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX `search_index_builds_build_version_uidx` ON `search_index_builds` (`build_version`);
+--> statement-breakpoint
+CREATE INDEX `search_index_builds_status_started_idx` ON `search_index_builds` (`status`,`started_at`);

apps/dashboard/drizzle/meta/_journal.json

@@ -15,6 +15,13 @@
 			"when": 1775606872196,
 			"tag": "0001_outstanding_blizzard",
 			"breakpoints": true
+		},
+		{
+			"idx": 2,
+			"version": "6",
+			"when": 1776921000000,
+			"tag": "0004_diffkit_search_mvp",
+			"breakpoints": true
 		}
 	]
 }

apps/dashboard/src/components/navigation/command-palette.tsx

@@ -11,16 +11,21 @@ import {
 import { cn } from "@diffkit/ui/lib/utils";
 import { useQuery, useQueryClient } from "@tanstack/react-query";
 import { getRouteApi, useRouter } from "@tanstack/react-router";
+import type { ReactNode } from "react";
 import { useEffect, useMemo, useRef, useState } from "react";
 import type { CommandItem, CommandItemMeta } from "#/lib/command-palette/types";
 import {
 	cacheSearchResults,
 	getCommandSearchItems,
+	getSearchCodeCommandItems,
 	useCommandItems,
 } from "#/lib/command-palette/use-command-items";
 import { useCommandPalette } from "#/lib/command-palette/use-command-palette";
 import { formatRelativeTime } from "#/lib/format-relative-time";
-import { githubCommandPaletteSearchQueryOptions } from "#/lib/github.query";
+import {
+	codeSearchQueryOptions,
+	githubCommandPaletteSearchQueryOptions,
+} from "#/lib/github.query";
 
 const routeApi = getRouteApi("/_protected");
 
@@ -31,6 +36,7 @@ export function CommandPalette() {
 	const { user } = routeApi.useRouteContext();
 	const scope = useMemo(() => ({ userId: user.id }), [user.id]);
 	const [search, setSearch] = useState("");
+	const [isCodeSearchDisabled, setIsCodeSearchDisabled] = useState(false);
 	const debouncedSearch = useDebouncedValue(search, 250);
 	const trimmedDebouncedSearch = debouncedSearch.trim();
 	const shouldSearchGitHub = open && trimmedDebouncedSearch.length >= 2;
@@ -42,13 +48,39 @@ export function CommandPalette() {
 		),
 		enabled: shouldSearchGitHub,
 	});
+	const codeSearchQuery = useQuery({
+		...codeSearchQueryOptions(scope, {
+			q: trimmedDebouncedSearch,
+			page: "1",
+		}),
+		enabled: shouldSearchGitHub && !isCodeSearchDisabled,
+	});
 	const searchItems = useMemo(
 		() => getCommandSearchItems(githubSearchQuery.data),
 		[githubSearchQuery.data],
 	);
+	const codeSearchItems = useMemo(
+		() =>
+			getSearchCodeCommandItems(codeSearchQuery.data, async (item) => {
+				const [owner, repo, ...rest] = item.repo.split("/");
+				if (!(owner && repo) || rest.length > 0) {
+					return;
+				}
+				const routeSplat = `main/${item.path}`;
+				await router.navigate({
+					to: "/$owner/$repo/blob/$",
+					params: {
+						owner,
+						repo,
+						_splat: routeSplat,
+					},
+				});
+			}),
+		[codeSearchQuery.data, router],
+	);
 	const allItems = useMemo(
-		() => mergeCommandItems(items, searchItems),
-		[items, searchItems],
+		() => mergeCommandItems(items, searchItems, codeSearchItems),
+		[items, searchItems, codeSearchItems],
 	);
 
 	const cachedSearchDataRef = useRef(githubSearchQuery.data);
@@ -59,6 +91,12 @@ export function CommandPalette() {
 		cacheSearchResults(queryClient, scope, data);
 	}, [githubSearchQuery.data, queryClient, scope]);
 
+	useEffect(() => {
+		if (codeSearchQuery.data?.code_search_disabled) {
+			setIsCodeSearchDisabled(true);
+		}
+	}, [codeSearchQuery.data?.code_search_disabled]);
+
 	const groups = new Map<string, CommandItem[]>();
 	for (const item of allItems) {
 		const list = groups.get(item.group) ?? [];
@@ -94,7 +132,9 @@ export function CommandPalette() {
 				<CommandEmpty>
 					{getEmptyMessage(
 						search,
-						shouldSearchGitHub && githubSearchQuery.isFetching,
+						shouldSearchGitHub &&
+							(githubSearchQuery.isFetching ||
+								(!isCodeSearchDisabled && codeSearchQuery.isFetching)),
 					)}
 				</CommandEmpty>
 				{Array.from(groups.entries()).map(([groupName, groupItems]) => (
@@ -110,10 +150,17 @@ export function CommandPalette() {
 										className={cn("size-4 shrink-0", item.iconClassName)}
 									/>
 								)}
-								<div className="mr-4 min-w-0 flex-1">
-									<p className="truncate text-sm">{item.label}</p>
-									{item.meta && <ItemMeta meta={item.meta} />}
-								</div>
+								{item.meta?.codeSearch ? (
+									<CodeSearchItemMeta
+										meta={item.meta.codeSearch}
+										query={trimmedDebouncedSearch}
+									/>
+								) : (
+									<div className="mr-4 min-w-0 flex-1">
+										<p className="truncate text-sm">{item.label}</p>
+										{item.meta && <ItemMeta meta={item.meta} />}
+									</div>
+								)}
 								{item.meta?.comments != null && item.meta.comments > 0 && (
 									<span className="ml-auto flex shrink-0 items-center gap-1 text-[11px] text-muted-foreground">
 										<CommentIcon className="size-4" />
@@ -147,10 +194,11 @@ function useDebouncedValue(value: string, delayMs: number) {
 function mergeCommandItems(
 	localItems: CommandItem[],
 	searchItems: CommandItem[],
+	codeItems: CommandItem[],
 ) {
 	const itemsById = new Map<string, CommandItem>();
 
-	for (const item of [...localItems, ...searchItems]) {
+	for (const item of [...localItems, ...searchItems, ...codeItems]) {
 		if (!itemsById.has(item.id)) {
 			itemsById.set(item.id, item);
 		}
@@ -203,3 +251,69 @@ function ItemMeta({ meta }: { meta: CommandItemMeta }) {
 		</span>
 	);
 }
+
+function CodeSearchItemMeta({
+	meta,
+	query,
+}: {
+	meta: NonNullable<CommandItemMeta["codeSearch"]>;
+	query: string;
+}) {
+	return (
+		<div className="mr-4 min-w-0 flex-1">
+			<p className="truncate text-[11px] text-muted-foreground">{meta.repo}</p>
+			<p className="truncate text-sm">{meta.path}</p>
+			<div className="mt-1 space-y-0.5 font-mono text-[11px] text-muted-foreground">
+				{meta.snippets.map((snippet) => (
+					<div
+						key={`${meta.repo}:${meta.path}:${snippet.lineNumber}`}
+						className="flex items-start gap-2"
+					>
+						<span className="w-8 shrink-0 text-right text-[10px] text-muted-foreground/70">
+							{snippet.lineNumber}
+						</span>
+						<span className="min-w-0 flex-1 truncate">
+							{highlightQueryMatch(snippet.line, query)}
+						</span>
+					</div>
+				))}
+			</div>
+		</div>
+	);
+}
+
+function highlightQueryMatch(text: string, query: string): ReactNode {
+	const normalizedQuery = query.trim();
+	if (!normalizedQuery) {
+		return text;
+	}
+
+	const lowerText = text.toLowerCase();
+	const lowerQuery = normalizedQuery.toLowerCase();
+	const parts: ReactNode[] = [];
+	let cursor = 0;
+	let hitIndex = 0;
+
+	while (cursor < text.length) {
+		const foundAt = lowerText.indexOf(lowerQuery, cursor);
+		if (foundAt === -1) {
+			parts.push(text.slice(cursor));
+			break;
+		}
+		if (foundAt > cursor) {
+			parts.push(text.slice(cursor, foundAt));
+		}
+		const end = foundAt + normalizedQuery.length;
+		parts.push(
+			<span
+				key={`${foundAt}-${hitIndex++}`}
+				className="rounded-sm bg-blue-500/20 px-0.5 text-foreground"
+			>
+				{text.slice(foundAt, end)}
+			</span>,
+		);
+		cursor = end;
+	}
+
+	return parts;
+}