fix(jobs): pre-flight workspace check before gapfill/FBA; humanize errors

app/(user-data)/my-jobs/page.tsx

@@ -22,6 +22,7 @@ import { DataGrid, GridColDef, GridPaginationModel, GridSortModel } from '@mui/x
 
 import { getJobsFromApi } from '@/lib/api/modelseed';
 import { listTrackedJobs, TrackedJob } from '@/lib/api/jobTracker';
+import { formatJobError } from '@/lib/utils/jobErrors';
 import { USE_MODELSEED_API } from '@/lib/api/config';
 import AuthGuard from '@/components/auth/AuthGuard';
 import DataControlHeader, { withQuickSearchHeaders } from '@/components/layout/DataControlHeader';
@@ -135,8 +136,13 @@ function mergeApiAndTrackedJobs(
             statusHistory.set(id, { status, timestamp: now, sameCount: 1 });
         }
 
-        const errorMsg = job.error ? String(job.error) : undefined;
         const outputPath = args?.output_path ? String(args.output_path) : undefined;
+        // Some legacy backend job records still carry the raw
+        // `_ERROR_Object not found!_ERROR_` string. Translate to actionable
+        // wording, substituting the job's own model ref when present so the
+        // tooltip points users at the path their reconstruct never produced.
+        const modelArg = typeof args?.model === 'string' ? String(args.model) : undefined;
+        const errorMsg = formatJobError(job.error, modelArg);
         const app = String(params?.command ?? job.app ?? job.type ?? 'Unknown');
 
         rows.push({

app/model/[...path]/page.tsx

@@ -53,6 +53,7 @@ import {
     type TrackedJob,
 } from '@/lib/api/jobTracker';
 import { parseWorkspaceDate } from '@/lib/utils/date';
+import { formatJobError } from '@/lib/utils/jobErrors';
 import ModelDetailHeader from '@/components/ui/ModelDetailHeader';
 import type { FbaAdvancedOptions } from '@/components/ui/MediaSelectionDialog';
 import DownloadModelMenu from '@/components/ui/DownloadModelMenu';
@@ -2042,10 +2043,26 @@ export default function ModelDetailPage({ params }: { params: Promise<{ path: st
         setActionLoading(kind);
         setActionMessage(null);
         const selectedMedia = media || defaultMedia;
+        const modelRef = workspaceCandidates[0];
         try {
+            // Pre-flight: confirm the model object actually exists in the workspace
+            // before enqueuing the celery job. Without this, users who navigate
+            // (or follow a stale link) to a model ref whose backing object is
+            // missing trigger a backend WorkspaceError per submission — the symptom
+            // surfacing in Flower as `_ERROR_Object not found!_ERROR_`. Catching
+            // it here avoids the wasted job and shows actionable wording.
+            try {
+                await workspaceGet([modelRef]);
+            } catch (probeErr) {
+                throw new Error(
+                    formatJobError(probeErr, modelRef) ??
+                    `No model found at '${modelRef}'.`,
+                );
+            }
+
             // Build FBA payload with optional advanced options (reaction knockouts)
             const fbaPayload: Record<string, unknown> = {
-                model: workspaceCandidates[0],
+                model: modelRef,
                 media: selectedMedia,
                 media_supplement: [],
             };
@@ -2060,7 +2077,7 @@ export default function ModelDetailPage({ params }: { params: Promise<{ path: st
                 kind === 'fba'
                     ? await submitFbaJobFromApi(fbaPayload)
                     : await submitGapfillJobFromApi({
-                        model: workspaceCandidates[0],
+                        model: modelRef,
                         template_type: 'gn',
                         media: selectedMedia,
                     });
@@ -2083,8 +2100,8 @@ export default function ModelDetailPage({ params }: { params: Promise<{ path: st
                     : `${kind === 'fba' ? 'FBA' : 'Gapfill'} job submitted.`,
             );
         } catch (err) {
-            const message = err instanceof Error ? err.message : `Failed to submit ${kind} job`;
-            setActionMessage(message);
+            const raw = err instanceof Error ? err.message : `Failed to submit ${kind} job`;
+            setActionMessage(formatJobError(raw, modelRef) ?? raw);
         } finally {
             setActionLoading(null);
         }

lib/utils/jobErrors.ts

@@ -0,0 +1,32 @@
+/**
+ * Humanize Celery/Workspace error messages surfaced by the modelseed_api backend
+ * so users see actionable wording instead of raw exception text.
+ *
+ * Two forms are handled:
+ *  - Legacy form (older job records, pre-2026-06): `_ERROR_Object not found!_ERROR_`
+ *    — the inscrutable PATRIC workspace internal error.
+ *  - New form (after José's backend change): `No model found at '<path>'. Check that
+ *    your reconstruct job completed successfully and saved a model to this path,
+ *    or pass a different ref.` — already user-readable, passed through unchanged.
+ *
+ * @param raw - Raw error string from `job.error` / submission rejection
+ * @param modelRef - The model workspace path the job was targeting (optional;
+ *   used to substitute `<path>` into the legacy-form message).
+ * @returns A user-facing error string, or `undefined` when `raw` is falsy.
+ */
+export function formatJobError(raw: unknown, modelRef?: string): string | undefined {
+    if (raw === null || raw === undefined) return undefined;
+    const text = String(raw).trim();
+    if (!text) return undefined;
+
+    if (/_ERROR_Object not found!_ERROR_/.test(text) || /Object not found/i.test(text)) {
+        const pathClause = modelRef ? ` at '${modelRef}'` : '';
+        return (
+            `No model found${pathClause}. Check that your reconstruct job ` +
+            'completed successfully and saved a model to this path, or pass a ' +
+            'different ref.'
+        );
+    }
+
+    return text;
+}

tests/unit/api/biochem.test.ts

@@ -15,8 +15,17 @@ describe('Biochem API Integration Tests', () => {
 
   beforeAll(async () => {
     biochemApi = await loadBiochemApi();
+    // Race the live probe against an internal timeout so the catch path runs
+    // (marking isApiAvailable = false) before the vitest hookTimeout aborts the
+    // hook itself. Otherwise CI fails on slow networks even though the suite is
+    // designed to skip gracefully when the API is unreachable.
     try {
-      const res = await biochemApi.getReactions({ limit: 1 });
+      const res = await Promise.race([
+        biochemApi.getReactions({ limit: 1 }),
+        new Promise<never>((_, reject) =>
+          setTimeout(() => reject(new Error('Biochem API probe timed out after 7s')), 7000),
+        ),
+      ]);
       expect(res.docs).toBeDefined();
     } catch (e: unknown) {
       console.warn('Biochem API is unavailable, skipping tests:', getErrorMessage(e));

tests/unit/utils/jobErrors.test.ts

@@ -0,0 +1,58 @@
+import { describe, it, expect } from 'vitest';
+import { formatJobError } from '@/lib/utils/jobErrors';
+
+describe('formatJobError', () => {
+    it('returns undefined for empty inputs', () => {
+        expect(formatJobError(null)).toBeUndefined();
+        expect(formatJobError(undefined)).toBeUndefined();
+        expect(formatJobError('')).toBeUndefined();
+        expect(formatJobError('   ')).toBeUndefined();
+    });
+
+    it('translates the legacy _ERROR_Object not found!_ERROR_ wording', () => {
+        const out = formatJobError("WorkspaceError('_ERROR_Object not found!_ERROR_')");
+        expect(out).toContain('No model found');
+        expect(out).toContain('reconstruct job');
+        expect(out).not.toContain('_ERROR_');
+    });
+
+    it('embeds the model ref into the legacy-form message when provided', () => {
+        const out = formatJobError(
+            "WorkspaceError('_ERROR_Object not found!_ERROR_')",
+            '/alice@patricbrc.org/modelseed/Ecoli/model',
+        );
+        expect(out).toContain("at '/alice@patricbrc.org/modelseed/Ecoli/model'");
+    });
+
+    it("handles a less-decorated 'Object not found' substring", () => {
+        const out = formatJobError('Object not found', '/u/modelseed/x/model');
+        expect(out).toContain("at '/u/modelseed/x/model'");
+        expect(out).toContain('No model found');
+    });
+
+    it("passes through the new backend message unchanged", () => {
+        const backendNew =
+            "No model found at '/alice/modelseed/Ecoli/model'. Check that your " +
+            'reconstruct job completed successfully and saved a model to this path, ' +
+            'or pass a different ref.';
+        const out = formatJobError(backendNew);
+        expect(out).toContain('No model found');
+        expect(out).toContain('reconstruct job');
+        expect(out).not.toContain('_ERROR_');
+    });
+
+    it("leaves unrelated errors untouched", () => {
+        expect(formatJobError('Token validation failed: bogus signer')).toBe(
+            'Token validation failed: bogus signer',
+        );
+        expect(formatJobError('HTTPError("504 Gateway Timeout")')).toBe(
+            'HTTPError("504 Gateway Timeout")',
+        );
+    });
+
+    it('coerces non-string inputs via String()', () => {
+        const err = new Error("WorkspaceError('_ERROR_Object not found!_ERROR_')");
+        const out = formatJobError(err);
+        expect(out).toContain('No model found');
+    });
+});