diff --git a/packages/cli/README.md b/packages/cli/README.md index 2f2de98c33..9c8bfb5ff6 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -64,6 +64,23 @@ the effective lane once at boot. A background query that is shed before execution returns HTTP 503 with `Retry-After: 1` and `code: "STORE_SCHEDULER_BUSY"`. +## RDF file input + +Commands with `--file` accept N-Quads (`.nq`), N-Triples (`.nt`), Turtle (`.ttl`), +TriG (`.trig`), JSON quad arrays (`.json`), and JSON-LD (`.jsonld`). JSON-LD +supports inline `@context`, lists, named graphs and typed/language literals. +Relative identifiers resolve against the input file's `file:` URL unless an +inline `@base` overrides it. Expansion that would discard statements fails. + +JSON-LD named graphs can be stored in Working Memory with `dkg ka create --no-finalize`. +A default-finalizing create rejects them before contacting the daemon: sealing, SWM +sharing and VM publication do not yet preserve named-graph identity. Rewrite the +document into the default graph to use those transitions. + +JSON-LD ingestion does not fetch remote contexts or `@import` URLs. Embed the +required context inline before importing a file. Existing simple quad arrays +remain supported in both `.json` and `.jsonld` files. + ## Running a Core Node (relay operator) A Core Node is a publicly-reachable host that runs a libp2p circuit-relay v2 diff --git a/packages/cli/package.json b/packages/cli/package.json index e1ff534064..1e538e30ee 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -67,6 +67,7 @@ "ethers": "^6", "js-yaml": "^4.1.1", "jsonc-parser": "3.3.1", + "jsonld": "^8.3.3", "n3": "^2.0.1", "semver": "^7.7.4", "typescript": "^5.7", @@ -79,7 +80,8 @@ "@types/semver": "^7.8.0", "@vitest/coverage-v8": "^4.0.18", "esbuild": "0.27.7", - "vitest": "^4.0.18" + "vitest": "^4.0.18", + "@types/jsonld": "^1.5.15" }, "publishConfig": { "access": "public" diff --git a/packages/cli/src/cli-helpers.ts b/packages/cli/src/cli-helpers.ts index fd3446af3f..9e350b70d5 100644 --- a/packages/cli/src/cli-helpers.ts +++ b/packages/cli/src/cli-helpers.ts @@ -1,3 +1,4 @@ +import type { ParsedRdf, SimpleQuad } from './rdf-parser.js'; import { Command } from 'commander'; import { readFileSync } from 'node:fs'; import { createInterface } from 'node:readline'; @@ -148,41 +149,45 @@ function loadStructuredFile(filePath: string): any { return yaml.load(content); } -async function loadQuadsFromInput( +async function loadRdfFromInput( opts: ActionOpts, defaultGraph: string, -): Promise> { +): Promise { const rdfParser = await import('./rdf-parser.js'); if (opts.file) { const { readFile } = await import('node:fs/promises'); + const { pathToFileURL } = await import('node:url'); const raw = await readFile(opts.file, 'utf-8'); const format = opts.format ?? rdfParser.detectFormat(opts.file); - const quads = await rdfParser.parseRdf(raw, format, defaultGraph); - console.log(`Parsed ${quads.length} quad(s) from ${opts.file} (${format})`); - return quads; + const input = await rdfParser.parseRdfInput(raw, format, defaultGraph, pathToFileURL(opts.file).href); + console.log(`Parsed ${input.quads.length} quad(s) from ${opts.file} (${format})`); + return input; } if (opts.triples) { - const parsed = JSON.parse(opts.triples); - return parsed.map((q: Record) => ({ ...q, graph: q.graph || defaultGraph })); + return rdfParser.parseRdfInput(opts.triples, 'json', defaultGraph); } if (opts.subject && opts.predicate && opts.object) { - return [{ + return { sourceKind: 'legacy-quads', quads: [{ subject: opts.subject, predicate: opts.predicate, object: opts.object.startsWith('"') || opts.object.startsWith('http') || opts.object.startsWith('did:') ? opts.object : `"${opts.object}"`, graph: defaultGraph, - }]; + }] }; } console.error(`Provide --file (${rdfParser.supportedExtensions().join(', ')}), --triples, or --subject/--predicate/--object`); process.exit(1); } +async function loadQuadsFromInput(opts: ActionOpts, defaultGraph: string): Promise { + return (await loadRdfFromInput(opts, defaultGraph)).quads; +} + function probeHostForApiHost(apiHost: string | undefined): string { if (!apiHost || apiHost === '0.0.0.0') return '127.0.0.1'; if (apiHost === '::') return '::1'; @@ -380,6 +385,7 @@ export { parseOptionalVerifyTimeoutOption, loadStructuredFile, loadQuadsFromInput, + loadRdfFromInput, resolveDaemonEntryPoint, probeHostForApiHost, selectedDkgHomeForEnv, diff --git a/packages/cli/src/commands/knowledge-asset.ts b/packages/cli/src/commands/knowledge-asset.ts index 4c18a340fc..18cbc10e23 100644 --- a/packages/cli/src/commands/knowledge-asset.ts +++ b/packages/cli/src/commands/knowledge-asset.ts @@ -7,12 +7,11 @@ import { type KnowledgeAssetPublishResponse, type KnowledgeAssetShareJobState, type KnowledgeAssetShareResponse, - type KnowledgeAssetWritableQuad, type PreSignedAuthorAttestationPayload, } from '../api-client.js'; import { formatQuadObject, - loadQuadsFromInput, + loadRdfFromInput, loadStructuredFile, type ActionOpts, } from '../cli-helpers.js'; @@ -53,20 +52,8 @@ function hasQuadInput(opts: ActionOpts): boolean { ); } -async function loadWritableQuads(opts: ActionOpts): Promise { - const quads = await loadQuadsFromInput( - { - ...opts, - file: inputFilePath(opts), - }, - '', - ); - return quads.map((quad) => ({ - subject: quad.subject, - predicate: quad.predicate, - object: quad.object, - graph: quad.graph ?? '', - })); +async function loadWritableInput(opts: ActionOpts) { + return loadRdfFromInput({ ...opts, file: inputFilePath(opts) }, ''); } function parsePreSignedAuthorAttestation(raw: unknown): PreSignedAuthorAttestationPayload | undefined { @@ -243,7 +230,14 @@ export function registerKnowledgeAssetCommand(program: Command): void { )))) .action(async (name: string, opts: ActionOpts) => runAction(async () => { const contextGraphId = requiredContextGraphId(opts); - const quads = hasQuadInput(opts) ? await loadWritableQuads(opts) : undefined; + const input = hasQuadInput(opts) ? await loadWritableInput(opts) : undefined; + const quads = input?.quads; + if (opts.finalize !== false && input?.sourceKind === 'jsonld' && quads?.some((quad) => quad.graph !== '')) { + throw new Error( + 'JSON-LD named graphs cannot be finalized yet. Use ka create --no-finalize to keep them in Working Memory, ' + + 'or rewrite the document into the default graph before finalizing or sharing.', + ); + } if (opts.share === true && (!quads || quads.length === 0 || opts.finalize === false)) { throw new Error('--share requires non-empty payload quads and finalize enabled'); } @@ -279,7 +273,7 @@ export function registerKnowledgeAssetCommand(program: Command): void { ))) .action(async (name: string, opts: ActionOpts) => runAction(async () => { const contextGraphId = requiredContextGraphId(opts); - const quads = await loadWritableQuads(opts); + const { quads } = await loadWritableInput(opts); const client = await ApiClient.connect(); const result = await client.knowledgeAssetWrite(contextGraphId, name, quads, { ...(subGraphName(opts) ? { subGraphName: subGraphName(opts) } : {}), diff --git a/packages/cli/src/rdf-parser.ts b/packages/cli/src/rdf-parser.ts index af9097d30a..22410a1edf 100644 --- a/packages/cli/src/rdf-parser.ts +++ b/packages/cli/src/rdf-parser.ts @@ -1,4 +1,5 @@ import { Parser, type Quad as N3Quad } from 'n3'; +import type { JsonLdDocument, Options as JsonLdOptions } from 'jsonld'; import { formatCanonicalRdfTerm } from '@origintrail-official/dkg-rdf-utils'; export interface SimpleQuad { @@ -8,6 +9,12 @@ export interface SimpleQuad { graph: string; } +/** Neutral syntax provenance; consumers own policies for the resulting dataset. */ +export interface ParsedRdf { + sourceKind: 'jsonld' | 'legacy-quads' | 'rdf'; + quads: SimpleQuad[]; +} + export type RdfFormat = 'nquads' | 'ntriples' | 'turtle' | 'trig' | 'json' | 'jsonld'; const EXTENSION_MAP: Record = { @@ -40,38 +47,57 @@ export function supportedExtensions(): string[] { * For formats without named graph support (N-Triples, Turtle), * the defaultGraph is used. */ -export async function parseRdf( +export async function parseRdfInput( content: string, format: RdfFormat, defaultGraph: string, -): Promise { - if (format === 'json') { - const parsed = JSON.parse(content); - const arr = Array.isArray(parsed) ? parsed : parsed.quads; - return arr.map((q: any) => ({ - subject: q.subject, - predicate: q.predicate, - object: q.object, - graph: q.graph || defaultGraph, - })); - } - - if (format === 'jsonld') { - // JSON-LD → N-Quads conversion would require the jsonld library. - // For now, treat as our JSON quad format if it has subject/predicate/object, - // otherwise report unsupported. - const parsed = JSON.parse(content); - if (Array.isArray(parsed) && parsed[0]?.subject) { - return parsed.map((q: any) => ({ - subject: q.subject, - predicate: q.predicate, - object: q.object, - graph: q.graph || defaultGraph, - })); + baseIRI?: string, +): Promise { + if (format === 'json' || format === 'jsonld') { + const parsed: unknown = JSON.parse(content); + const legacy = decodeLegacyQuads( + format === 'json' && isRecord(parsed) ? parsed.quads : parsed, + defaultGraph, + ); + if (legacy) return { sourceKind: 'legacy-quads', quads: legacy }; + if (format === 'json') throw new Error('JSON input must contain an array of subject/predicate/object quads'); + if (parsed === null || typeof parsed !== 'object') { + throw new Error('JSON-LD input must be an object or array'); + } + const { default: jsonld } = await import('jsonld'); + const remoteContextError = new Error('Remote JSON-LD contexts are disabled; embed an inline @context before ingesting the file'); + let remoteLoadAttempted = false; + // jsonld.js 8 supports safe mode; the older upstream declaration omits it. + const options: JsonLdOptions.ToRdf & { safe: true } = { + format: 'application/n-quads', + base: baseIRI, + safe: true, + documentLoader: async () => { + remoteLoadAttempted = true; + throw remoteContextError; + }, + }; + let nquads: object | string; + try { + // jsonld.js validates the JSON-LD grammar and rejects lossy expansion. + nquads = await jsonld.toRDF(parsed as JsonLdDocument, options); + } catch (error) { + // The loader owns this policy error, regardless of how jsonld.js wraps it. + if (remoteLoadAttempted) throw remoteContextError; + throw error; } - throw new Error('JSON-LD with @context requires the jsonld library. Use .nq, .nt, .ttl, or .trig instead.'); + if (typeof nquads !== 'string') throw new Error('JSON-LD conversion did not return N-Quads'); + return { sourceKind: 'jsonld', quads: await parseN3Quads(nquads, 'nquads', defaultGraph) }; } + return { sourceKind: 'rdf', quads: await parseN3Quads(content, format, defaultGraph) }; +} + +function parseN3Quads( + content: string, + format: Exclude, + defaultGraph: string, +): Promise { // N3 parser handles N-Triples, N-Quads, Turtle, TriG const n3Format = N3_FORMAT_MAP[format]; if (!n3Format) throw new Error(`Unsupported format: ${format}`); @@ -93,3 +119,35 @@ export async function parseRdf( }); }); } + +/** Array-only compatibility facade for callers that do not need source provenance. */ +export async function parseRdf( + content: string, + format: RdfFormat, + defaultGraph: string, + baseIRI?: string, +): Promise { + return (await parseRdfInput(content, format, defaultGraph, baseIRI)).quads; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +type LegacyQuad = Omit & { graph?: string | null }; + +function isLegacyQuad(value: unknown): value is LegacyQuad { + return isRecord(value) + && typeof value.subject === 'string' + && typeof value.predicate === 'string' + && typeof value.object === 'string' + && (value.graph == null || typeof value.graph === 'string') + && !Object.keys(value).some((key) => key.startsWith('@')); +} + +function decodeLegacyQuads(value: unknown, defaultGraph: string): SimpleQuad[] | undefined { + if (!Array.isArray(value) || !value.every(isLegacyQuad)) return undefined; + return value.map(({ subject, predicate, object, graph }) => ({ + subject, predicate, object, graph: graph || defaultGraph, + })); +} diff --git a/packages/cli/test/jsonld-knowledge-asset-lifecycle.test.ts b/packages/cli/test/jsonld-knowledge-asset-lifecycle.test.ts new file mode 100644 index 0000000000..8afb4fd7b1 --- /dev/null +++ b/packages/cli/test/jsonld-knowledge-asset-lifecycle.test.ts @@ -0,0 +1,69 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { execFile } from 'node:child_process'; +import { writeFile } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; +import { promisify } from 'node:util'; +import { startLiveDaemon, stopLiveDaemon, postJson, getJson, type LiveDaemon } from './helpers/live-daemon.js'; + +const execFileAsync = promisify(execFile); +const CLI_ENTRY = resolve(import.meta.dirname, '../dist/cli.js'); +const CG = 'jsonld-write-lifecycle'; +const RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#'; +let daemon: LiveDaemon | undefined; + +async function runCli(args: string[]) { + if (!daemon) throw new Error('daemon is not ready'); + return execFileAsync(process.execPath, [CLI_ENTRY, ...args], { + env: { ...process.env, DKG_HOME: daemon.home, DKG_API_PORT: String(daemon.apiPort), DKG_AUTH_TOKEN: daemon.token ?? '' }, + timeout: 120_000, + }); +} + +beforeAll(async () => { + daemon = await startLiveDaemon({ authEnabled: true }); + const created = await postJson(daemon, '/api/context-graph/create', { id: CG, name: CG, accessPolicy: 1 }); + expect(created.status, JSON.stringify(created.body)).toBeLessThan(300); + const registered = await postJson(daemon, '/api/context-graph/register', { id: CG, accessPolicy: 1 }); + expect(registered.status, JSON.stringify(registered.body)).toBe(200); +}, 120_000); + +afterAll(async () => { await stopLiveDaemon(daemon); }); + +type Quad = { subject: string; predicate: string; object: string; graph: string }; +function assertLinks(quads: Quad[], kind: 'list' | 'nested') { + const root = quads.find((q) => q.subject === `urn:${kind}` && q.predicate === `https://example.org/${kind === 'list' ? 'items' : 'child'}`); + expect(root).toBeDefined(); + if (kind === 'nested') { + expect(quads).toContainEqual(expect.objectContaining({ subject: root!.object, predicate: 'https://example.org/name', object: '"nested"' })); + } else { + const first = quads.find((q) => q.subject === root!.object && q.predicate === `${RDF}first`); + const rest = quads.find((q) => q.subject === root!.object && q.predicate === `${RDF}rest`); + expect(first?.object).toBe('urn:first'); + expect(rest).toBeDefined(); + expect(quads).toContainEqual(expect.objectContaining({ subject: rest!.object, predicate: `${RDF}first`, object: 'urn:second' })); + expect(quads).toContainEqual(expect.objectContaining({ subject: rest!.object, predicate: `${RDF}rest`, object: `${RDF}nil` })); + } +} + +describe('JSON-LD CLI writes through the real daemon', () => { + it.each(['list', 'nested'] as const)('writes and finalizes linked %s RDF without losing blank-node edges', async (kind) => { + const document = kind === 'list' + ? { '@id': 'urn:list', 'https://example.org/items': { '@list': [{ '@id': 'urn:first' }, { '@id': 'urn:second' }] } } + : { '@id': 'urn:nested', 'https://example.org/child': { 'https://example.org/name': 'nested' } }; + const path = join(daemon!.home, `${kind}.jsonld`); + await writeFile(path, JSON.stringify(document)); + await runCli(['ka', 'create', kind, '-c', CG, '--no-finalize']); + await runCli(['ka', 'write', kind, '-c', CG, '-f', path]); + const before = await getJson(daemon!, `/api/knowledge-assets/${kind}/wm/quads?contextGraphId=${CG}`); + expect(before.status, JSON.stringify(before.body)).toBe(200); + const beforeQuads = before.body.quads as Quad[]; + expect(beforeQuads.some((q) => q.object.startsWith('_:'))).toBe(true); + assertLinks(beforeQuads, kind); + await runCli(['ka', 'finalize', kind, '-c', CG]); + const after = await getJson(daemon!, `/api/knowledge-assets/${kind}/wm/quads?contextGraphId=${CG}`); + expect(after.status, JSON.stringify(after.body)).toBe(200); + const afterQuads = after.body.quads as Quad[]; + expect(afterQuads.some((q) => q.subject.startsWith('_:') || q.object.startsWith('_:'))).toBe(false); + assertLinks(afterQuads, kind); + }); +}); diff --git a/packages/cli/test/knowledge-asset-cli-smoke.test.ts b/packages/cli/test/knowledge-asset-cli-smoke.test.ts index 50b1942ea9..a6579a8aba 100644 --- a/packages/cli/test/knowledge-asset-cli-smoke.test.ts +++ b/packages/cli/test/knowledge-asset-cli-smoke.test.ts @@ -41,6 +41,19 @@ describe.sequential('knowledge-asset CLI smoke', () => { '', ].join('\n'), ); + await writeFile(join(dkgHome, 'named.jsonld'), JSON.stringify({ + '@context': { name: 'http://schema.org/name' }, + '@id': 'urn:graph:named', + '@graph': [{ '@id': 'urn:company:named', name: 'Named' }], + })); + await writeFile(join(dkgHome, 'mixed.jsonld'), JSON.stringify([ + { '@id': 'urn:company:default', 'http://schema.org/name': 'Default' }, + { '@id': 'urn:graph:named', '@graph': [{ '@id': 'urn:company:named', 'http://schema.org/name': 'Named' }] }, + ])); + await writeFile(join(dkgHome, 'legacy.jsonld'), JSON.stringify([{ + subject: 'urn:company:legacy', predicate: 'http://schema.org/name', + object: '"Legacy"', graph: 'urn:graph:legacy', + }])); await writeFile( join(dkgHome, 'attestation.json'), JSON.stringify(PRE_SIGNED_AUTHOR_ATTESTATION), @@ -345,6 +358,62 @@ describe.sequential('knowledge-asset CLI smoke', () => { ]); }, 30000); + it('rejects finalizing named-graph JSON-LD before any daemon request', async () => { + calls = []; + await expect(runCli([ + 'ka', 'create', 'named', '-c', 'research', '-f', join(dkgHome, 'named.jsonld'), + ], testEnv(dkgHome, smokeApiPort))).rejects.toMatchObject({ + stderr: expect.stringContaining('--no-finalize'), + }); + expect(calls).toEqual([]); + }, 30000); + + it('rejects a mixed default/named JSON-LD dataset before any daemon request', async () => { + calls = []; + await expect(runCli([ + 'ka', 'create', 'mixed', '-c', 'research', '-f', join(dkgHome, 'mixed.jsonld'), + ], testEnv(dkgHome, smokeApiPort))).rejects.toMatchObject({ stderr: expect.stringContaining('--no-finalize') }); + expect(calls).toEqual([]); + }, 30000); + + it.each([null, [{ subject: 'urn:invalid', object: '"missing predicate"' }]])('uses the same legacy validation for JSON files and --triples: %j', async (input) => { + calls = []; + const raw = JSON.stringify(input); + const path = join(dkgHome, 'invalid-quads.json'); + await writeFile(path, raw); + for (const inputArgs of [['--file', path], ['--triples', raw]]) { + await expect(runCli(['ka', 'write', 'invalid', '-c', 'research', ...inputArgs], testEnv(dkgHome, smokeApiPort))) + .rejects.toMatchObject({ stderr: expect.stringContaining('JSON input must contain an array of subject/predicate/object quads') }); + } + expect(calls).toEqual([]); + }, 30000); + + it('preserves named-graph JSON-LD in a WM-only create', async () => { + calls = []; + await runCli([ + 'ka', 'create', 'named', '-c', 'research', '-f', join(dkgHome, 'named.jsonld'), + '--no-finalize', + ], testEnv(dkgHome, smokeApiPort)); + expect(calls.find((call) => call.url === '/api/knowledge-assets')?.body).toMatchObject({ + finalize: false, + quads: [{ subject: 'urn:company:named', predicate: 'http://schema.org/name', + object: '"Named"', graph: 'urn:graph:named' }], + }); + }, 30000); + + it('still submits legacy JSON-LD named-graph quad arrays for default-finalizing create', async () => { + calls = []; + await runCli([ + 'ka', 'create', 'legacy', '-c', 'research', '-f', join(dkgHome, 'legacy.jsonld'), + ], testEnv(dkgHome, smokeApiPort)); + const body = calls.find((call) => call.url === '/api/knowledge-assets')?.body; + expect(body?.finalize).toBeUndefined(); + expect(body?.quads).toEqual([{ + subject: 'urn:company:legacy', predicate: 'http://schema.org/name', + object: '"Legacy"', graph: 'urn:graph:legacy', + }]); + }, 30000); + it('preserves named graph metadata when the first parsed quad is default graph', async () => { calls = []; const env = testEnv(dkgHome, smokeApiPort); diff --git a/packages/cli/test/rdf-parser.test.ts b/packages/cli/test/rdf-parser.test.ts index a03294a772..f494593dd3 100644 --- a/packages/cli/test/rdf-parser.test.ts +++ b/packages/cli/test/rdf-parser.test.ts @@ -1,7 +1,14 @@ import { describe, it, expect } from 'vitest'; +import { createServer } from 'node:http'; +import { once } from 'node:events'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { pathToFileURL } from 'node:url'; import { Parser } from 'n3'; +import { OxigraphStore } from '@origintrail-official/dkg-storage'; +import { detectFormat, supportedExtensions, parseRdf, parseRdfInput } from '../src/rdf-parser.js'; import { assertSafeRdfTerm } from '@origintrail-official/dkg-core'; -import { detectFormat, supportedExtensions, parseRdf } from '../src/rdf-parser.js'; describe('detectFormat', () => { it.each([ @@ -108,24 +115,176 @@ describe('parseRdf', () => { }); describe('jsonld format', () => { - it('throws for @context-based JSON-LD (unsupported)', async () => { - const jsonld = JSON.stringify({ - '@context': 'https://schema.org/', - '@id': 'urn:x', - name: 'Test', + it('expands a local context into RDF and applies the default graph', async () => { + const content = JSON.stringify({ + '@context': { name: 'https://schema.org/name' }, '@id': 'urn:person:alice', name: 'Alice', }); - await expect(parseRdf(jsonld, 'jsonld', DEFAULT_GRAPH)).rejects.toThrow( - /JSON-LD with @context/, - ); + expect(await parseRdf(content, 'jsonld', DEFAULT_GRAPH)).toEqual([ + { subject: 'urn:person:alice', predicate: 'https://schema.org/name', object: '"Alice"', graph: DEFAULT_GRAPH }, + ]); }); - it('accepts JSON-LD that has subject/predicate/object shape', async () => { - const content = JSON.stringify([ - { subject: 'urn:a', predicate: 'urn:p', object: '"val"' }, - ]); + it('reports syntax provenance while preserving named graphs independently of lifecycle policy', async () => { + const quads = [{ subject: 'urn:s', predicate: 'urn:p', object: '"v"', graph: 'urn:g' }]; + const document = { '@id': 'urn:g', '@graph': [{ '@id': 'urn:s', 'urn:p': 'v' }] }; + expect(await parseRdfInput(JSON.stringify(document), 'jsonld', DEFAULT_GRAPH)) + .toEqual({ sourceKind: 'jsonld', quads }); + expect(await parseRdfInput(JSON.stringify(quads), 'jsonld', DEFAULT_GRAPH)) + .toEqual({ sourceKind: 'legacy-quads', quads }); + expect(await parseRdfInput(' "v" .', 'nquads', DEFAULT_GRAPH)) + .toEqual({ sourceKind: 'rdf', quads }); + }); + + it('preserves named graphs, language/datatype literals and nested blank-node links', async () => { + const content = JSON.stringify({ + '@context': { ex: 'https://example.org/', label: 'ex:label', count: 'ex:count', detail: 'ex:detail' }, + '@id': 'urn:named-graph', + '@graph': [{ '@id': 'urn:event', label: { '@value': 'Dobar dan', '@language': 'sr' }, count: 7, + detail: { label: 'Nested' } }], + }); const quads = await parseRdf(content, 'jsonld', DEFAULT_GRAPH); - expect(quads).toHaveLength(1); + expect(quads).toHaveLength(4); + expect(quads.every((q) => q.graph === 'urn:named-graph')).toBe(true); + expect(quads).toContainEqual({ subject: 'urn:event', predicate: 'https://example.org/label', object: '"Dobar dan"@sr', graph: 'urn:named-graph' }); + expect(quads).toContainEqual({ subject: 'urn:event', predicate: 'https://example.org/count', object: '"7"^^', graph: 'urn:named-graph' }); + const detail = quads.find((q) => q.predicate === 'https://example.org/detail')!.object; + expect(detail).toMatch(/^_:/); + expect(quads).toContainEqual({ subject: detail, predicate: 'https://example.org/label', object: '"Nested"', graph: 'urn:named-graph' }); }); + + it('preserves list order and plain JSON-LD arrays', async () => { + const quads = await parseRdf(JSON.stringify([ + { '@id': 'urn:list', 'https://example.org/items': { '@list': [{ '@id': 'urn:first' }, { '@id': 'urn:second' }] } }, + { '@id': 'urn:other', 'https://example.org/label': 'Other' }, + ]), 'jsonld', DEFAULT_GRAPH); + const rdf = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#'; + const head = quads.find((q) => q.subject === 'urn:list')!.object; + const tail = quads.find((q) => q.subject === head && q.predicate === `${rdf}rest`)!.object; + expect(quads).toContainEqual({ subject: head, predicate: `${rdf}first`, object: 'urn:first', graph: DEFAULT_GRAPH }); + expect(quads).toContainEqual({ subject: tail, predicate: `${rdf}first`, object: 'urn:second', graph: DEFAULT_GRAPH }); + expect(quads).toContainEqual({ subject: tail, predicate: `${rdf}rest`, object: `${rdf}nil`, graph: DEFAULT_GRAPH }); + expect(quads).toContainEqual({ subject: 'urn:other', predicate: 'https://example.org/label', object: '"Other"', graph: DEFAULT_GRAPH }); + }); + + it.each(['context', 'import'])('rejects remote %s loading without requesting the local endpoint', async (kind) => { + let requests = 0; + const server = createServer((_req, res) => { + requests++; + res.writeHead(200, { 'Content-Type': 'application/ld+json' }); + res.end(JSON.stringify({ '@context': { label: 'https://example.org/label' } })); + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('Expected a TCP listener'); + try { + const context = `http://127.0.0.1:${address.port}/context`; + const input = { '@context': kind === 'context' ? context : { '@import': context }, '@id': 'urn:event', label: 'Remote context' }; + await expect(parseRdf(JSON.stringify(input), 'jsonld', DEFAULT_GRAPH)).rejects.toThrow(/Remote JSON-LD contexts are disabled/); + expect(requests).toBe(0); + } finally { + await new Promise((resolve, reject) => server.close((err) => err ? reject(err) : resolve())); + } + }); + + it('round-trips escaped text through actual storage', async () => { + const text = 'He said "hello".\nPath C:\\temp\tŽ'; + const quads = await parseRdf(JSON.stringify({ '@id': 'urn:event', 'https://example.org/text': text }), 'jsonld', DEFAULT_GRAPH); + const store = new OxigraphStore(); + try { + await store.insert(quads); + const result = await store.query(`SELECT ?text WHERE { GRAPH <${DEFAULT_GRAPH}> { ?text } }`); + expect(result).toMatchObject({ type: 'bindings', bindings: [{ text: quads[0].object }] }); + if (result.type !== 'bindings') throw new Error('Expected literal bindings'); + const stored = new Parser({ format: 'N-Triples' }).parse(` ${result.bindings[0].text} .`); + expect(stored[0].object.value).toBe(text); + // The canonical lexical form escapes once; it remains valid N-Quads. + expect(await parseRdf(` ${quads[0].object} .`, 'ntriples', DEFAULT_GRAPH)).toEqual(quads); + } finally { + await store.close(); + } + }); + + it('honors an inline base for document-relative identifiers', async () => { + const input = { '@context': { '@base': 'https://example.org/data/', name: 'https://schema.org/name' }, '@id': 'asset/1', name: 'Alice' }; + expect(await parseRdf(JSON.stringify(input), 'jsonld', DEFAULT_GRAPH, 'file:///tmp/input.jsonld')).toEqual([ + { subject: 'https://example.org/data/asset/1', predicate: 'https://schema.org/name', object: '"Alice"', graph: DEFAULT_GRAPH }, + ]); + }); + + it.each(['null', '42', '"https://example.org/not-a-document"'])('rejects scalar JSON-LD %s', async (content) => { + await expect(parseRdf(content, 'jsonld', DEFAULT_GRAPH)).rejects.toThrow('JSON-LD input must be an object or array'); + }); + + it.each([ + { '@id': 'asset/1', 'https://schema.org/name': 'Alice' }, + { '@id': 'urn:event', name: 'Unmapped property' }, + ])('rejects lossy JSON-LD conversion without silently omitting statements', async (input) => { + await expect(parseRdf(JSON.stringify(input), 'jsonld', DEFAULT_GRAPH)).rejects.toThrow(/Safe mode validation/); + }); + + it('rejects malformed JSON-LD contexts', async () => { + await expect(parseRdf('{"@context":42,"@id":"urn:event"}', 'jsonld', DEFAULT_GRAPH)) + .rejects.toThrow(/context/i); + }); + + it('accepts an empty JSON-LD array as an empty dataset', async () => { + expect(await parseRdf('[]', 'jsonld', DEFAULT_GRAPH)).toEqual([]); + }); + + it('does not mistake contextual JSON-LD fields for a legacy quad', async () => { + const content = [{ + '@context': { '@vocab': 'https://example.org/' }, '@id': 'urn:event', + subject: 'Subject', predicate: 'Predicate', object: 'Object', + }]; + const quads = await parseRdf(JSON.stringify(content), 'jsonld', DEFAULT_GRAPH); + expect(quads).toHaveLength(3); + expect(quads.every((q) => q.subject === 'urn:event')).toBe(true); + expect(quads.map((q) => q.predicate).sort()).toEqual([ + 'https://example.org/object', 'https://example.org/predicate', 'https://example.org/subject', + ]); + }); + + it.each(['urn:event', 'asset/1'])('loads a JSON-LD file with identifier %s through the CLI input boundary', async (id) => { + const { loadQuadsFromInput } = await import('../src/cli-helpers.js'); + const directory = await mkdtemp(join(tmpdir(), 'dkg-jsonld-ingest-')); + try { + const file = join(directory, 'event.jsonld'); + await writeFile(file, JSON.stringify({ '@context': { name: 'https://schema.org/name' }, '@id': id, name: 'CLI input' })); + expect(await loadQuadsFromInput({ file }, DEFAULT_GRAPH)).toEqual([ + { subject: new URL(id, pathToFileURL(file)).href, predicate: 'https://schema.org/name', object: '"CLI input"', graph: DEFAULT_GRAPH }, + ]); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + + it('reports inline --triples and --subject/--predicate/--object input as legacy quads', async () => { + // `ka create` applies its JSON-LD named-graph rule by this provenance, + // so inline input must never be reported as a JSON-LD document. + const { loadRdfFromInput } = await import('../src/cli-helpers.js'); + + expect(await loadRdfFromInput({ + triples: JSON.stringify([{ subject: 'urn:a', predicate: 'urn:p', object: '"val"' }]), + }, DEFAULT_GRAPH)).toEqual({ + sourceKind: 'legacy-quads', + quads: [{ subject: 'urn:a', predicate: 'urn:p', object: '"val"', graph: DEFAULT_GRAPH }], + }); + expect(await loadRdfFromInput({ + subject: 'urn:a', predicate: 'urn:p', object: 'plain words', + }, DEFAULT_GRAPH)).toEqual({ + sourceKind: 'legacy-quads', + quads: [{ subject: 'urn:a', predicate: 'urn:p', object: '"plain words"', graph: DEFAULT_GRAPH }], + }); + }); + + it.each([undefined, '', 'urn:named'])('keeps legacy JSON and JSON-LD graph handling identical (%s)', async (graph) => { + const content = JSON.stringify([{ subject: 'urn:a', predicate: 'urn:p', object: '"val"', graph }]); + const expected = [{ subject: 'urn:a', predicate: 'urn:p', object: '"val"', graph: graph || DEFAULT_GRAPH }]; + expect(await parseRdf(content, 'jsonld', DEFAULT_GRAPH)).toEqual(expected); + expect(await parseRdf(content, 'json', DEFAULT_GRAPH)).toEqual(expected); + }); + }); describe('error handling', () => { diff --git a/packages/cli/test/writable-quad-terms.test.ts b/packages/cli/test/writable-quad-terms.test.ts new file mode 100644 index 0000000000..5ffaff6a4f --- /dev/null +++ b/packages/cli/test/writable-quad-terms.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from 'vitest'; +import { validateWritableQuads } from '../src/daemon/knowledge-asset-quad-validation.js'; + +// The blank-node label matrix written for JSON-LD ingestion, run against the +// one validator the Knowledge Asset write routes use. +const quad = (terms: Record) => ({ subject: 'urn:s', predicate: 'urn:p', object: '"value"', ...terms }); + +const validLabels = ['_:b0', '_:0', '_:a.b', '_:b-1', '_:é', '_:节点', '_:a\u0300', '_:\u{10000}']; +const invalidLabels = ['_:', '_:-x', '_:.x', '_:x.', '_:a b', '_:a\n', '_:a', '_:a#comment', '_:a\\u0062', '_:\u0300', '_:\ud800', '_:\u{F0000}', ' _:b0', '_:b0 ']; + +describe.each(['subject', 'object'] as const)('writable RDF blank-node %s', (field) => { + it.each(validLabels)('accepts the complete label %s', (term) => { + expect(validateWritableQuads('quads', [quad({ [field]: term })])).toBeNull(); + }); + it.each(invalidLabels)('rejects malformed or injectable label %s', (term) => { + expect(validateWritableQuads('quads', [quad({ [field]: term })])?.error).toContain(`quads[0].${field}`); + }); +}); + +it.each(['"literal"', 'urn:o', 'https://example.org/o'])('preserves ordinary RDF object %s', (object) => { + expect(validateWritableQuads('quads', [quad({ object })])).toBeNull(); +}); +// N-Triples allows ':' inside a label. The write routes take the SPARQL 1.1 +// form (rdf-utils), which does not, so the label stays writable in an update. +it.each(['subject', 'object'] as const)('rejects a colon in a blank-node %s label', (field) => { + expect(validateWritableQuads('quads', [quad({ [field]: '_::x' })])?.error).toContain(`quads[0].${field}`); +}); +it.each(['hello', '123', 'urn:o> . { + expect(validateWritableQuads('quads', [quad({ object })])?.error).toContain('quads[0].object'); +}); diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json index e0a7968c90..222feb04a9 100644 --- a/packages/cli/tsconfig.json +++ b/packages/cli/tsconfig.json @@ -15,6 +15,7 @@ { "path": "../node-ui" }, { "path": "../adapter-openclaw" }, { "path": "../local-llm" }, - { "path": "../mcp-dkg" } + { "path": "../mcp-dkg" }, + { "path": "../rdf-utils" } ] } diff --git a/packages/cli/vitest.unit.config.ts b/packages/cli/vitest.unit.config.ts index b88826fb31..2ac06c5194 100644 --- a/packages/cli/vitest.unit.config.ts +++ b/packages/cli/vitest.unit.config.ts @@ -15,6 +15,7 @@ export default defineConfig({ 'test/mcp-config-document.test.ts', 'test/mcp-physical-config.test.ts', 'test/mcp-config-metadata.test.ts', + 'test/rdf-parser.test.ts', 'test/api-client.test.ts', 'test/finalized-publish-options.test.ts', 'test/messenger-outbox-memory.test.ts', @@ -252,6 +253,7 @@ export default defineConfig({ // OxigraphWorkerStore; no hardhat needed. 'test/write-preflight-resilience.test.ts', 'test/http-literal-size-validation.test.ts', + 'test/writable-quad-terms.test.ts', 'test/http-quad-term-validation.test.ts', // CLI subprocess smoke with stub daemon only; no hardhat needed. 'test/context-graph-join-policy-cli.test.ts', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c487fc3e3b..0137222389 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -751,6 +751,9 @@ importers: jsonc-parser: specifier: 3.3.1 version: 3.3.1 + jsonld: + specifier: ^8.3.3 + version: 8.3.3(web-streams-polyfill@3.3.3) n3: specifier: ^2.0.1 version: 2.0.1 @@ -770,6 +773,9 @@ importers: '@types/js-yaml': specifier: ^4.0.9 version: 4.0.9 + '@types/jsonld': + specifier: ^1.5.15 + version: 1.5.15 '@types/n3': specifier: ^1.26.1 version: 1.26.1 @@ -3814,6 +3820,9 @@ packages: '@types/jsesc@2.5.1': resolution: {integrity: sha512-9VN+6yxLOPLOav+7PwjZbxiID2bVaeq0ED4qSQmdQTdjnXJSaCVKTR58t15oqH1H5t8Ng2ZX1SabJVoN9Q34bw==} + '@types/jsonld@1.5.15': + resolution: {integrity: sha512-PlAFPZjL+AuGYmwlqwKEL0IMP8M8RexH0NIPGfCVWSQ041H2rR/8OlyZSD7KsCVoN8vCfWdtWDBxX8yBVP+xow==} + '@types/mdast@4.0.4': resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==} @@ -10489,6 +10498,8 @@ snapshots: '@types/jsesc@2.5.1': {} + '@types/jsonld@1.5.15': {} + '@types/mdast@4.0.4': dependencies: '@types/unist': 3.0.3