Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions packages/cli/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,23 @@ the effective lane once at boot. A background query that is shed before
execution returns HTTP 503 with `Retry-After: 1` and
`code: "STORE_SCHEDULER_BUSY"`.

## RDF file input

Commands with `--file` accept N-Quads (`.nq`), N-Triples (`.nt`), Turtle (`.ttl`),
TriG (`.trig`), JSON quad arrays (`.json`), and JSON-LD (`.jsonld`). JSON-LD
supports inline `@context`, lists, named graphs and typed/language literals.
Relative identifiers resolve against the input file's `file:` URL unless an
inline `@base` overrides it. Expansion that would discard statements fails.

JSON-LD named graphs can be stored in Working Memory with `dkg ka create --no-finalize`.
A default-finalizing create rejects them before contacting the daemon: sealing, SWM
sharing and VM publication do not yet preserve named-graph identity. Rewrite the
document into the default graph to use those transitions.

JSON-LD ingestion does not fetch remote contexts or `@import` URLs. Embed the
required context inline before importing a file. Existing simple quad arrays
remain supported in both `.json` and `.jsonld` files.

## Running a Core Node (relay operator)

A Core Node is a publicly-reachable host that runs a libp2p circuit-relay v2
Expand Down
7 changes: 5 additions & 2 deletions packages/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
"@origintrail-official/dkg-node-ui": "workspace:*",
"@origintrail-official/dkg-okf": "workspace:*",
"@origintrail-official/dkg-publisher": "workspace:*",
"@origintrail-official/dkg-rdf-utils": "workspace:*",
"@origintrail-official/dkg-storage": "workspace:*",
"@toml-tools/parser": "1.0.0",
"better-sqlite3": "12.11.1",
Expand All @@ -66,7 +67,8 @@
"jsonc-parser": "3.3.1",
"n3": "^2.0.1",
"semver": "^7.7.4",
"typescript": "^5.7"
"typescript": "^5.7",
"jsonld": "^8.3.3"
},
"devDependencies": {
"@types/better-sqlite3": "^7",
Expand All @@ -75,7 +77,8 @@
"@types/semver": "^7.8.0",
"@vitest/coverage-v8": "^4.0.18",
"esbuild": "0.27.7",
"vitest": "^4.0.18"
"vitest": "^4.0.18",
"@types/jsonld": "^1.5.15"
},
"publishConfig": {
"access": "public"
Expand Down
24 changes: 15 additions & 9 deletions packages/cli/src/cli-helpers.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import type { ParsedRdf, SimpleQuad } from './rdf-parser.js';
import { Command } from 'commander';
import { readFileSync } from 'node:fs';
import { createInterface } from 'node:readline';
Expand Down Expand Up @@ -148,41 +149,45 @@ function loadStructuredFile(filePath: string): any {
return yaml.load(content);
}

async function loadQuadsFromInput(
async function loadRdfFromInput(
opts: ActionOpts,
defaultGraph: string,
): Promise<Array<{ subject: string; predicate: string; object: string; graph: string }>> {
): Promise<ParsedRdf> {
const rdfParser = await import('./rdf-parser.js');

if (opts.file) {
const { readFile } = await import('node:fs/promises');
const { pathToFileURL } = await import('node:url');
const raw = await readFile(opts.file, 'utf-8');
const format = opts.format ?? rdfParser.detectFormat(opts.file);
const quads = await rdfParser.parseRdf(raw, format, defaultGraph);
console.log(`Parsed ${quads.length} quad(s) from ${opts.file} (${format})`);
return quads;
const input = await rdfParser.parseRdfInput(raw, format, defaultGraph, pathToFileURL(opts.file).href);
console.log(`Parsed ${input.quads.length} quad(s) from ${opts.file} (${format})`);
return input;
}

if (opts.triples) {
const parsed = JSON.parse(opts.triples);
return parsed.map((q: Record<string, string>) => ({ ...q, graph: q.graph || defaultGraph }));
return rdfParser.parseRdfInput(opts.triples, 'json', defaultGraph);
}

if (opts.subject && opts.predicate && opts.object) {
return [{
return { sourceKind: 'legacy-quads', quads: [{
subject: opts.subject,
predicate: opts.predicate,
object: opts.object.startsWith('"') || opts.object.startsWith('http') || opts.object.startsWith('did:')
? opts.object
: `"${opts.object}"`,
graph: defaultGraph,
}];
}] };
}

console.error(`Provide --file (${rdfParser.supportedExtensions().join(', ')}), --triples, or --subject/--predicate/--object`);
process.exit(1);
}

async function loadQuadsFromInput(opts: ActionOpts, defaultGraph: string): Promise<SimpleQuad[]> {
return (await loadRdfFromInput(opts, defaultGraph)).quads;
}

function probeHostForApiHost(apiHost: string | undefined): string {
if (!apiHost || apiHost === '0.0.0.0') return '127.0.0.1';
if (apiHost === '::') return '::1';
Expand Down Expand Up @@ -374,6 +379,7 @@ export {
parseOptionalVerifyTimeoutOption,
loadStructuredFile,
loadQuadsFromInput,
loadRdfFromInput,
resolveDaemonEntryPoint,
probeHostForApiHost,
selectedDkgHomeForEnv,
Expand Down
30 changes: 12 additions & 18 deletions packages/cli/src/commands/knowledge-asset.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,11 @@ import {
type KnowledgeAssetPublishResponse,
type KnowledgeAssetShareJobState,
type KnowledgeAssetShareResponse,
type KnowledgeAssetWritableQuad,
type PreSignedAuthorAttestationPayload,
} from '../api-client.js';
import {
formatQuadObject,
loadQuadsFromInput,
loadRdfFromInput,
loadStructuredFile,
type ActionOpts,
} from '../cli-helpers.js';
Expand Down Expand Up @@ -53,20 +52,8 @@ function hasQuadInput(opts: ActionOpts): boolean {
);
}

async function loadWritableQuads(opts: ActionOpts): Promise<KnowledgeAssetWritableQuad[]> {
const quads = await loadQuadsFromInput(
{
...opts,
file: inputFilePath(opts),
},
'',
);
return quads.map((quad) => ({
subject: quad.subject,
predicate: quad.predicate,
object: quad.object,
graph: quad.graph ?? '',
}));
async function loadWritableInput(opts: ActionOpts) {
return loadRdfFromInput({ ...opts, file: inputFilePath(opts) }, '');
}

function parsePreSignedAuthorAttestation(raw: unknown): PreSignedAuthorAttestationPayload | undefined {
Expand Down Expand Up @@ -243,7 +230,14 @@ export function registerKnowledgeAssetCommand(program: Command): void {
))))
.action(async (name: string, opts: ActionOpts) => runAction(async () => {
const contextGraphId = requiredContextGraphId(opts);
const quads = hasQuadInput(opts) ? await loadWritableQuads(opts) : undefined;
const input = hasQuadInput(opts) ? await loadWritableInput(opts) : undefined;
const quads = input?.quads;
if (opts.finalize !== false && input?.sourceKind === 'jsonld' && quads?.some((quad) => quad.graph !== '')) {
Comment thread
branarakic-agent marked this conversation as resolved.
throw new Error(
'JSON-LD named graphs cannot be finalized yet. Use ka create --no-finalize to keep them in Working Memory, '
+ 'or rewrite the document into the default graph before finalizing or sharing.',
);
}
if (opts.share === true && (!quads || quads.length === 0 || opts.finalize === false)) {
throw new Error('--share requires non-empty payload quads and finalize enabled');
}
Expand Down Expand Up @@ -279,7 +273,7 @@ export function registerKnowledgeAssetCommand(program: Command): void {
)))
.action(async (name: string, opts: ActionOpts) => runAction(async () => {
const contextGraphId = requiredContextGraphId(opts);
const quads = await loadWritableQuads(opts);
const { quads } = await loadWritableInput(opts);
Comment thread
branarakic-agent marked this conversation as resolved.
const client = await ApiClient.connect();
const result = await client.knowledgeAssetWrite(contextGraphId, name, quads, {
...(subGraphName(opts) ? { subGraphName: subGraphName(opts) } : {}),
Expand Down
41 changes: 23 additions & 18 deletions packages/cli/src/daemon/http-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -266,26 +266,31 @@ export function validateWritableQuadLiteralSizes(
}
}

/**
* GH #306 / #787 (follow-up) — validate each quad's `object` term is either a
* quoted RDF literal (`"…"`) or an absolute IRI. Shared by lifecycle write
* routes and other quad-accepting validation paths: the shape guard
* ({@link isWritableQuad}) only checks that fields
* are strings, so an object that is neither a literal nor an IRI (e.g. a bare
* word `hello` or a number `123`) slips past them and crashes the RDF parser
* with an uncaught "No scheme found in an absolute IRI" → HTTP 500 instead of an
* actionable 400.
*/
export function validateQuadObjectTerms(
// N-Triples BLANK_NODE_LABEL (including its Unicode name ranges).
Comment thread
branarakic-agent marked this conversation as resolved.
// https://www.w3.org/TR/n-triples/#grammar-production-BLANK_NODE_LABEL
const PN_CHARS_BASE = String.raw`A-Za-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u{10000}-\u{EFFFF}`;
const PN_CHARS_U = `${PN_CHARS_BASE}_:`;
const PN_CHARS = String.raw`${PN_CHARS_U}0-9\-\u00B7\u0300-\u036F\u203F-\u2040`;
const BLANK_NODE_LABEL = new RegExp(`^_:[${PN_CHARS_U}0-9](?:[${PN_CHARS}.]*[${PN_CHARS}])?$`, 'u');

/** Preserve valid RDF blank-node links while rejecting unsafe labels before storage. */
export function validateWritableQuadTerms(
label: string,
quads: ReadonlyArray<{ object: string }>,
quads: ReadonlyArray<{ subject: string; object: string }>,
): string | null {
const badIndex = quads.findIndex((q) => {
const object = q.object.trim();
return !object.startsWith('"') && !isSafeIri(object);
});
if (badIndex === -1) return null;
return `Invalid "${label}[${badIndex}].object": RDF object must be a quoted literal term or absolute IRI`;
for (const [index, quad] of quads.entries()) {
if (quad.subject.trim().startsWith('_:') && !BLANK_NODE_LABEL.test(quad.subject)) {
return `Invalid "${label}[${index}].subject": RDF blank node must have a valid blank-node label`;
}
const object = quad.object.trim();
const validObject = object.startsWith('_:')
? BLANK_NODE_LABEL.test(quad.object)
: object.startsWith('"') || isSafeIri(object);
if (!validObject) {
return `Invalid "${label}[${index}].object": RDF object must be a quoted literal term, absolute IRI, or valid blank-node label`;
}
}
return null;
}

/**
Expand Down
6 changes: 3 additions & 3 deletions packages/cli/src/daemon/routes/knowledge-assets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ import {
validateOptionalSubGraphName,
validateRequiredContextGraphId,
isWritableQuad,
validateQuadObjectTerms,
validateWritableQuadTerms,
respondIfReconcileUnavailable,
respondIfStoreUnavailable,
classifyStoreUnavailable,
Expand Down Expand Up @@ -1353,8 +1353,8 @@ export async function handleKnowledgeAssetsRoutes(ctx: RequestContext): Promise<
return jsonResponse(res, 400, { error: '"quads" must be an array of { subject, predicate, object } objects (graph optional); string-shaped quads are not accepted' });
}
// GH #306/#787 (follow-up) — reject objects that are neither a quoted
// literal nor an absolute IRI before they reach (and crash) the parser.
const wmObjErr = validateQuadObjectTerms("quads", parsed.quads);
// literal, absolute IRI or valid blank node before they reach storage.
const wmObjErr = validateWritableQuadTerms("quads", parsed.quads);
if (wmObjErr) return jsonResponse(res, 400, { error: wmObjErr });
const literalSize = validateWritableQuadLiteralSizes("quads", parsed.quads);
if (!literalSize.ok) return jsonResponse(res, 400, literalSize.body);
Expand Down
1 change: 0 additions & 1 deletion packages/cli/src/daemon/routes/memory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,6 @@ import {
import {
resolveNameToPeerId,
isWritableQuad,
validateQuadObjectTerms,
validateWritableQuadLiteralSizes,
oversizedRdfLiteralResponseBody,
jsonResponse,
Expand Down
Loading
Loading