QVAC-27164 feat[api|bc]: support EmbeddingGemma 2 multimodal embeddings

docs/website/content/_unpublished/sdk/next/ai-capabilities/meta.json

@@ -3,6 +3,7 @@
   "pages": [
     "text-generation",
     "text-embeddings",
+    "multimodal-embeddings",
     "rag",
     "fine-tuning",
     "multimodal",

docs/website/content/_unpublished/sdk/next/ai-capabilities/multimodal-embeddings.mdx

@@ -0,0 +1,86 @@
+---
+title: Multimodal embeddings
+icon: Hash
+description: Embed text, images, and audio in a shared vector space for cross-media search and retrieval.
+schemaType: HowTo
+---
+
+## Overview
+
+Multimodal embeddings use the Fabric LLM inference engine through [`@qvac/embed-llamacpp`](https://github.com/tetherto/qvac/tree/main/packages/embed-llamacpp). Load EmbeddingGemma 2 using `modelType: "llamacpp-embedding"` to embed text, images, and audio in a shared vector space.
+
+Pass an `inputs` array to [`embed()`](/sdk/reference/api#embed). It returns `number[][]`, with one 768-dimensional vector per item. Compare these vectors to retrieve images or audio using a text query.
+
+<Callout type="info">
+For text-only models and the `text` input form, see [Text embeddings](/sdk/ai-capabilities/text-embeddings).
+</Callout>
+
+## Functions
+
+Use the following sequence of function calls:
+
+1. [`loadModel()`](/sdk/reference/api#loadmodel)
+2. [`embed()`](/sdk/reference/api#embed)
+3. [`unloadModel()`](/sdk/reference/api#unloadmodel)
+
+For how to use each function, see [SDK API reference](/sdk/reference/api/).
+
+## Models
+
+Use `EMBEDDINGGEMMA_2_Q8_0` for the model and `MMPROJ_EMBEDDINGGEMMA_2_Q8_0` for its matching image and audio projector. Import both constants from `@qvac/sdk`, load the model with `modelType: "llamacpp-embedding"`, and pass the projector through `modelConfig.projectionModelSrc`. Model and projector sources also accept local paths, HTTP(S) URLs, and registry URIs through the usual model download lifecycle.
+
+For models available as constants, see [SDK Models](/sdk#models).
+
+## Inputs
+
+Pass `inputs` to [`embed()`](/sdk/reference/api#embed). Each item is a text string or an ordered array of text strings and `{ media: { type, value } }` parts. Use `type: "base64"` for encoded image or audio bytes, or `type: "filePath"` for a path accessible to the inference worker. The result is `number[][]`, with one vector per input item, including a batch with only one item. Plain text still works with a loaded projector. Choose either `text` or `inputs` in each call. Task prompts such as `task: search result | query: ` remain text you supply.
+
+Media requires a projector. Images include JPEG, PNG and BMP; audio supports WAV, MP3 and FLAC. WebP, AAC/M4A, Ogg and video are unsupported.
+
+## Model configuration
+
+* `ctxSize` sets the runtime context size in tokens. Omitted, it uses the trained context size. Set it explicitly to bound memory for long-context models.
+* `batchSize` defaults to `1024`. A mixed input's text and media tokens must fit together in this budget and in `ctxSize`. A large EmbeddingGemma 2 image can take 1120 tokens; the example uses `2048` for both settings.
+* `imageMaxTokens` and `imageMinTokens` set positive per-image token budgets. Omitted, they keep the projector defaults. The minimum must not exceed the maximum.
+* `mmprojNoAudio: true` skips loading the audio encoder and rejects audio input. The default is `false`.
+* `mmprojUseGpu` overrides projector placement. Omitted, it uses the addon's device policy. It is ignored when the model runs on CPU.
+* `imageMaxMegapixels` limits image size before decoding. The default is `50`.
+* `audioMaxSeconds` limits audio duration before decoding. The default is `600`.
+
+Projector options require `projectionModelSrc`. A memory assessment with a projector is unsupported because the fitter does not count projector memory. Each model processes one request at a time; an overlapping call rejects as busy. Await the current call before starting another. Cancellation uses the existing [`cancel()`](/sdk/reference/api#cancel) API and applies to the model's active job.
+
+## Example
+
+Save the example as `embed-multimodal.ts` and pass an image and an audio file:
+
+```bash
+npx tsx embed-multimodal.ts image.jpg audio.wav
+```
+
+The following script embeds text, an image, and audio with EmbeddingGemma 2 in one shared vector space:
+
+<Tabs>
+<Tab value="js" label="JavaScript" default>
+<WrapCode>
+
+```js file=<rootDir>/packages/sdk/dist/examples/embed-multimodal.js title="embed-multimodal.js" lineNumbers
+```
+</WrapCode>
+</Tab>
+
+<Tab value="ts" label="TypeScript">
+<WrapCode>
+
+```ts file=<rootDir>/packages/sdk/examples/embed-multimodal.ts title="embed-multimodal.ts" lineNumbers
+```
+</WrapCode>
+</Tab>
+</Tabs>
+
+<Callout type="info">
+The Python client supports this capability through the same worker. A dedicated Python example is not yet published. See the [Python SDK](/sdk/python-sdk) for the API surface.
+</Callout>
+
+<Callout type="success">
+**Tip:** all examples throughout this documentation are self-contained and runnable. For instructions on how to run them, see the [JS/TS quickstart](/sdk/js-ts-sdk#quickstart) or the [Python quickstart](/sdk/python-sdk#quickstart).
+</Callout>

docs/website/content/_unpublished/sdk/next/ai-capabilities/text-embeddings.mdx

@@ -11,6 +11,10 @@ Text embeddings uses [`qvac-fabric-llm.cpp`](https://github.com/tetherto/qvac-fa
 
 `embed()` returns a single embedding vector (`number[]`) for single text input, or an array of embedding vectors (`number[][]`) for batch input.
 
+<Callout type="info">
+For image and audio inputs with EmbeddingGemma 2, see [Multimodal embeddings](/sdk/ai-capabilities/multimodal-embeddings).
+</Callout>
+
 `modelConfig.pooling` accepts `"none"`, `"mean"`, `"cls"`, or `"last"`. Omit it to use the model default. `"rank"` fails validation, including in `deviceDefaults` in [`qvac.config.*`](/sdk/configuration): it produces reranker scores, not embeddings, and `embed()` does not support reranker models; use [reranking](/sdk/ai-capabilities/reranking) for those.
 
 ## Functions

docs/website/content/_unpublished/sdk/next/index.mdx

@@ -57,6 +57,7 @@ The SDK is cross-platform, type-safe, and pluggable, exposing all QVAC capabilit
 
 * [**Text generation:**](/sdk/ai-capabilities/text-generation) LLM inference for text generation and chat via [`qvac-fabric-llm.cpp`](https://github.com/tetherto/qvac-fabric-llm.cpp).
 * [**Text embeddings:**](/sdk/ai-capabilities/text-embeddings) vector embedding generation for semantic search, clustering, and retrieval, via `qvac-fabric-llm.cpp`.
+* [**Multimodal embeddings:**](/sdk/ai-capabilities/multimodal-embeddings) text, image, and audio embeddings in a shared vector space for cross-media search and retrieval.
 * [**RAG:**](/sdk/ai-capabilities/rag) out-of-the-box retrieval-augmented generation workflow.
 * [**Fine-tuning:**](/sdk/ai-capabilities/fine-tuning) adapting LLMs to domain-specific tasks via LoRA.
 * [**Multimodal:**](/sdk/ai-capabilities/multimodal) LLM inference over text, images, and other media within a single conversation context.

packages/inference/src/api/embed.ts

@@ -1,6 +1,7 @@
 import { send } from '@/dispatch'
 import {
   type EmbedParams,
+  type EmbedInput,
   type EmbedRequest,
   type EmbedStats,
   type RPCOptions
@@ -21,7 +22,7 @@ import { generateRequestId } from '@/runtime/request-id'
  * @throws {QvacErrorBase} When the response type is invalid or when the embedding fails
  */
 export function embed(
-  params: { modelId: string; text: string },
+  params: { modelId: string; text: string; inputs?: never },
   options?: RPCOptions
 ): Promise<{ embedding: number[]; stats?: EmbedStats }> & { requestId: string }
 
@@ -37,12 +38,39 @@ export function embed(
  * @throws {QvacErrorBase} When the response type is invalid or when the embedding fails
  */
 export function embed(
-  params: { modelId: string; text: string[] },
+  params: { modelId: string; text: string[]; inputs?: never },
   options?: RPCOptions
 ): Promise<{ embedding: number[][]; stats?: EmbedStats }> & {
   requestId: string
 }
 
+/**
+ * Embeds ordered text and media inputs.
+ *
+ * @overloadLabel "Text and media batch"
+ * @param params - Model identifier and ordered inputs to embed
+ * @param options - Optional RPC options including per-call profiling
+ * @returns One embedding vector per input, with optional statistics and a synchronous requestId for cancellation.
+ * @throws {QvacErrorBase} When validation or embedding fails
+ */
+export function embed(
+  params: { modelId: string; inputs: EmbedInput[]; text?: never },
+  options?: RPCOptions
+): Promise<{ embedding: number[][]; stats?: EmbedStats }> & { requestId: string }
+
+/**
+ * Embeds text or a batch of ordered text and media inputs.
+ *
+ * @param params - Model identifier and either text or inputs
+ * @param options - Optional RPC options including per-call profiling
+ * @returns Embeddings with optional statistics and a synchronous requestId for cancellation.
+ * @throws {QvacErrorBase} When validation or embedding fails
+ */
+export function embed(
+  params: EmbedParams,
+  options?: RPCOptions
+): Promise<{ embedding: number[] | number[][]; stats?: EmbedStats }> & { requestId: string }
+
 export function embed(
   params: EmbedParams,
   options?: RPCOptions

packages/inference/src/handlers/assess-model-fit.ts

@@ -206,7 +206,7 @@ export function estimateTargetFor(candidate: ModelFitCandidate): ModelFitEstimat
   const config = candidate.modelConfig ?? {}
   const modelType = modelTypeOf(candidate)
 
-  const contextTokens = config['ctx_size']
+  const contextTokens = config['ctxSize'] ?? config['ctx_size']
   const artifacts = companionRefs(config)
   const device = resolvedDevice(modelType, config)
   const windowMs =

packages/inference/src/logging/utils.ts

@@ -77,6 +77,10 @@ export function summarizeRequest(request: Request): Record<string, unknown> {
   }
 
   if (request.type === 'embed') {
+    if (request.inputs !== undefined) {
+      summary['inputs'] = `[${request.inputs.length} embedding inputs]`
+      return summary
+    }
     const text = request['text']
     if (Array.isArray(text)) {
       const totalLen = text.reduce((sum, t) => sum + t.length, 0)

packages/inference/src/model-fit/request/engines/llama.ts

@@ -30,7 +30,7 @@ export interface LlamaFitRequestParams {
 export function createLlamaFitRequest(params: LlamaFitRequestParams): FitRequestPlan {
   const { loadKind } = params
 
-  if (loadKind === 'completion' && params.artifacts?.['projectionModelPath'] !== undefined) {
+  if (params.artifacts?.['projectionModelPath'] !== undefined) {
     return { supported: false, detail: 'multimodal projection loads are not representable' }
   }
 

packages/inference/src/plugins/builtin/llamacpp-embedding/plugin.ts

@@ -6,15 +6,17 @@ import {
   embedRequestSchema,
   embedResponseSchema,
   ModelType,
-  embedConfigBaseSchema,
+  embedLoadConfigSchema,
   ADDON_EMBEDDING,
   type CreateModelParams,
   type PluginModelResult,
-  type EmbedConfig
+  type EmbedConfig,
+  type EmbedConfigInput,
+  type ResolveContext
 } from '@/schemas/index'
 import { createStreamLogger, registerAddonLogger, getEngineLogger } from '@/logging/index'
 import { getFirstShardPath } from '@/utils/index'
-import { embed } from '@/plugins/ops/embed'
+import { embed, markEmbeddingProjector } from '@/plugins/ops/embed'
 import { forwardModelExecution } from '@/profiling/model-execution'
 import { isMobile } from '@/runtime/state'
 import { stripMultiGpuKeys } from '@/utils/multi-gpu-mobile'
@@ -42,7 +44,12 @@ const turbovecIndexProvider: TurboVecIndexProvider = {
   }
 }
 
-function createEmbeddingsModel(modelId: string, modelPath: string, embedConfig: EmbedConfig) {
+function createEmbeddingsModel(
+  modelId: string,
+  modelPath: string,
+  embedConfig: EmbedConfig,
+  projectionModelPath?: string
+) {
   const logger = createStreamLogger(modelId, ModelType.llamacppEmbedding)
   registerAddonLogger(modelId, ModelType.llamacppEmbedding, logger)
 
@@ -58,11 +65,15 @@ function createEmbeddingsModel(modelId: string, modelPath: string, embedConfig:
   }
 
   const model = new EmbedLlamacpp({
-    files: { model: [getFirstShardPath(modelPath)] },
+    files: {
+      model: [getFirstShardPath(modelPath)],
+      ...(projectionModelPath && { projectionModel: projectionModelPath })
+    },
     config,
     logger,
     opts: { stats: true }
   })
+  if (projectionModelPath) markEmbeddingProjector(model)
 
   return { model }
 }
@@ -71,13 +82,25 @@ export const embeddingsPlugin = definePlugin({
   modelType: ModelType.llamacppEmbedding,
   displayName: 'Embeddings (llama.cpp)',
   addonPackage: ADDON_EMBEDDING,
-  loadConfigSchema: embedConfigBaseSchema,
+  loadConfigSchema: embedLoadConfigSchema,
   assessFit: embedAssessFit,
 
+  async resolveConfig(cfg: EmbedConfigInput, ctx: ResolveContext) {
+    const { projectionModelSrc, ...config } = cfg
+    if (!projectionModelSrc) return { config }
+    const projectionModelPath = await ctx.resolveModelPath(projectionModelSrc)
+    return { config, artifacts: { projectionModelPath } }
+  },
+
   createModel(params: CreateModelParams): PluginModelResult {
     const embedConfig = (params.modelConfig ?? {}) as EmbedConfig
 
-    const { model } = createEmbeddingsModel(params.modelId, params.modelPath, embedConfig)
+    const { model } = createEmbeddingsModel(
+      params.modelId,
+      params.modelPath,
+      embedConfig,
+      params.artifacts?.['projectionModelPath']
+    )
 
     return { model }
   },
@@ -93,10 +116,9 @@ export const embeddingsPlugin = definePlugin({
 
       handler: async function (request) {
         const embedResult = await embed(
-          {
-            modelId: request.modelId,
-            text: request.text
-          },
+          request.inputs !== undefined
+            ? { modelId: request.modelId, inputs: request.inputs }
+            : { modelId: request.modelId, text: request.text },
           request.requestId
         )
 

packages/inference/src/plugins/builtin/llamacpp-embedding/transform.ts

@@ -14,6 +14,10 @@ export function transformEmbedConfig(embedConfig: EmbedConfig): GGMLConfig {
     batch_size: `${embedConfig.batchSize}` as `${number}`
   }
 
+  if (embedConfig.ctxSize !== undefined) {
+    config.ctx_size = `${embedConfig.ctxSize}`
+  }
+
   if (embedConfig.poll !== undefined) {
     config.poll = `${embedConfig.poll}`
   }
@@ -63,5 +67,29 @@ export function transformEmbedConfig(embedConfig: EmbedConfig): GGMLConfig {
     config.openclCacheDir = embedConfig.openclCacheDir
   }
 
+  if (embedConfig.imageMaxTokens !== undefined) {
+    config['image-max-tokens'] = `${embedConfig.imageMaxTokens}`
+  }
+
+  if (embedConfig.imageMinTokens !== undefined) {
+    config['image-min-tokens'] = `${embedConfig.imageMinTokens}`
+  }
+
+  if (embedConfig.imageMaxMegapixels !== undefined) {
+    config['image-max-megapixels'] = `${embedConfig.imageMaxMegapixels}`
+  }
+
+  if (embedConfig.audioMaxSeconds !== undefined) {
+    config['audio-max-seconds'] = `${embedConfig.audioMaxSeconds}`
+  }
+
+  if (embedConfig.mmprojNoAudio !== undefined) {
+    config['mmproj-no-audio'] = embedConfig.mmprojNoAudio ? '1' : '0'
+  }
+
+  if (embedConfig.mmprojUseGpu !== undefined) {
+    config['mmproj-use-gpu'] = embedConfig.mmprojUseGpu ? '1' : '0'
+  }
+
   return config
 }