Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
"test:versions": "node scripts/check-versions.mjs",
"test": "npm run test:versions && npm run test:js && npm run test:python && npm run test:rust",
"check": "npm run check -w @embedding-atlas/component -w @embedding-atlas/viewer -w @embedding-atlas/embedding-atlas -w @embedding-atlas/examples -w @embedding-atlas/backend",
"check-format": "prettier -c . && uvx ruff format --check --exclude '*.ipynb' --exclude '*.md'"
"check-format": "prettier -c . && uv run --frozen --only-group lint ruff format --check --exclude '*.ipynb' --exclude '*.md'"
},
"devDependencies": {
"prettier": "^3.8.1",
Expand Down
3 changes: 2 additions & 1 deletion packages/backend/embedding_atlas/async_map.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@

import asyncio
import random
from typing import Awaitable, Callable, TypeVar
from collections.abc import Awaitable, Callable
from typing import TypeVar

from tqdm.auto import tqdm

Expand Down
11 changes: 6 additions & 5 deletions packages/backend/embedding_atlas/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,16 +31,17 @@
"""

import base64
import functools
import hashlib
import hmac
import json
import logging
import secrets
import struct
from functools import lru_cache
from collections.abc import Callable
from io import BytesIO, TextIOWrapper
from pathlib import Path
from typing import IO, Any, Callable
from typing import IO, Any

import numpy as np
from cryptography.hazmat.primitives import hashes
Expand Down Expand Up @@ -267,7 +268,7 @@ async def async_file_cache_value(

if cache_path.exists():
try:
with open(cache_path, "rb") as file:
with open(cache_path, "rb") as file: # noqa: ASYNC230
data = _decrypt_data(file.read(), key=encryption_key)

result = deserializer(BytesIO(data))
Expand All @@ -290,7 +291,7 @@ async def async_file_cache_value(
serializer(value, buffer)
encrypted_data = _encrypt_data(buffer.getvalue(), key=encryption_key)

with open(cache_path_tmp, "wb") as file:
with open(cache_path_tmp, "wb") as file: # noqa: ASYNC230
file.write(encrypted_data)

cache_path_tmp.rename(cache_path)
Expand All @@ -307,7 +308,7 @@ def _resolve_cache_root(cache_root: str | Path | None = None) -> Path:
return Path(cache_root).resolve()


@lru_cache(maxsize=None)
@functools.cache
def _get_constants(cache_root: Path) -> dict[str, bytes]:
cache_root.mkdir(parents=True, exist_ok=True)
constants_path = cache_root / "cache_constants.json"
Expand Down
11 changes: 11 additions & 0 deletions packages/backend/embedding_atlas/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -390,6 +390,15 @@ def import_modules(names: list[str]):
default=None,
help="Path to a file containing labels for the embedding view. The file should be a table with columns 'x', 'y', 'text', and optionally 'level' and 'priority'",
)
@click.option(
"--state",
"initial_state",
type=JSONParamType(),
default=None,
help="Initial app state (charts, layouts, column styles, ...), as exported from the app. "
"Pass a JSON string or a path to a JSON file. If the state contains charts, "
"they replace the default charts.",
)
@click.option(
"--mcp/--no-mcp",
"enable_mcp",
Expand Down Expand Up @@ -441,6 +450,7 @@ def main(
point_size: float | None,
stop_words: str | None,
labels: str | None,
initial_state: dict | None,
enable_mcp: bool,
):
apply_logging_config()
Expand Down Expand Up @@ -654,6 +664,7 @@ def main(
point_size=point_size,
stop_words=stop_words_resolved,
labels=labels_resolved,
initial_state=initial_state,
additional_tables=additional_tables_meta or None,
)

Expand Down
20 changes: 0 additions & 20 deletions packages/backend/embedding_atlas/options.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,15 +54,6 @@ class EmbeddingAtlasOptions(TypedDict, total=False):
point_size:
Override the default point size for the embedding view.

show_table:
Whether to display the data table when the widget opens.

show_charts:
Whether to display charts when the widget opens.

show_embedding:
Whether to display the embedding view when the widget opens.

initial_state:
The initial Embedding Atlas state.

Expand All @@ -89,10 +80,6 @@ class EmbeddingAtlasOptions(TypedDict, total=False):
labels: list[dict] | None
stop_words: list[str] | None

show_table: bool | None
show_charts: bool | None
show_embedding: bool | None

initial_state: dict | None

additional_tables: dict | None
Expand Down Expand Up @@ -149,11 +136,4 @@ def set_prop(key: str, value):
# Additional tables (id/relation for cross-table filtering)
set_prop("additionalTables", options.get("additional_tables"))

# Layout
set_prop("initialState.layoutStates.list.showTable", options.get("show_table"))
set_prop("initialState.layoutStates.list.showCharts", options.get("show_charts"))
set_prop(
"initialState.layoutStates.list.showEmbedding", options.get("show_embedding")
)

return props
4 changes: 2 additions & 2 deletions packages/backend/embedding_atlas/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
import json
import re
import uuid
from collections.abc import Callable
from functools import lru_cache, partial
from io import BytesIO
from typing import Callable

import duckdb
from fastapi import FastAPI, HTTPException, Request, Response, WebSocket
Expand Down Expand Up @@ -282,7 +282,7 @@ async def send_request(self, request: dict) -> dict:
raise HTTPException(status_code=503, detail="WebSocket disconnected")
else:
raise HTTPException(
status_code=500, detail=f"Internal server error: {str(e)}"
status_code=500, detail=f"Internal server error: {e!s}"
)

async def send_close(self):
Expand Down
9 changes: 0 additions & 9 deletions packages/backend/embedding_atlas/streamlit.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,15 +59,6 @@ def embedding_atlas(
point_size:
Override the default point size for the embedding view.

show_table:
Whether to display the data table when the widget opens.

show_charts:
Whether to display charts when the widget opens.

show_embedding:
Whether to display the embedding view when the widget opens.

key:
The key of the Streamlit widget.

Expand Down
5 changes: 3 additions & 2 deletions packages/backend/embedding_atlas/utils.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# Copyright (c) 2025 Apple Inc. Licensed under MIT License.

import logging
import sys
from pathlib import Path
from typing import Any

Expand Down Expand Up @@ -35,7 +36,7 @@ def load_huggingface_data(filename: str, splits: list[str] | None) -> pd.DataFra
print(
"⚠️ Loading Hugging Face datasets requires the `datasets` package to be installed. Please run `pip install datasets`, then try again."
)
exit(-1)
sys.exit(-1)

if Path(filename).is_dir():
ds: Any = load_from_disk(filename)
Expand All @@ -51,7 +52,7 @@ def load_huggingface_data(filename: str, splits: list[str] | None) -> pd.DataFra

if splits is None or len(splits) == 0:
ds_split_options = []
for key in ds.keys():
for key in ds:
option = (f"{key} ({ds[key].num_rows} rows)", key)
ds_split_options.append(option)
split_question = [
Expand Down
9 changes: 0 additions & 9 deletions packages/backend/embedding_atlas/widget.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,15 +85,6 @@ def __init__(
point_size:
Override the default point size for the embedding view.

show_table:
Whether to display the data table when the widget opens.

show_charts:
Whether to display charts when the widget opens.

show_embedding:
Whether to display the embedding view when the widget opens.

connection (DuckDBPyConnection, optional):
A DuckDB connection. Defaults to duckdb.connect().
"""
Expand Down
1 change: 0 additions & 1 deletion packages/backend/examples/streamlit.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@ def main():
x="projection_x",
y="projection_y",
neighbors="neighbors",
show_table=True,
)

# Show selected rows in a Streamlit data frame
Expand Down
4 changes: 3 additions & 1 deletion packages/docs/generate_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ def generate_dataset_embedding(
modality: str = "text",
model: str | None = "all-MiniLM-L6-v2",
pagerank: bool = False,
umap_args: dict = {},
umap_args: dict | None = None,
):
click.echo(click.style(f"Processing {url}", fg="cyan"))

Expand All @@ -48,6 +48,8 @@ def generate_dataset_embedding(
_ = data_frame
df = duckdb.query(query).to_df()

if umap_args is None:
umap_args = {}
umap_args = {"random_state": 42} | umap_args

df = compute_projection(
Expand Down
1 change: 0 additions & 1 deletion packages/docs/streamlit.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ df = compute_projection(df, inputs="description", modality="text",
value = embedding_atlas(
df, text="description",
x="projection_x", y="projection_y", neighbors="neighbors",
show_table=True
)
```

Expand Down
1 change: 0 additions & 1 deletion packages/umap/benchmark/benchmark/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
@click.group()
def cli():
"""UMAP / NNDescent benchmark suite."""
pass


@cli.command()
Expand Down
2 changes: 1 addition & 1 deletion packages/umap/benchmark/benchmark/align.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ def _kabsch(emb, ref):
with warnings.catch_warnings():
warnings.simplefilter("ignore")
H = E.T @ R
U, S, Vt = np.linalg.svd(H)
U, _, Vt = np.linalg.svd(H)
if np.any(~np.isfinite(U)) or np.any(~np.isfinite(Vt)):
return emb.astype(np.float32), np.inf

Expand Down
6 changes: 4 additions & 2 deletions packages/umap/benchmark/benchmark/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,9 @@ def run_cmd(cmd, desc="", timeout=None, threads="multi"):
print(f"{'=' * 60}")
t0 = time.perf_counter()
try:
result = subprocess.run(cmd, timeout=timeout, env=bench_env(threads))
result = subprocess.run(
cmd, timeout=timeout, env=bench_env(threads), check=False
)
elapsed = time.perf_counter() - t0
if result.returncode != 0:
print(f" FAILED (exit code {result.returncode}) in {elapsed:.1f}s")
Expand Down Expand Up @@ -100,7 +102,7 @@ def __init__(self, path, columns):
Path(path).parent.mkdir(parents=True, exist_ok=True)
self.path = path
self.columns = columns
self._file = open(path, "w", newline="")
self._file = open(path, "w", newline="") # noqa: SIM115
self._writer = csv.DictWriter(
self._file, fieldnames=columns, extrasaction="ignore"
)
Expand Down
4 changes: 2 additions & 2 deletions packages/umap/benchmark/benchmark/html_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ def _load_embeddings(ds_dir, rows):
# Align to reference (Rust CPU preferred)
ref_name = next((name for _, name in _IMPL_ORDER if name in embeddings), None)
ref_emb = embeddings[ref_name]
for name in embeddings:
for name in embeddings: # noqa: PLC0206
if name != ref_name:
embeddings[name] = align_to_reference(embeddings[name], ref_emb)

Expand All @@ -233,7 +233,7 @@ def _load_embeddings(ds_dir, rows):
maxs = all_coords.max(axis=0)
ranges = maxs - mins
ranges[ranges == 0] = 1
for name in embeddings:
for name in embeddings: # noqa: PLC0206
embeddings[name] = ((embeddings[name] - mins) / ranges).astype(np.float32)

# Serialize
Expand Down
14 changes: 14 additions & 0 deletions packages/viewer/src/assets/chart-features-list.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
2 changes: 2 additions & 0 deletions packages/viewer/src/assets/chart_icons.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import chart_cards from "./chart-cards.svg?raw";
import chart_content_viewer from "./chart-content-viewer.svg?raw";
import chart_ecdf from "./chart-ecdf.svg?raw";
import chart_embedding from "./chart-embedding.svg?raw";
import chart_features_list from "./chart-features-list.svg?raw";
import chart_h_bar from "./chart-h-bar.svg?raw";
import chart_heatmap from "./chart-heatmap.svg?raw";
import chart_line from "./chart-line.svg?raw";
Expand All @@ -24,6 +25,7 @@ export const chartIcons: Record<string, string> = {
"chart-content-viewer": chart_content_viewer,
"chart-ecdf": chart_ecdf,
"chart-embedding": chart_embedding,
"chart-features-list": chart_features_list,
"chart-h-bar": chart_h_bar,
"chart-heatmap": chart_heatmap,
"chart-line": chart_line,
Expand Down
2 changes: 2 additions & 0 deletions packages/viewer/src/assets/icons.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ export { default as IconPin } from "~icons/fluent/pin-16-filled";
export { default as IconPinRegular } from "~icons/fluent/pin-16-regular";
export { default as IconSelected } from "~icons/fluent/checkmark-circle-16-filled";
export { default as IconUnselected } from "~icons/fluent/circle-16-regular";
export { default as IconInfo } from "~icons/fluent/info-16-filled";
export { default as IconInfoRegular } from "~icons/fluent/info-16-regular";

// Icons for list layout view toggles (hide/show parts)
export { default as IconEmbeddingView } from "~icons/fluent/data-scatter-20-filled";
Expand Down
Loading
Loading