diff --git a/.changeset/witty-bobcats-care.md b/.changeset/witty-bobcats-care.md new file mode 100644 index 0000000000..7fc156fb02 --- /dev/null +++ b/.changeset/witty-bobcats-care.md @@ -0,0 +1,17 @@ +--- +'@evidence-dev/duckdb': major +'@evidence-dev/universal-sql': major +'@evidence-dev/db-commons': minor +'@evidence-dev/core-components': minor +--- + +Update DuckDB to latest packages: + +- Switch to @duckdb/node-api from duckdb-async +- Update duckdb-wasm to latest release + +This release also has small data fixes across several packages: + +- Better handling of NULL values when discovering column types +- Fix batch processing of parquet files +- Fix error with temporary parquet files when reloading data in dev environment diff --git a/packages/datasources/duckdb/index.cjs b/packages/datasources/duckdb/index.cjs index 0dae79c8e3..cb4ffb172e 100644 --- a/packages/datasources/duckdb/index.cjs +++ b/packages/datasources/duckdb/index.cjs @@ -3,9 +3,10 @@ const { TypeFidelity, asyncIterableToBatchedAsyncGenerator, cleanQuery, - exhaustStream + exhaustStream, + splitSQLStatement } = require('@evidence-dev/db-commons'); -const { Database } = require('duckdb-async'); +const { DuckDBInstance } = require('@duckdb/node-api'); const path = require('path'); const fs = require('fs/promises'); @@ -29,6 +30,11 @@ function standardizeRow(obj) { * @returns {EvidenceType | undefined} */ function nativeTypeToEvidenceType(data) { + // Handle null/undefined explicitly: let callers decide (they will mark as INFERRED) + if (data === null || data === undefined) return undefined; + // Some runtimes may yield bigint; normalize higher-level handling elsewhere but + // treat here as a number type. + if (typeof data === 'bigint') return EvidenceType.NUMBER; switch (typeof data) { case 'number': return EvidenceType.NUMBER; @@ -40,7 +46,9 @@ function nativeTypeToEvidenceType(data) { if (data instanceof Date) { return EvidenceType.DATE; } - throw new Error(`Unsupported object type: ${data}`); + // For other object types (arrays, plain objects, buffers, etc.) return + // undefined so the caller will mark the column as inferred (STRING). + return undefined; default: return EvidenceType.STRING; } @@ -51,6 +59,9 @@ function nativeTypeToEvidenceType(data) { * @returns {import('@evidence-dev/db-commons').ColumnDefinition[]} */ const mapResultsToEvidenceColumnTypes = function (rows) { + // If there are no rows, return an empty column list. Caller may provide DESCRIBE-based types. + if (!rows || rows.length === 0) return []; + return Object.entries(rows[0]).map(([name, value]) => { /** @type {TypeFidelity} */ let typeFidelity = TypeFidelity.PRECISE; @@ -125,11 +136,15 @@ const runQuery = async (queryString, database, batchSize = 100000) => { } else if (database.directory) { // Local database stored in source directory filename = path.join(database.directory, database.filename); + } else { + // filename provided without a directory; use as given (may be absolute or relative) + filename = database.filename; } } const mode = filename !== ':memory:' ? 'READ_ONLY' : 'READ_WRITE'; - const db = await Database.create(filename, { + // Create a DuckDB instance and connection using the new node-api + const db = await DuckDBInstance.create(filename, { access_mode: mode, custom_user_agent: 'evidence-dev' }); @@ -141,27 +156,107 @@ const runQuery = async (queryString, database, batchSize = 100000) => { const initScript = await fs.readFile(path.resolve(database.directory, 'initialize.sql'), { encoding: 'utf-8' }); - await conn.exec(initScript); + // run the initialization script + await conn.run(initScript).catch(() => null); } } - const stream = conn.stream(queryString); + // Split the incoming SQL into statements and treat all but the last as prefix + // statements which should be executed before metadata queries or the main + // streaming query. This allows SET/USE/CREATE statements to affect the + // session. + const statements = splitSQLStatement(queryString); + const prefixStatements = statements.slice(0, -1); + const mainStatement = statements.length > 0 ? statements[statements.length - 1] : ''; - const count_query = `WITH root as (${cleanQuery(queryString)}) SELECT COUNT(*) FROM root`; - const expected_count = await db.all(count_query).catch(() => null); - const expected_row_count = expected_count?.[0]['count_star()']; + const count_query = mainStatement + ? `WITH root as (${cleanQuery(mainStatement)}) SELECT COUNT(*) FROM root` + : null; + let expected_row_count = null; + try { + // Execute prefix statements first so they apply to the session + for (const ps of prefixStatements) { + try { + await conn.run(ps).catch(() => null); + } catch (err) { + // ignore errors from prefix statements to mimic existing behavior + } + } - const column_query = `DESCRIBE ${cleanQuery(queryString)}`; - const column_types = await db - .all(column_query) - .then(duckdbDescribeToEvidenceType) - .catch(() => null); + if (count_query) { + const countReader = await conn.runAndReadAll(count_query); + await countReader.readAll(); + const countRow = countReader.getRowObjectsJS()?.[0]; + if (countRow) { + // take the first value from the row (column name may vary) + expected_row_count = Object.values(countRow)[0]; + } + } + } catch (e) { + expected_row_count = null; + } - const results = await asyncIterableToBatchedAsyncGenerator(stream, batchSize, { + const column_query = mainStatement ? `DESCRIBE ${cleanQuery(mainStatement)}` : null; + let column_types = null; + try { + if (column_query) { + const colReader = await conn.runAndReadAll(column_query); + await colReader.readAll(); + const describeRows = colReader.getRowObjectsJS(); + column_types = duckdbDescribeToEvidenceType(describeRows); + } + } catch (e) { + column_types = null; + } + + // Create an async generator that yields batches using the DuckDBResultReader + // Stream only the main statement (last statement). If there is no main + // statement, create an empty generator. + let reader = null; + if (mainStatement) { + reader = await conn.streamAndRead(mainStatement); + } + + const rowsAsyncIterable = (async function* () { + try { + let prev = 0; + if (reader) { + // keep reading until done + while (!reader.done) { + const target = prev + batchSize; + await reader.readUntil(target); + const allRows = reader.getRowObjectsJS(); + const newRows = allRows.slice(prev).map(standardizeRow); + // yield rows one-by-one (asyncIterableToBatchedAsyncGenerator expects single-row yields) + for (const r of newRows) { + yield r; + } + prev = reader.currentRowCount; + } + } + } finally { + // disconnect the connection and close the instance synchronously + try { + conn.disconnectSync(); + } catch (err) {} + try { + db.closeSync(); + } catch (err) {} + } + })(); + + const results = await asyncIterableToBatchedAsyncGenerator(rowsAsyncIterable, batchSize, { mapResultsToEvidenceColumnTypes: column_types == null ? mapResultsToEvidenceColumnTypes : undefined, standardizeRow, - closeConnection: () => db.close() + closeConnection: () => { + try { + conn.disconnectSync(); + } catch (err) {} + try { + db.closeSync(); + } catch (err) {} + } }); if (column_types != null) { results.columnTypes = column_types; diff --git a/packages/datasources/duckdb/package.json b/packages/datasources/duckdb/package.json index 41f37839f3..b54dcf3dea 100644 --- a/packages/datasources/duckdb/package.json +++ b/packages/datasources/duckdb/package.json @@ -10,7 +10,7 @@ }, "dependencies": { "@evidence-dev/db-commons": "workspace:*", - "duckdb-async": "1.1.3" + "@duckdb/node-api": "^1.4.2-r.1" }, "devDependencies": { "dotenv": "^16.0.1" diff --git a/packages/datasources/duckdb/test/test.js b/packages/datasources/duckdb/test/test.js index 3b6f043044..7285728a6b 100644 --- a/packages/datasources/duckdb/test/test.js +++ b/packages/datasources/duckdb/test/test.js @@ -1,9 +1,26 @@ import { test } from 'uvu'; import * as assert from 'uvu/assert'; +import path from 'path'; +import { fileURLToPath } from 'url'; import runQuery from '../index.cjs'; import { batchedAsyncGeneratorToArray, TypeFidelity } from '@evidence-dev/db-commons'; import 'dotenv/config'; +test('basic select from needful_things.duckdb', async () => { + // Resolve the database file path relative to the repository root (from this test file) + const __filename = fileURLToPath(import.meta.url); + const __dirname = path.dirname(__filename); + const dbPath = path.join(__dirname, '..', '..', '..', '..', 'needful_things.duckdb'); + // Select the rows from sqlite_master; this test database contains 7 entries + const query = 'SELECT name FROM sqlite_master'; + const { rows: rowGen } = await runQuery(query, { filename: dbPath }); + const rows = await batchedAsyncGeneratorToArray(rowGen); + assert.instance(rows, Array); + assert.type(rows[0], 'object'); + // Expect exactly 7 rows in this test database + assert.equal(rows.length, 7); +}); + // Types to test // BOOLEAN // TINYINT @@ -181,6 +198,184 @@ test('query runs', async () => { } }); +test('handles nulls in first row', async () => { + try { + const { rows: row_generator, columnTypes } = await runQuery( + `select + NULL as boolean_col, + CAST(NULL AS TINYINT) as tinyint_col, + CAST(NULL AS SMALLINT) as smallint_col, + CAST(NULL AS INTEGER) as int_col, + CAST(NULL AS BIGINT) as bigint_col, + CAST(NULL AS HUGEINT) as hugeint_col, + CAST(NULL AS UTINYINT) as utinyint_col, + CAST(NULL AS USMALLINT) as usmallint_col, + CAST(NULL AS UINTEGER) as uint_col, + CAST(NULL AS UBIGINT) as ubigint_col, + CAST(NULL AS DATE) as date_col, + CAST(NULL AS TIME) as time_col, + CAST(NULL AS TIMESTAMP) as timestamp_col, + CAST(NULL AS TIMESTAMP_S) as timestamp_s_col, + CAST(NULL AS TIMESTAMP_MS) as timestamp_ms_col, + CAST(NULL AS TIMESTAMP_NS) as timestamp_ns_col, + CAST(NULL AS TIME WITH TIME ZONE) as time_with_tz_col, + CAST(NULL AS TIMESTAMP WITH TIME ZONE) as timestamp_with_tz_col, + CAST(NULL AS FLOAT) as float_col, + CAST(NULL AS DOUBLE) as double_col, + CAST(NULL AS DECIMAL(4,1)) as decimal_4_1_col, + CAST(NULL AS DECIMAL(9,4)) as decimal_9_4_col, + CAST(NULL AS DECIMAL(18,6)) as decimal_18_6_col, + CAST(NULL AS DECIMAL(38,10)) as decimal_38_10_col, + CAST(NULL AS UUID) as uuid_col, + CAST(NULL AS INTERVAL) as interval_col, + CAST(NULL AS VARCHAR) as varchar_col, + CAST(NULL AS BLOB) as blob_col, + CAST(NULL AS BIT) as bit_col + UNION ALL + select + true as boolean_col, + CAST(127 AS TINYINT) as tinyint_col, + CAST(32767 AS SMALLINT) as smallint_col, + CAST(2147483647 AS INTEGER) as int_col, + CAST(9223372036854775807 AS BIGINT) as bigint_col, + CAST('9223372036854775807000000' AS HUGEINT) as hugeint_col, + CAST(255 AS UTINYINT) as utinyint_col, + CAST(65535 AS USMALLINT) as usmallint_col, + CAST(4294967295 AS UINTEGER) as uint_col, + CAST(18446744073709551615 AS UBIGINT) as ubigint_col, + CURRENT_DATE as date_col, + CURRENT_TIME as time_col, + CURRENT_TIMESTAMP as timestamp_col, + CAST('2023-12-13' AS TIMESTAMP_S) as timestamp_s_col, + CAST('2023-12-13 12:34:56.789' AS TIMESTAMP_MS) as timestamp_ms_col, + CAST('2023-12-13 12:34:56.789123456' AS TIMESTAMP_NS) as timestamp_ns_col, + TIMETZ '1992-09-20 11:30:00.123456-02:00' as time_with_tz_col, + TIMESTAMPTZ '1992-09-20 11:30:00.123456' as timestamp_with_tz_col, + CAST(3.14 AS FLOAT) as float_col, + CAST(3.14 AS DOUBLE) as double_col, + CAST(3.14 AS DECIMAL(4,1)) as decimal_4_1_col, + CAST(3.14 AS DECIMAL(9,4)) as decimal_9_4_col, + CAST(3.14 AS DECIMAL(18,6)) as decimal_18_6_col, + CAST(3.14 AS DECIMAL(38,10)) as decimal_38_10_col, + CAST('550e8400-e29b-41d4-a716-446655440000' AS UUID) as uuid_col, + CAST('1 day 2 hours 30 minutes' AS INTERVAL) as interval_col, + 'Evidence' as varchar_col, + CAST('SGVsbG8gd29ybGQ=' AS BLOB) as blob_col, + CAST(1 AS BIT) as bit_col + ` + ); + + const rows = await batchedAsyncGeneratorToArray(row_generator); + assert.instance(rows, Array); + assert.instance(columnTypes, Array); + assert.type(rows[0], 'object'); + assert.equal(columnTypes.length, 29); + + const actualColumnTypes = columnTypes.map((columnType) => columnType.evidenceType); + const actualColumnNames = columnTypes.map((columnType) => columnType.name); + const actualTypePrecisions = columnTypes.map((columnType) => columnType.typeFidelity); + + const expectedColumnTypes = [ + 'boolean', + 'number', + 'number', + 'number', + 'number', + 'number', + 'number', + 'number', + 'number', + 'number', + 'date', + 'string', + 'date', + 'date', + 'date', + 'date', + 'string', + 'date', + 'number', + 'number', + 'number', + 'number', + 'number', + 'number', + 'string', + 'string', + 'string', + 'string', + 'string' + ]; + + const expectedColumnNames = [ + 'boolean_col', + 'tinyint_col', + 'smallint_col', + 'int_col', + 'bigint_col', + 'hugeint_col', + 'utinyint_col', + 'usmallint_col', + 'uint_col', + 'ubigint_col', + 'date_col', + 'time_col', + 'timestamp_col', + 'timestamp_s_col', + 'timestamp_ms_col', + 'timestamp_ns_col', + 'time_with_tz_col', + 'timestamp_with_tz_col', + 'float_col', + 'double_col', + 'decimal_4_1_col', + 'decimal_9_4_col', + 'decimal_18_6_col', + 'decimal_38_10_col', + 'uuid_col', + 'interval_col', + 'varchar_col', + 'blob_col', + 'bit_col' + ]; + + const expectedTypePrecision = Array(29).fill(TypeFidelity.PRECISE); + + assert.equal( + true, + expectedColumnTypes.length === actualColumnTypes.length && + expectedColumnTypes.every((value, index) => value === actualColumnTypes[index]) + ); + assert.equal( + true, + expectedColumnNames.length === actualColumnNames.length && + expectedColumnNames.every((value, index) => value === actualColumnNames[index]) + ); + assert.equal( + true, + expectedTypePrecision.length === actualTypePrecisions.length && + expectedTypePrecision.every((value, index) => value === actualTypePrecisions[index]) + ); + } catch (e) { + console.error('Error in null-first-row test:', e.message); + throw e; + } +}); + +test('expectedRowCount present for UNION with NULL-first-row', async () => { + // Simple reproduction: first row has NULL, second row has a value + const query = `select NULL as a UNION ALL select 1 as a`; + const { rows, expectedRowCount } = await runQuery(query, undefined, 2); + // consume rows to ensure the stream runs + const arr = []; + for await (const batch of rows()) { + arr.push(...batch); + } + assert.equal(arr.length, 2); + assert.type(expectedRowCount, 'number'); + assert.equal(expectedRowCount, 2); +}); + test('query batches results properly', async () => { try { const { rows, expectedRowCount } = await runQuery( @@ -203,4 +398,145 @@ test('query batches results properly', async () => { } }); +test('handles trailing comments', async () => { + // Single-line trailing comment (run in-memory) + const q1 = 'select 1 as one -- trailing comment'; + const { rows: rowGen1 } = await runQuery(q1, undefined); + const rows1 = await batchedAsyncGeneratorToArray(rowGen1); + assert.equal(rows1[0].one, 1); + + // Block trailing comment + const q2 = 'select 2 as two /* trailing block comment */'; + const { rows: rowGen2 } = await runQuery(q2, undefined); + const rows2 = await batchedAsyncGeneratorToArray(rowGen2); + assert.equal(rows2[0].two, 2); +}); + +test('handles leading SET statements before main query', async () => { + try { + const query = ` + SET VARIABLE VAR1 = DATE '2023-10-04'; + SET VARIABLE VAR2 = '23'; + select 1 union all select 2 union all select 3; + `; + const { rows, expectedRowCount } = await runQuery(query, undefined, 2); + const arr = []; + for await (const batch of rows()) { + arr.push(batch); + } + // should batch into [2,1] + assert.equal(arr[0].length, 2); + assert.equal(arr[1].length, 1); + assert.equal(expectedRowCount, 3); + } catch (e) { + throw Error(e); + } +}); + +test('semicolon inside single-quoted string should not split statements', async () => { + try { + const query = "SET V='this;is;a;string'; select 1 union all select 2;"; + const { rows, expectedRowCount } = await runQuery(query, undefined, 2); + const arr = []; + for await (const batch of rows()) { + arr.push(batch); + } + assert.equal(expectedRowCount, 2); + assert.equal(arr[0].length, 2); + } catch (e) { + throw Error(e); + } +}); + +test('semicolon inside block comment should not split statements', async () => { + try { + const query = '/* comment; still comment; */ select 1 union all select 2;'; + const { rows, expectedRowCount } = await runQuery(query, undefined, 2); + const arr = []; + for await (const batch of rows()) { + arr.push(batch); + } + assert.equal(expectedRowCount, 2); + assert.equal(arr[0].length, 2); + } catch (e) { + throw Error(e); + } +}); + +test('handles leading SET statements before main query', async () => { + try { + const query = ` + SET VARIABLE VAR1 = DATE '2023-10-04'; + SET VARIABLE VAR2 = '23'; + select 1 union all select 2 union all select 3; + `; + const { rows, expectedRowCount } = await runQuery(query, undefined, 2); + const arr = []; + for await (const batch of rows()) { + arr.push(batch); + } + // should batch into [2,1] + assert.equal(arr[0].length, 2); + assert.equal(arr[1].length, 1); + assert.equal(expectedRowCount, 3); + } catch (e) { + throw Error(e); + } +}); + +test('semicolon inside single-quoted string should not split statements', async () => { + try { + const query = "SET V='this;is;a;string'; select 1 union all select 2;"; + const { rows, expectedRowCount } = await runQuery(query, undefined, 2); + const arr = []; + for await (const batch of rows()) { + arr.push(batch); + } + assert.equal(expectedRowCount, 2); + assert.equal(arr[0].length, 2); + } catch (e) { + throw Error(e); + } +}); + +test('semicolon inside block comment should not split statements', async () => { + try { + const query = '/* comment; still comment; */ select 1 union all select 2;'; + const { rows, expectedRowCount } = await runQuery(query, undefined, 2); + const arr = []; + for await (const batch of rows()) { + arr.push(batch); + } + assert.equal(expectedRowCount, 2); + assert.equal(arr[0].length, 2); + } catch (e) { + throw Error(e); + } +}); + +test('USE statement before query should apply to metadata queries', async () => { + try { + // Create a temporary on-disk DuckDB file to test that USE affects + // subsequent queries in the same session. We create a schema and table + // in prefix statements, then USE the schema and select from the table. + // Use an in-memory database so we can CREATE schema/table in the test + const dbFile = ':memory:'; + const query = ` + CREATE SCHEMA s; + CREATE TABLE s.t AS SELECT 1 AS x UNION ALL SELECT 2 AS x; + USE s; + SELECT x FROM t; + `; + const { rows, expectedRowCount } = await runQuery(query, { filename: dbFile }, 2); + const arr = []; + for await (const batch of rows()) { + arr.push(batch); + } + assert.equal(expectedRowCount, 2); + assert.equal(arr[0].length, 2); + } catch (e) { + throw Error(e); + } +}); + test.run(); diff --git a/packages/lib/db-commons/index.cjs b/packages/lib/db-commons/index.cjs index 88464614a9..ede569e423 100644 --- a/packages/lib/db-commons/index.cjs +++ b/packages/lib/db-commons/index.cjs @@ -197,6 +197,149 @@ const cleanQuery = (query) => { return cleanedString + '\n'; }; +/** + * Split a SQL script into individual statements, respecting single/double/backtick + * quotes and block/line comments so semicolons inside those constructs don't split. + * Returns an array of statements (strings) without trailing semicolons and trimmed. + * @param {string} sql + * @returns {string[]} + */ +const splitSQLStatement = function (sql) { + const statements = []; + let cur = ''; + let inSingle = false; + let inDouble = false; + let inBacktick = false; + let inLineComment = false; + let inBlockComment = false; + for (let i = 0; i < sql.length; i++) { + const ch = sql[i]; + const next = sql[i + 1]; + + if (inLineComment) { + cur += ch; + if (ch === '\n') { + inLineComment = false; + } + continue; + } + if (inBlockComment) { + cur += ch; + if (ch === '*' && next === '/') { + cur += next; + i++; + inBlockComment = false; + } + continue; + } + + // start of line comment + if (!inSingle && !inDouble && !inBacktick && ch === '-' && next === '-') { + cur += ch; + inLineComment = true; + continue; + } + // start of block comment + if (!inSingle && !inDouble && !inBacktick && ch === '/' && next === '*') { + cur += ch; + inBlockComment = true; + continue; + } + + // quotes + if (!inDouble && !inBacktick && ch === "'") { + inSingle = !inSingle; + cur += ch; + continue; + } + if (!inSingle && !inBacktick && ch === '"') { + inDouble = !inDouble; + cur += ch; + continue; + } + if (!inSingle && !inDouble && ch === '`') { + inBacktick = !inBacktick; + cur += ch; + continue; + } + + // semicolon splits statements only when not inside quotes or comments + if (ch === ';' && !inSingle && !inDouble && !inBacktick && !inLineComment && !inBlockComment) { + const s = cur.trim(); + // When splitting, ignore segments that contain only comments/whitespace. + // If the segment starts with comment(s) but contains SQL after them, + // we should still keep it. To determine this, strip leading comments + // and whitespace and check whether anything remains. + if (s.length > 0) { + let remainder = s; + // Remove leading whitespace and comments (both line and block) iteratively + while (true) { + remainder = remainder.trimStart(); + if (remainder.startsWith('--')) { + const nl = remainder.indexOf('\n'); + if (nl === -1) { + remainder = ''; + break; + } else { + remainder = remainder.slice(nl + 1); + continue; + } + } + if (remainder.startsWith('/*')) { + const end = remainder.indexOf('*/', 2); + if (end === -1) { + remainder = ''; + break; + } else { + remainder = remainder.slice(end + 2); + continue; + } + } + break; + } + if (remainder.trim().length > 0) statements.push(s); + } + cur = ''; + continue; + } + + cur += ch; + } + + const last = cur.trim(); + // Ignore trailing content that is only comments. See logic above for stripping + // leading comments from a segment before deciding whether it contains SQL. + if (last.length > 0) { + let remainder = last; + while (true) { + remainder = remainder.trimStart(); + if (remainder.startsWith('--')) { + const nl = remainder.indexOf('\n'); + if (nl === -1) { + remainder = ''; + break; + } else { + remainder = remainder.slice(nl + 1); + continue; + } + } + if (remainder.startsWith('/*')) { + const end = remainder.indexOf('*/', 2); + if (end === -1) { + remainder = ''; + break; + } else { + remainder = remainder.slice(end + 2); + continue; + } + } + break; + } + if (remainder.trim().length > 0) statements.push(last); + } + return statements; +}; + /** * @param {QueryResult} stream * @returns {Promise} @@ -217,5 +360,6 @@ exports.asyncIterableToBatchedAsyncGenerator = asyncIterableToBatchedAsyncGenera exports.batchedAsyncGeneratorToArray = batchedAsyncGeneratorToArray; exports.cleanQuery = cleanQuery; exports.exhaustStream = exhaustStream; +exports.splitSQLStatement = splitSQLStatement; exports.getEnv = require('./src/getEnv.cjs').getEnv; diff --git a/packages/lib/universal-sql/package.json b/packages/lib/universal-sql/package.json index a31635a994..0abe02daa7 100644 --- a/packages/lib/universal-sql/package.json +++ b/packages/lib/universal-sql/package.json @@ -14,7 +14,7 @@ }, "type": "module", "dependencies": { - "@duckdb/duckdb-wasm": "1.29.0", + "@duckdb/duckdb-wasm": "^1.30.0", "apache-arrow": "17.0.0", "chalk": "^5.2.0", "cli-progress": "^3.12.0", diff --git a/packages/lib/universal-sql/src/build-parquet.js b/packages/lib/universal-sql/src/build-parquet.js index 42dd46709a..3577ffd0b1 100644 --- a/packages/lib/universal-sql/src/build-parquet.js +++ b/packages/lib/universal-sql/src/build-parquet.js @@ -95,6 +95,12 @@ export async function buildMultipartParquet( let tmpFilenames = []; let rowCount = 0; + // Unique id for this build run to avoid reusing the same temp filenames across + // consecutive builds in the same session. DuckDB (wasm) may keep handles or + // mappings to filenames; creating unique filenames per build avoids races or + // protocol errors when files are replaced. + const buildId = `${Date.now()}.${Math.random().toString(36).slice(2, 8)}`; + const flush = async (results) => { log.debug(`Flushing batch ${batchNum} with ${results.length} rows`); let { meta, done } = log.measure('flush'); @@ -124,11 +130,20 @@ export async function buildMultipartParquet( const parquetBuffer = writeParquet(Table.fromIPCStream(IPC), writerProperties); - const tempFilename = path.join(tmpDir, `${outputSubpath}.${batchNum}.parquet`); - await fs.mkdir(path.dirname(tempFilename), { recursive: true }); - await fs.writeFile(tempFilename, parquetBuffer); + const finalTempFilename = path.join(tmpDir, `${outputSubpath}.${buildId}.${batchNum}.parquet`); + // Write to a unique temporary file first, then atomically rename into place. + // This avoids a race where DuckDB may attempt to read a file while it's being written + // (e.g., during rapid dev refreshes). Using an atomic rename ensures readers only + // see the completed file. + const uniqueSuffix = `.${Date.now()}.${Math.random().toString(36).slice(2, 8)}.tmp`; + const writeTempFilename = finalTempFilename + uniqueSuffix; + + await fs.mkdir(path.dirname(finalTempFilename), { recursive: true }); + await fs.writeFile(writeTempFilename, parquetBuffer); + // Atomically move into final location + await fs.rename(writeTempFilename, finalTempFilename); - tmpFilenames.push(tempFilename); + tmpFilenames.push(finalTempFilename); rowCount += results.length; done(); diff --git a/packages/lib/universal-sql/src/build-parquet.spec.js b/packages/lib/universal-sql/src/build-parquet.spec.js index 620f987bc5..7c9ef885b9 100644 --- a/packages/lib/universal-sql/src/build-parquet.spec.js +++ b/packages/lib/universal-sql/src/build-parquet.spec.js @@ -73,10 +73,13 @@ describe('buildMultipartParquet', () => { // Make sure it contains data expect(stat.size).toBeGreaterThan(0); expect(fs.rm).toHaveBeenCalledOnce(); - expect(fs.rm).toHaveBeenCalledWith( - adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet/out.0.parquet'), - { force: true } + // verify the rm call targeted the intermediate directory and a tempfile + const calledPath = fs.rm.mock.calls[0][0]; + expect(path.dirname(calledPath)).toBe( + adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet') ); + // basename should look like out...0.parquet + expect(path.basename(calledPath)).toMatch(/^out\.\d+\.[0-9a-z]{6}\.0\.parquet$/); }); it('should write two temporary files when given enough rows for two batches', async () => { @@ -105,16 +108,16 @@ describe('buildMultipartParquet', () => { // Make sure it contains data expect(stat.size).toBeGreaterThan(0); expect(fs.rm).toHaveBeenCalledTimes(2); - expect(fs.rm).toHaveBeenNthCalledWith( - 1, - adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet/out.0.parquet'), - { force: true } + const calledPath1 = fs.rm.mock.calls[0][0]; + const calledPath2 = fs.rm.mock.calls[1][0]; + expect(path.dirname(calledPath1)).toBe( + adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet') ); - expect(fs.rm).toHaveBeenNthCalledWith( - 2, - adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet/out.1.parquet'), - { force: true } + expect(path.dirname(calledPath2)).toBe( + adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet') ); + expect(path.basename(calledPath1)).toMatch(/^out\.\d+\.[0-9a-z]{6}\.0\.parquet$/); + expect(path.basename(calledPath2)).toMatch(/^out\.\d+\.[0-9a-z]{6}\.1\.parquet$/); }); it('should accept an array as the data argument', async () => { @@ -133,12 +136,11 @@ describe('buildMultipartParquet', () => { // Make sure it contains data expect(stat.size).toBeGreaterThan(0); expect(fs.rm).toHaveBeenCalledOnce(); - expect(fs.rm).toHaveBeenCalledWith( - adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet/out.0.parquet'), - { - force: true - } + const calledPathA = fs.rm.mock.calls[0][0]; + expect(path.dirname(calledPathA)).toBe( + adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet') ); + expect(path.basename(calledPathA)).toMatch(/^out\.\d+\.[0-9a-z]{6}\.0\.parquet$/); }); it('should handle a very large number of batches', async () => { diff --git a/packages/ui/core-components/CHANGELOG.md b/packages/ui/core-components/CHANGELOG.md index 0718e0b6e2..4ad9a2d7b7 100644 --- a/packages/ui/core-components/CHANGELOG.md +++ b/packages/ui/core-components/CHANGELOG.md @@ -7,7 +7,6 @@ - 6d2782e64: Fixed base path not being applied to data file (parquet) URLs. This resolves an issue where applications deployed with a base path would fail to load data files, resulting in 404 errors. The fix restores the dependency injection pattern for the `addBasePath` function in `setParquetURLs`, ensuring that base paths are correctly applied to all data requests in both monorepo and published package environments. - - @evidence-dev/component-utilities@4.0.10 - @evidence-dev/tailwind@3.1.1 diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 18ac486acc..4a556b6204 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -639,12 +639,12 @@ importers: packages/datasources/duckdb: dependencies: + '@duckdb/node-api': + specifier: ^1.4.2-r.1 + version: 1.4.2-r.1 '@evidence-dev/db-commons': specifier: workspace:* version: link:../../lib/db-commons - duckdb-async: - specifier: 1.1.3 - version: 1.1.3 devDependencies: dotenv: specifier: ^16.0.1 @@ -1270,8 +1270,8 @@ importers: packages/lib/universal-sql: dependencies: '@duckdb/duckdb-wasm': - specifier: 1.29.0 - version: 1.29.0 + specifier: ^1.30.0 + version: 1.30.0 apache-arrow: specifier: 17.0.0 version: 17.0.0 @@ -3838,6 +3838,69 @@ packages: resolution: {integrity: sha512-8Zq7vafQuIz9gklC/9375KE38UlkaS2n8+yvG+/JK7irm3DjwYNJHL4xfplIj0bSHFIg6we5XhWYFqtE/vO3+Q==} dependencies: apache-arrow: 17.0.0 + dev: true + + /@duckdb/duckdb-wasm@1.30.0: + resolution: {integrity: sha512-9aWrm+4ayl4sTlvGtl/b+LxrUyXaac3yyVqkoJ3F7Vkd62PoS8PcQIRJ/KjXBW36LP1CnPY5jjvFyIcTFLtcXA==} + dependencies: + apache-arrow: 17.0.0 + dev: false + + /@duckdb/node-api@1.4.2-r.1: + resolution: {integrity: sha512-8Ef4R/Lq+rXTpcqZIJZe6ALfgMGxy88HmiPvRpWrRw1fUTy85x1U0NnGrqCklZsmWyZUdPwVYj90MQOF2MY/TA==} + dependencies: + '@duckdb/node-bindings': 1.4.2-r.1 + dev: false + + /@duckdb/node-bindings-darwin-arm64@1.4.2-r.1: + resolution: {integrity: sha512-C4yBI3zBX7iZ9iq8zJDvPatXA+2xM22sg1kX3fx76nG+qTqKtU/dGFVYL7Fx6cBYxBO1ZZ8Y+vjgYb6/bich8A==} + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: false + optional: true + + /@duckdb/node-bindings-darwin-x64@1.4.2-r.1: + resolution: {integrity: sha512-dgTSBuEfWyhymYovsGoRESjqcJuZWwJqla9l89SSsBDrGYcUp+EHsXUF73oUCspzTesT2lOvHrDufO8pKgtudw==} + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: false + optional: true + + /@duckdb/node-bindings-linux-arm64@1.4.2-r.1: + resolution: {integrity: sha512-r2Ml1LvU2vkVMx4YU04T79FjGkYg8YMVtbOz7j740SZGIi5Cch5P1/oy48jJBWRqoaXuqimpYWeTZiGVeCQiZA==} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@duckdb/node-bindings-linux-x64@1.4.2-r.1: + resolution: {integrity: sha512-ed5KiNIu1rqSva5rgo4PRVYSmBolAMVqGFWsYWLoRZ94ka2F/dHwJNkarCTmBFCEDGVZWzWzjRyWTQgYTvQxTg==} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@duckdb/node-bindings-win32-x64@1.4.2-r.1: + resolution: {integrity: sha512-kIIT8tuoW3mzr9EPcdSoKfv9CgOhTqRryHDI10Z0nuhc9UhqOWPUM/LnSUebfo1mdD9Drm7YrKCKYxNTr5dqBQ==} + cpu: [x64] + os: [win32] + requiresBuild: true + dev: false + optional: true + + /@duckdb/node-bindings@1.4.2-r.1: + resolution: {integrity: sha512-z0pJCdEnIj0Famkip6w7JZ/A17nm5VcYc6H7isOHXfEnPhBQ9PBusUTFgIzl6+3J8HOFQOv0szJq46zldbsHfQ==} + optionalDependencies: + '@duckdb/node-bindings-darwin-arm64': 1.4.2-r.1 + '@duckdb/node-bindings-darwin-x64': 1.4.2-r.1 + '@duckdb/node-bindings-linux-arm64': 1.4.2-r.1 + '@duckdb/node-bindings-linux-x64': 1.4.2-r.1 + '@duckdb/node-bindings-win32-x64': 1.4.2-r.1 + dev: false /@esbuild/aix-ppc64@0.25.12: resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} @@ -4336,7 +4399,9 @@ packages: /@gar/promisify@1.1.3: resolution: {integrity: sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==} + requiresBuild: true dev: false + optional: true /@google-cloud/bigquery@6.2.0: resolution: {integrity: sha512-jkyu8n5WR5HoF4saVc/ukfAE8uEjqkWRu3Bn3+hhJ7RTNR8unloS9J6JKzsGtLlA8gS3gHSceRg/CpCl5WFg+g==} @@ -5300,14 +5365,6 @@ packages: dev: false optional: true - /@npmcli/fs@2.1.2: - resolution: {integrity: sha512-yOJKRvohFOaLqipNtwYB9WugyZKhC/DZC4VYPmpaCzDBrA8YpK3qHZ8/HGscMnE4GqbkLNuVcCnxkeQEdGt6LQ==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - dependencies: - '@gar/promisify': 1.1.3 - semver: 7.7.3 - dev: false - /@npmcli/move-file@1.1.2: resolution: {integrity: sha512-1SUf/Cg2GzGDyaf15aR9St9TWlb+XvbZXWpDx8YKs7MLzMH/BCeopv+y9vzrzgkfykCGuWOlSu3mZhj2+FQcrg==} engines: {node: '>=10'} @@ -5319,15 +5376,6 @@ packages: dev: false optional: true - /@npmcli/move-file@2.0.1: - resolution: {integrity: sha512-mJd2Z5TjYWq/ttPLLGqArdtnC74J6bOzg4rMDnN+p1xTacZ2yPRCk2y0oSWQtygLR9YVQXgOcONrwtnk3JupxQ==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - deprecated: This functionality has been moved to @npmcli/fs - dependencies: - mkdirp: 1.0.4 - rimraf: 3.0.2 - dev: false - /@parcel/bundler-default@2.16.0(@parcel/core@2.16.0): resolution: {integrity: sha512-8kY+TUhir7qm+TgSMeMd8CP2PVoZjXamiZ8+mbXws4jKw6IrIVDQf8TkBZKGk7ncKJEteiX4ybbmiPjho8cHuA==} engines: {node: '>= 16.0.0', parcel: ^2.16.0} @@ -9336,6 +9384,7 @@ packages: /abbrev@1.1.1: resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==} + requiresBuild: true dev: false /abort-controller@3.0.0: @@ -9387,17 +9436,21 @@ packages: /agentkeepalive@4.6.0: resolution: {integrity: sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==} engines: {node: '>= 8.0.0'} + requiresBuild: true dependencies: humanize-ms: 1.2.1 dev: false + optional: true /aggregate-error@3.1.0: resolution: {integrity: sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==} engines: {node: '>=8'} + requiresBuild: true dependencies: clean-stack: 2.2.0 indent-string: 4.0.0 dev: false + optional: true /ajv-formats@2.1.1(ajv@8.17.1): resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==} @@ -9557,10 +9610,12 @@ packages: resolution: {integrity: sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg==} engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} deprecated: This package is no longer supported. + requiresBuild: true dependencies: delegates: 1.0.0 readable-stream: 3.6.2 dev: false + optional: true /arg@5.0.2: resolution: {integrity: sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==} @@ -10141,32 +10196,6 @@ packages: dev: false optional: true - /cacache@16.1.3: - resolution: {integrity: sha512-/+Emcj9DAXxX4cwlLmRI9c166RuL3w30zp4R7Joiv2cQTtTtA+jeuCAjH3ZlGnYS3tKENSrKhAzVVP9GVyzeYQ==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - dependencies: - '@npmcli/fs': 2.1.2 - '@npmcli/move-file': 2.0.1 - chownr: 2.0.0 - fs-minipass: 2.1.0 - glob: 8.1.0 - infer-owner: 1.0.4 - lru-cache: 7.18.3 - minipass: 3.3.6 - minipass-collect: 1.0.2 - minipass-flush: 1.0.5 - minipass-pipeline: 1.2.4 - mkdirp: 1.0.4 - p-map: 4.0.0 - promise-inflight: 1.0.1 - rimraf: 3.0.2 - ssri: 9.0.1 - tar: 6.2.1 - unique-filename: 2.0.1 - transitivePeerDependencies: - - bluebird - dev: false - /call-bind-apply-helpers@1.0.2: resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} engines: {node: '>= 0.4'} @@ -10418,6 +10447,7 @@ packages: engines: {node: '>=6'} requiresBuild: true dev: false + optional: true /cli-cursor@4.0.0: resolution: {integrity: sha512-VGtlMu3x/4DOtIUwEkRezxUZ2lBacNJCHash0N0WeZDBS+7Ux1dm3XWAgWYxLJFMMdOeXMHXorshEFhbMSGelg==} @@ -11295,29 +11325,6 @@ packages: engines: {node: '>=4'} dev: false - /duckdb-async@1.1.3: - resolution: {integrity: sha512-8Q8BYTjfzjPFJ3J8bafLGUcdxI2ZXPxpw3u8x+SFr7s3T7JM0Nn26LKis4rA0i4TCBSZ9ilhB8NOJ/9HrPGSlw==} - dependencies: - duckdb: 1.1.3 - transitivePeerDependencies: - - bluebird - - encoding - - supports-color - dev: false - - /duckdb@1.1.3: - resolution: {integrity: sha512-tIpZr2NsSkYmfGC1ETl75RuVsaDyjvR3yAOrECcIyw7bdluzcyzEXOXoiuT+4t54hT+CppZv43gk/HiZdKW9Vw==} - requiresBuild: true - dependencies: - '@mapbox/node-pre-gyp': 1.0.11 - node-addon-api: 7.1.1 - node-gyp: 9.4.1 - transitivePeerDependencies: - - bluebird - - encoding - - supports-color - dev: false - /dunder-proto@1.0.1: resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} engines: {node: '>= 0.4'} @@ -11471,7 +11478,9 @@ packages: /env-paths@2.2.1: resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==} engines: {node: '>=6'} + requiresBuild: true dev: false + optional: true /envinfo@7.20.0: resolution: {integrity: sha512-+zUomDcLXsVkQ37vUqWBvQwLaLlj8eZPSi61llaEFAVBY5mhcXdaSw1pSJVl4yTYD5g/gEfpNl28YYk4IPvrrg==} @@ -11483,6 +11492,7 @@ packages: resolution: {integrity: sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==} requiresBuild: true dev: false + optional: true /error-ex@1.3.4: resolution: {integrity: sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==} @@ -12128,10 +12138,6 @@ packages: jest-util: 29.7.0 dev: true - /exponential-backoff@3.1.3: - resolution: {integrity: sha512-ZgEeZXj30q+I0EN+CbSSpIyPaJ5HVQD18Z1m+u1FXbAeT94mr1zw50q4q6jiiC447Nl/YTcIYSAftiGqetwXCA==} - dev: false - /export-to-csv@0.2.1: resolution: {integrity: sha512-KTbrd3CAZ0cFceJEZr1e5uiMasabeCpXq1/5uvVxDl53o4jXJHnltasQoj2NkzrxD8hU9kdwjnMhoir/7nNx/A==} @@ -12530,6 +12536,7 @@ packages: resolution: {integrity: sha512-f9m+BEN5jkg6a0fZjleidjN51VE1X+mPFQ2DJ0uv1V39oCLCbsGe6yjbBnp7eK7z/+GAon99a3nHuqbuuthyPg==} engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} deprecated: This package is no longer supported. + requiresBuild: true dependencies: aproba: 2.1.0 color-support: 1.1.3 @@ -12540,6 +12547,7 @@ packages: strip-ansi: 6.0.1 wide-align: 1.1.5 dev: false + optional: true /gaxios@5.1.3: resolution: {integrity: sha512-95hVgBRgEIRQQQHIbnxBXeHbW4TqFk4ZDJW7wmVtvYar72FdhRIo1UGOLS2eRAKCPEdPBWu+M7+A33D9CdX9rA==} @@ -12783,6 +12791,7 @@ packages: inherits: 2.0.4 minimatch: 5.1.6 once: 1.4.0 + dev: true /globals@13.24.0: resolution: {integrity: sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==} @@ -13068,7 +13077,9 @@ packages: /http-cache-semantics@4.2.0: resolution: {integrity: sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==} + requiresBuild: true dev: false + optional: true /http-errors@2.0.0: resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==} @@ -13172,6 +13183,7 @@ packages: dependencies: ms: 2.1.3 dev: false + optional: true /hyperdyperid@1.2.0: resolution: {integrity: sha512-Y93lCzHYgGWdrJ66yIktxiaGULYc6oGiABxhcO5AufBeOyoIdZF7bIfLaOrbM0iGIOXQQgxxRrFEnb+Y6w1n4A==} @@ -13244,7 +13256,9 @@ packages: /infer-owner@1.0.4: resolution: {integrity: sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==} + requiresBuild: true dev: false + optional: true /inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} @@ -13457,7 +13471,9 @@ packages: /is-lambda@1.0.1: resolution: {integrity: sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==} + requiresBuild: true dev: false + optional: true /is-map@2.0.3: resolution: {integrity: sha512-1Qed0/Hr2m+YqxnM09CjA2d/i6YZNfF6R2oRAOj36eUdS6qIV/huPJNSEpKbupewFs+ZsJlxsjjPbc0/afW6Lw==} @@ -14890,31 +14906,6 @@ packages: dependencies: semver: 7.7.3 - /make-fetch-happen@10.2.1: - resolution: {integrity: sha512-NgOPbRiaQM10DYXvN3/hhGVI2M5MtITFryzBGxHM5p4wnFxsVCbxkrBrDsk+EZ5OB4jEOT7AjDxtdF+KVEFT7w==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - dependencies: - agentkeepalive: 4.6.0 - cacache: 16.1.3 - http-cache-semantics: 4.2.0 - http-proxy-agent: 5.0.0 - https-proxy-agent: 5.0.1 - is-lambda: 1.0.1 - lru-cache: 7.18.3 - minipass: 3.3.6 - minipass-collect: 1.0.2 - minipass-fetch: 2.1.2 - minipass-flush: 1.0.5 - minipass-pipeline: 1.2.4 - negotiator: 0.6.4 - promise-retry: 2.0.1 - socks-proxy-agent: 7.0.0 - ssri: 9.0.1 - transitivePeerDependencies: - - bluebird - - supports-color - dev: false - /make-fetch-happen@9.1.0: resolution: {integrity: sha512-+zopwDy7DNknmwPQplem5lAZX/eCOzSvSNNcSKm5eVwTkOBzoktEfXsa9L23J/GIRhxRsaxzkPEhrJEpE2F4Gg==} engines: {node: '>= 10'} @@ -15140,6 +15131,7 @@ packages: engines: {node: '>=10'} dependencies: brace-expansion: 2.0.2 + dev: true /minimatch@9.0.3: resolution: {integrity: sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==} @@ -15169,9 +15161,11 @@ packages: /minipass-collect@1.0.2: resolution: {integrity: sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==} engines: {node: '>= 8'} + requiresBuild: true dependencies: minipass: 3.3.6 dev: false + optional: true /minipass-fetch@1.4.1: resolution: {integrity: sha512-CGH1eblLq26Y15+Azk7ey4xh0J/XfJfrCox5LDJiKqI2Q2iwOLOKrlmIaODiSQS8d18jalF6y2K2ePUm0CmShw==} @@ -15186,37 +15180,32 @@ packages: dev: false optional: true - /minipass-fetch@2.1.2: - resolution: {integrity: sha512-LT49Zi2/WMROHYoqGgdlQIZh8mLPZmOrN2NdJjMXxYe4nkN6FUyuPuOAOedNJDrx0IRGg9+4guZewtp8hE6TxA==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - dependencies: - minipass: 3.3.6 - minipass-sized: 1.0.3 - minizlib: 2.1.2 - optionalDependencies: - encoding: 0.1.13 - dev: false - /minipass-flush@1.0.5: resolution: {integrity: sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==} engines: {node: '>= 8'} + requiresBuild: true dependencies: minipass: 3.3.6 dev: false + optional: true /minipass-pipeline@1.2.4: resolution: {integrity: sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==} engines: {node: '>=8'} + requiresBuild: true dependencies: minipass: 3.3.6 dev: false + optional: true /minipass-sized@1.0.3: resolution: {integrity: sha512-MbkQQ2CTiBMlA2Dm/5cY+9SWFEN8pzzOXi6rlM5Xxq0Yqbda5ZQy9sU75a673FE9ZK0Zsbr6Y5iP6u9nktfg2g==} engines: {node: '>=8'} + requiresBuild: true dependencies: minipass: 3.3.6 dev: false + optional: true /minipass@3.3.6: resolution: {integrity: sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==} @@ -15445,7 +15434,9 @@ packages: /negotiator@0.6.4: resolution: {integrity: sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==} engines: {node: '>= 0.6'} + requiresBuild: true dev: false + optional: true /neo-async@2.6.2: resolution: {integrity: sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==} @@ -15485,6 +15476,7 @@ packages: /node-addon-api@7.1.1: resolution: {integrity: sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==} + dev: true /node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} @@ -15554,27 +15546,6 @@ packages: dev: false optional: true - /node-gyp@9.4.1: - resolution: {integrity: sha512-OQkWKbjQKbGkMf/xqI1jjy3oCTgMKJac58G2+bjZb3fza6gW2YrCSdMQYaoTb70crvE//Gngr4f0AgVHmqHvBQ==} - engines: {node: ^12.13 || ^14.13 || >=16} - hasBin: true - dependencies: - env-paths: 2.2.1 - exponential-backoff: 3.1.3 - glob: 7.2.3 - graceful-fs: 4.2.11 - make-fetch-happen: 10.2.1 - nopt: 6.0.0 - npmlog: 6.0.2 - rimraf: 3.0.2 - semver: 7.7.3 - tar: 6.2.1 - which: 2.0.2 - transitivePeerDependencies: - - bluebird - - supports-color - dev: false - /node-int64@0.4.0: resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} @@ -15589,14 +15560,6 @@ packages: abbrev: 1.1.1 dev: false - /nopt@6.0.0: - resolution: {integrity: sha512-ZwLpbTgdhuZUnZzjd7nb1ZV+4DoiC6/sfiVKok72ym/4Tlf+DFdlHYmT2JPmcNNWV6Pi3SDf1kT+A4r9RTuT9g==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - hasBin: true - dependencies: - abbrev: 1.1.1 - dev: false - /normalize-package-data@2.5.0: resolution: {integrity: sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==} dependencies: @@ -15683,12 +15646,14 @@ packages: resolution: {integrity: sha512-/vBvz5Jfr9dT/aFWd0FIRf+T/Q2WBsLENygUaFUqstqsycmZAP/t5BvFJTK0viFmSUxiUKTUplWy5vt+rvKIxg==} engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} deprecated: This package is no longer supported. + requiresBuild: true dependencies: are-we-there-yet: 3.0.1 console-control-strings: 1.1.0 gauge: 4.0.4 set-blocking: 2.0.0 dev: false + optional: true /nth-check@2.1.1: resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} @@ -15952,9 +15917,11 @@ packages: /p-map@4.0.0: resolution: {integrity: sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==} engines: {node: '>=10'} + requiresBuild: true dependencies: aggregate-error: 3.1.0 dev: false + optional: true /p-timeout@3.2.0: resolution: {integrity: sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==} @@ -16016,9 +15983,6 @@ packages: resolution: {integrity: sha512-4sgnoYixTR6Qq6265tjmufXQj7wxvZo4VJHrYfbnfWQWfW5CgF80IiM+dy050pYgtBAMvh+8vJDDYiSto1YPUA==} engines: {node: '>= 16.0.0'} hasBin: true - peerDependenciesMeta: - '@parcel/core': - optional: true dependencies: '@parcel/config-default': 2.16.0(@parcel/core@2.16.0) '@parcel/core': 2.16.0 @@ -16589,20 +16553,24 @@ packages: /promise-inflight@1.0.1: resolution: {integrity: sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==} + requiresBuild: true peerDependencies: bluebird: '*' peerDependenciesMeta: bluebird: optional: true dev: false + optional: true /promise-retry@2.0.1: resolution: {integrity: sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==} engines: {node: '>=10'} + requiresBuild: true dependencies: err-code: 2.0.3 retry: 0.12.0 dev: false + optional: true /prompts@2.4.2: resolution: {integrity: sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==} @@ -17060,6 +17028,7 @@ packages: engines: {node: '>= 4'} requiresBuild: true dev: false + optional: true /retry@0.13.1: resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} @@ -17546,17 +17515,6 @@ packages: dev: false optional: true - /socks-proxy-agent@7.0.0: - resolution: {integrity: sha512-Fgl0YPZ902wEsAyiQ+idGd1A7rSFx/ayC1CQVMw5P+EQx2V0SgpGtf6OKFhVjPflPUl9YMmEOnmfjCdMUsygww==} - engines: {node: '>= 10'} - dependencies: - agent-base: 6.0.2 - debug: 4.4.3 - socks: 2.8.7 - transitivePeerDependencies: - - supports-color - dev: false - /socks-proxy-agent@8.0.5: resolution: {integrity: sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==} engines: {node: '>= 14'} @@ -17672,9 +17630,6 @@ packages: /sqlite3@5.1.6: resolution: {integrity: sha512-olYkWoKFVNSSSQNvxVUfjiVbz3YtBwTJj+mfV5zpHmqW3sELx2Cf4QCdirMelhM5Zh+KDVaKgQHqCxrqiWHybw==} requiresBuild: true - peerDependenciesMeta: - node-gyp: - optional: true dependencies: '@mapbox/node-pre-gyp': 1.0.11 node-addon-api: 4.3.0 @@ -17711,13 +17666,6 @@ packages: dev: false optional: true - /ssri@9.0.1: - resolution: {integrity: sha512-o57Wcn66jMQvfHG1FlYbWeZWW/dHZhJXjpIcTfXldXEk5nz5lStPo3mK0OJQfGR3RbZUlbISexbljkJzuEj/8Q==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - dependencies: - minipass: 3.3.6 - dev: false - /stack-trace@0.0.10: resolution: {integrity: sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==} dev: false @@ -19000,13 +18948,6 @@ packages: dev: false optional: true - /unique-filename@2.0.1: - resolution: {integrity: sha512-ODWHtkkdx3IAR+veKxFV+VBkUMcN+FaqzUUd7IZzt+0zhDZFPFxhlqwPF3YQvMHx1TD0tdgYl+kuPnJ8E6ql7A==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - dependencies: - unique-slug: 3.0.0 - dev: false - /unique-slug@2.0.2: resolution: {integrity: sha512-zoWr9ObaxALD3DOPfjPSqxt4fnZiWblxHIgeWqW8x7UqDzEtHEQLzji2cuJYQFCU6KmoJikOYAZlrTHHebjx2w==} requiresBuild: true @@ -19015,13 +18956,6 @@ packages: dev: false optional: true - /unique-slug@3.0.0: - resolution: {integrity: sha512-8EyMynh679x/0gqE9fT9oilG+qEt+ibFyqjuVTsZn1+CMxH+XLlpvr2UZx4nVcCwTpx81nICr2JQFkM+HPLq4w==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - dependencies: - imurmurhash: 0.1.4 - dev: false - /unist-util-is@4.1.0: resolution: {integrity: sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==}