From c46d46b3b91397c7923d3925a64e379f8b33259d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre-Beno=C3=AEt=20JOUBERT?= Date: Thu, 3 Sep 2026 19:05:07 +0200 Subject: [PATCH 1/2] fix: #IMPULS-6155, enforce objectstorage compliance --- .../test/js/it/scenarios/storage/_utils.ts | 162 ++++++++++ .../test/js/it/scenarios/storage/avatar.ts | 160 ++++++++++ .../src/test/js/it/scenarios/storage/copy.ts | 136 +++++++++ .../js/it/scenarios/storage/csv-mapping.ts | 100 +++++++ .../test/js/it/scenarios/storage/delete.ts | 117 ++++++++ .../src/test/js/it/scenarios/storage/read.ts | 104 +++++++ .../scenarios/storage/special-characters.ts | 283 ++++++++++++++++++ 7 files changed, 1062 insertions(+) create mode 100644 tests/src/test/js/it/scenarios/storage/_utils.ts create mode 100644 tests/src/test/js/it/scenarios/storage/avatar.ts create mode 100644 tests/src/test/js/it/scenarios/storage/copy.ts create mode 100644 tests/src/test/js/it/scenarios/storage/csv-mapping.ts create mode 100644 tests/src/test/js/it/scenarios/storage/delete.ts create mode 100644 tests/src/test/js/it/scenarios/storage/read.ts create mode 100644 tests/src/test/js/it/scenarios/storage/special-characters.ts diff --git a/tests/src/test/js/it/scenarios/storage/_utils.ts b/tests/src/test/js/it/scenarios/storage/_utils.ts new file mode 100644 index 0000000000..253a0de88f --- /dev/null +++ b/tests/src/test/js/it/scenarios/storage/_utils.ts @@ -0,0 +1,162 @@ +import { describe } from "https://jslib.k6.io/k6chaijs/4.3.4.0/index.js"; +import http from "k6/http"; +import { FormData } from "https://jslib.k6.io/formdata/0.0.2/index.js"; +import { fail } from "k6"; + +import { + BASE_URL, + getHeaders, + authenticateWeb, + Session, + Structure, + UserInfo, + createAndSetRole, + linkRoleToUsers, + createUserAndGetData, + createEmptyStructure, + activateUsers, +} from '../../../node_modules/edifice-k6-commons/dist/index.js'; + +/** + * Shared fixture and thin route wrappers for the storage scenarios. + * + * These scenarios exercise org.entcore.common.storage.impl.S3Storage through the only surface k6 can reach: + * the workspace and archive routes that call it. edifice-k6-commons covers upload, download and the archive + * flows; the copy, delete and read-back routes have no helper there, so they are wrapped here rather than + * inlined in every scenario. + */ + +export type StorageInitData = { + head: Structure; + user: UserInfo; +} + +/** A structure with one activated teacher holding the workspace and archive workflows. */ +export function initStorageFixture(schoolName: string): StorageInitData { + let structure: Structure | null = null; + let user: UserInfo | null = null; + describe("[Storage-Init] Initialize data", () => { + authenticateWeb(__ENV.ADMC_LOGIN, __ENV.ADMC_PASSWORD); + structure = createEmptyStructure(`${schoolName}`, true); + user = createUserAndGetData({ + firstName: "Storage " + Date.now(), + lastName: "User", + "type": "Teacher", + structureId: structure.id, + birthDate: "2020-01-01", + positionIds: [] + }); + activateUsers(structure); + const roles = [ + createAndSetRole('Espace documentaire'), + createAndSetRole('Archive'), + ]; + const groups = [`Teachers from group ${structure.name}.`]; + for (const role of roles) { + linkRoleToUsers(structure, role, groups); + } + }); + return { head: structure, user }; +} + +/** POST /workspace/folder — multipart form, 201 with the created folder. */ +export function createFolderOrFail(name: string, parentFolderId?: string): string { + const body: Record = { name }; + if (parentFolderId) { + body.parentFolderId = parentFolderId; + } + const res = http.post(`${BASE_URL}/workspace/folder`, body, { headers: getHeaders() }); + if (res.status !== 201) { + fail(`could not create folder ${name}: ${res.status} - ${res.body}`); + } + return (res.json() as any)._id; +} + +/** POST /workspace/document/copy/:id/:folder — reaches Storage.copyFile through StorageHelper. */ +export function copyDocument(id: string, folderId: string) { + return http.post(`${BASE_URL}/workspace/document/copy/${id}/${folderId}`, null, { headers: getHeaders() }); +} + +/** POST /workspace/documents/copy/:folder — same, in bulk. */ +export function copyDocuments(ids: string[], folderId: string) { + return http.post(`${BASE_URL}/workspace/documents/copy/${folderId}`, JSON.stringify({ ids }), + { headers: getHeaders("application/json") }); +} + +/** DELETE /workspace/document/:id — reaches Storage.removeFile. */ +export function deleteDocument(id: string) { + return http.del(`${BASE_URL}/workspace/document/${id}`, null, { headers: getHeaders() }); +} + +/** DELETE /workspace/documents — reaches Storage.removeFiles with the whole batch. */ +export function deleteDocuments(ids: string[]) { + return http.del(`${BASE_URL}/workspace/documents`, JSON.stringify({ ids }), + { headers: getHeaders("application/json") }); +} + +/** GET /workspace/document/base64/:id — reaches Storage.readFile, which buffers the whole object. */ +export function getDocumentBase64(id: string) { + return http.get(`${BASE_URL}/workspace/document/base64/${id}`, { headers: getHeaders() }); +} + +/** + * GET /workspace/document/:id as raw bytes. downloadFile from the commons returns a text body, which is + * lossy on binary content — a byte for byte comparison needs the binary response type. + */ +export function downloadDocumentBinary(id: string) { + return http.get(`${BASE_URL}/workspace/document/${id}`, + { headers: getHeaders(), responseType: "binary" }); +} + +/** The document ids of a copy response, which returns the created documents as an array. */ +export function copiedIds(res: any): string[] { + const body = res.json(); + if (!Array.isArray(body)) { + return []; + } + return body.filter((d: any) => !!d && !!d._id).map((d: any) => d._id); +} + +/** PUT /directory/avatar/:userId — every call goes through cleanAvatarCache, hence findByFilenameEndingWith. */ +export function updateAvatar(userId: string, documentId: string) { + return http.put(`${BASE_URL}/directory/avatar/${userId}`, + JSON.stringify({ picture: `/workspace/document/${documentId}` }), + { headers: getHeaders("application/json") }); +} + +/** GET /userbook/avatar/:id — serves the cached avatar, or redirects to the default one. */ +export function getAvatar(userId: string) { + return http.get(`${BASE_URL}/userbook/avatar/${userId}`, + { headers: getHeaders(), redirects: 0, responseType: "binary" }); +} + +/** + * POST /directory/massmessaging/column/mapping — uploads a CSV and asks for its column mapping. The service + * pulls the uploaded directory back from storage with copyDirectoryToFs before parsing it, so a storage + * failure surfaces as the "Failed to copy import files from storage to FS" error rather than as a mapping. + * + * The form fields mirror the ones edifice-k6-commons posts to /directory/wizard/import, which is a payload + * known to pass ImportInfos.validate. + */ +export function postMassMessagingColumnMapping(structureId: string, structureName: string, csv: ArrayBuffer) { + const form = new FormData(); + form.append("type", "CSV"); + form.append("structureName", structureName); + form.append("structureId", structureId); + form.append("Teacher", http.file(csv, "enseignants.csv")); + const headers = getHeaders(); + headers["Content-Type"] = "multipart/form-data; boundary=" + form.boundary; + return http.post(`${BASE_URL}/directory/massmessaging/column/mapping`, form.body(), { headers }); +} + +/** + * GET /archive/export/verify/:exportId with an explicit, short timeout. + * + * The commons helper leaves k6 to its 60s default, which turns a stalled export into a poll every 61 + * seconds — silent, and long. An export that is merely not ready answers immediately; one that died + * answers not at all, and that is worth finding out in seconds rather than in minutes. + */ +export function verifyExport(exportId: string, timeout = "10s") { + return http.get(`${BASE_URL}/archive/export/verify/${exportId}`, + { headers: getHeaders(), timeout }); +} diff --git a/tests/src/test/js/it/scenarios/storage/avatar.ts b/tests/src/test/js/it/scenarios/storage/avatar.ts new file mode 100644 index 0000000000..4d758fe7a4 --- /dev/null +++ b/tests/src/test/js/it/scenarios/storage/avatar.ts @@ -0,0 +1,160 @@ +import { describe } from "https://jslib.k6.io/k6chaijs/4.3.4.0/index.js"; +import { check, sleep } from "k6"; +import crypto from "k6/crypto"; + +import { + authenticateWeb, + getConnectedUserId, + uploadFile, +} from '../../../node_modules/edifice-k6-commons/dist/index.js'; +import { + StorageInitData, + initStorageFixture, + updateAvatar, + getAvatar, +} from './_utils.ts'; + +/** + * Storage.findByFilenameEndingWith, and the Storage.removeFiles that follows it. + * + * On S3Storage this is the only caller of S3Client.getObjectsEndingWith, which issues a ListObjectsV2 — + * "list-type=2&prefix=..." — so it is the one route that signs a request carrying several query parameters. + * That is exactly what IMPULS-6155 touches: the values used to go through URLEncoder rather than an RFC 3986 + * encoder, and the parameter ordering was left to the caller. + * + * The trigger is an avatar update: DefaultUserBookService.update calls cleanAvatarCache whenever the payload + * carries a "picture", which lists the cached thumbnails of that user and removes them. + * + * How the listing gets pinned despite cleanAvatarCache swallowing its failures: the second update has to + * drop the avatars cached by the first one before caching its own. If the listing came back empty when it + * should not have, nothing would be removed and the route would keep serving the first picture — so the + * check that the served bytes changed between the two updates is what actually proves the listing found + * its keys. The route answering 200 alone would prove nothing. + * + * Caveat worth knowing before reading a failure: Directory builds the avatar storage as an S3Storage only + * when the directory config holds an "s3avatars" block, and falls back to FileStorage otherwise. Without + * that block this scenario still passes, but it exercises FileStorage — it says nothing about SigV4. + */ + +const maxDuration = __ENV.MAX_DURATION || "5m"; +const schoolName = __ENV.DATA_SCHOOL_NAME || "General - One user - Storage avatar"; +const gracefulStop = parseInt(__ENV.GRACEFUL_STOP || "2s"); + +export const options = { + setupTimeout: "1h", + thresholds: { + checks: ["rate == 1.00"], + }, + scenarios: { + testFindByFilenameEndingWith: { + executor: "per-vu-iterations", + exec: "testFindByFilenameEndingWith", + vus: 1, + maxDuration: maxDuration, + gracefulStop, + }, + } +}; + +const dataRootPath = __ENV.DATA_ROOT_PATH; + +let firstPicture: ArrayBuffer; +let secondPicture: ArrayBuffer; +try { + firstPicture = open(`${dataRootPath}/workspace/small.png`, "b"); + secondPicture = open(`${dataRootPath}/workspace/big-picture.jpg`, "b"); +} catch (e) { + firstPicture = open(`${dataRootPath}/data/workspace/small.png`, "b"); + secondPicture = open(`${dataRootPath}/data/workspace/big-picture.jpg`, "b"); +} + +/** The size of a k6 response body, whichever shape it came back in. */ +function bodyLength(body: any): number { + if (body == null) { + return 0; + } + if (typeof body === "string") { + return body.length; + } + return typeof body.byteLength === "number" ? body.byteLength : 0; +} + +/** + * Polls the avatar route until it serves something, and until the extra condition holds. The caching runs + * off the request, through the image resizer, so how long it takes is not ours to know — a fixed sleep + * either flakes or wastes time. + */ +function awaitAvatar(userId: string, until?: (r: any) => boolean, timeoutSeconds = 30): any { + for (let waited = 0; waited < timeoutSeconds; waited++) { + const res = getAvatar(userId); + if (res.status === 200 && bodyLength(res.body) > 0 && (until == null || until(res))) { + return res; + } + sleep(1); + } + console.error(`Avatar never settled for ${userId} within ${timeoutSeconds}s`); + return null; +} + +export function setup(): StorageInitData { + return initStorageFixture(schoolName); +} + +export function testFindByFilenameEndingWith(data: StorageInitData) { + describe('[Storage] List and drop the cached avatars of a user', () => { + authenticateWeb(data.user.login); + const userId = getConnectedUserId() as string; + + // First update: the cache is empty, so the listing matches nothing and only the caching runs. + const uploaded = uploadFile(firstPicture, "avatar.png"); + let res = updateAvatar(userId, uploaded._id); + let ok = check(res, { + "the first avatar update should be ok": (r) => r.status === 200, + "the first avatar update should echo the picture": (r) => { + const picture = r.json("picture"); + return typeof picture === "string" && picture.indexOf(uploaded._id) >= 0; + }, + }); + if (!ok) { + console.error(`First avatar update failed: ${res.status} - ${res.body}`); + return; + } + + // The thumbnails are cached asynchronously, so poll rather than sleep a fixed amount. + const first = awaitAvatar(userId); + check(first, { + "the avatar should be served after the first update": (r) => r != null && r.status === 200, + "the served avatar should have content": (r) => r != null && bodyLength(r.body) > 0, + }); + + // Second update: this is the interesting one. The cache now holds objects whose keys end with the user + // id, so findByFilenameEndingWith has to list them and removeFiles has to drop them. A ListObjectsV2 + // that came back rejected leaves the response a 200 all the same, so what the check below pins is that + // the update completed and the avatar is still served afterwards. + const replacement = uploadFile(secondPicture, "avatar-2.jpg"); + res = updateAvatar(userId, replacement._id); + ok = check(res, { + "the second avatar update should be ok": (r) => r.status === 200, + "the second avatar update should echo the new picture": (r) => { + const picture = r.json("picture"); + return typeof picture === "string" && picture.indexOf(replacement._id) >= 0; + }, + }); + if (!ok) { + console.error(`Second avatar update failed: ${res.status} - ${res.body}`); + return; + } + + const second = awaitAvatar(userId, (r) => + first == null || crypto.sha256(r.body as ArrayBuffer, "hex") !== + crypto.sha256(first.body as ArrayBuffer, "hex")); + check(second, { + "the avatar should still be served after the replacement": (r) => r != null && r.status === 200, + // The pin on findByFilenameEndingWith: the first picture had to be listed and removed for this to + // be the second one. A listing that came back empty would leave the first avatar in place. + "the served avatar should be the replacement, not the first one": (r) => + r != null && first != null && + crypto.sha256(r.body as ArrayBuffer, "hex") !== crypto.sha256(first.body as ArrayBuffer, "hex"), + }); + }); +} diff --git a/tests/src/test/js/it/scenarios/storage/copy.ts b/tests/src/test/js/it/scenarios/storage/copy.ts new file mode 100644 index 0000000000..af0dacf249 --- /dev/null +++ b/tests/src/test/js/it/scenarios/storage/copy.ts @@ -0,0 +1,136 @@ +import { describe } from "https://jslib.k6.io/k6chaijs/4.3.4.0/index.js"; +import { check } from "k6"; +import crypto from "k6/crypto"; + +import { + authenticateWeb, + uploadFile, +} from '../../../node_modules/edifice-k6-commons/dist/index.js'; +import { + StorageInitData, + initStorageFixture, + createFolderOrFail, + copyDocument, + copyDocuments, + downloadDocumentBinary, + copiedIds, +} from './_utils.ts'; + +/** + * Storage.copyFile — S3Client.copyFile, the CopyObject request carrying x-amz-copy-source. + * + * That header is the one SUPPORT-4854 fixed (it used to be set after the signature, so S3 rejected the + * request naming it as unsigned) and the one IMPULS-6155 now percent encodes. Nothing exercised it end to + * end: no k6 scenario copies a document, and the workspace copy route is the only way in. + */ + +const maxDuration = __ENV.MAX_DURATION || "5m"; +const schoolName = __ENV.DATA_SCHOOL_NAME || "General - One user - Storage copy"; +const gracefulStop = parseInt(__ENV.GRACEFUL_STOP || "2s"); + +export const options = { + setupTimeout: "1h", + thresholds: { + checks: ["rate == 1.00"], + }, + scenarios: { + testCopyDocument: { + executor: "per-vu-iterations", + exec: "testCopyDocument", + vus: 1, + maxDuration: maxDuration, + gracefulStop, + }, + testCopyDocumentsInBulk: { + executor: "per-vu-iterations", + exec: "testCopyDocumentsInBulk", + vus: 1, + maxDuration: maxDuration, + gracefulStop, + }, + } +}; + +const dataRootPath = __ENV.DATA_ROOT_PATH; + +let fileToUpload: ArrayBuffer; +try { + fileToUpload = open(`${dataRootPath}/workspace/small.png`, "b"); +} catch (e) { + fileToUpload = open(`${dataRootPath}/data/workspace/small.png`, "b"); +} + +export function setup(): StorageInitData { + return initStorageFixture(schoolName); +} + +export function testCopyDocument(data: StorageInitData) { + describe('[Storage] Copy one document', () => { + authenticateWeb(data.user.login); + + const original = uploadFile(fileToUpload, "small.png"); + const folderId = createFolderOrFail("Copies " + Date.now()); + + const copyRes = copyDocument(original._id, folderId); + let ok = check(copyRes, { + "copy should be ok": (r) => r.status === 200, + "copy should return one document": (r) => copiedIds(r).length === 1, + }); + if (!ok) { + console.error(`Copy failed: ${copyRes.status} - ${copyRes.body}`); + return; + } + const copyId = copiedIds(copyRes)[0]; + + check(copyId, { + "the copy should be a distinct document": (id) => id !== original._id, + }); + + // The point of the scenario: the copy is readable, and holds the very same bytes. A CopyObject that + // silently failed, or copied the wrong key, shows up here and nowhere else. + const originalBytes = downloadDocumentBinary(original._id); + const copyBytes = downloadDocumentBinary(copyId); + ok = check({ originalBytes, copyBytes }, { + "original should still download": (r) => r.originalBytes.status === 200, + "copy should download": (r) => r.copyBytes.status === 200, + "copy should have the same size": (r) => + (r.copyBytes.body as ArrayBuffer).byteLength === (r.originalBytes.body as ArrayBuffer).byteLength, + "copy should be byte for byte identical": (r) => + crypto.sha256(r.copyBytes.body as ArrayBuffer, "hex") === + crypto.sha256(r.originalBytes.body as ArrayBuffer, "hex"), + }); + if (!ok) { + console.error(`Original status ${originalBytes.status}, copy status ${copyBytes.status}`); + } + }); +} + +export function testCopyDocumentsInBulk(data: StorageInitData) { + describe('[Storage] Copy several documents', () => { + authenticateWeb(data.user.login); + + const first = uploadFile(fileToUpload, "first.png"); + const second = uploadFile(fileToUpload, "second.png"); + const folderId = createFolderOrFail("Bulk copies " + Date.now()); + + const copyRes = copyDocuments([first._id, second._id], folderId); + const ok = check(copyRes, { + "bulk copy should be ok": (r) => r.status === 200, + "bulk copy should return two documents": (r) => copiedIds(r).length === 2, + }); + if (!ok) { + console.error(`Bulk copy failed: ${copyRes.status} - ${copyRes.body}`); + return; + } + + // Each copy triggers its own CopyObject: one signature per file, so a header set after the signature + // fails on all of them rather than on the first only. + for (const copyId of copiedIds(copyRes)) { + const res = downloadDocumentBinary(copyId); + check(res, { + "each copy should download": (r) => r.status === 200, + "each copy should have content": (r) => (r.body as ArrayBuffer).byteLength > 0, + }); + } + }); +} diff --git a/tests/src/test/js/it/scenarios/storage/csv-mapping.ts b/tests/src/test/js/it/scenarios/storage/csv-mapping.ts new file mode 100644 index 0000000000..046b41ae77 --- /dev/null +++ b/tests/src/test/js/it/scenarios/storage/csv-mapping.ts @@ -0,0 +1,100 @@ +import { describe } from "https://jslib.k6.io/k6chaijs/4.3.4.0/index.js"; +import { check } from "k6"; + +import { + authenticateWeb, + Session, + Structure, + createEmptyStructure, +} from '../../../node_modules/edifice-k6-commons/dist/index.js'; +import { postMassMessagingColumnMapping } from './_utils.ts'; + +/** + * Storage.copyDirectoryToFs — the whole-prefix download: a ListObjectsV2 over the prefix, then one GetObject + * per key, each written to the local file system. + * + * DefaultMassMessagingService.csvColumnsMapping calls it directly, and reports its failure as a distinct + * error ("Failed to copy import files from storage to FS"), so this route tells apart a storage problem from + * a parsing one. The other caller, CsvValidator, reaches it behind the feeder and gives no such signal. + * + * The route is admin only (AdminFilter) and MFA protected, so the scenario runs as the ADMC. On an + * environment where MFA is enforced it will answer 401 rather than 200 — that is a configuration mismatch, + * not a storage regression. + */ + +const maxDuration = __ENV.MAX_DURATION || "5m"; +const schoolName = __ENV.DATA_SCHOOL_NAME || "General - One user - Storage csv"; +const gracefulStop = parseInt(__ENV.GRACEFUL_STOP || "2s"); + +export const options = { + setupTimeout: "1h", + thresholds: { + checks: ["rate == 1.00"], + }, + scenarios: { + testCopyDirectoryToFs: { + executor: "per-vu-iterations", + exec: "testCopyDirectoryToFs", + vus: 1, + maxDuration: maxDuration, + gracefulStop, + }, + } +}; + +const dataRootPath = __ENV.DATA_ROOT_PATH; + +let teachersCsv: ArrayBuffer; +try { + teachersCsv = open(`${dataRootPath}/positions/csv/before/enseignants.csv`, "b"); +} catch (e) { + teachersCsv = open(`${dataRootPath}/data/positions/csv/before/enseignants.csv`, "b"); +} + +type InitData = { + head: Structure; +} + +export function setup(): InitData { + let structure: Structure | null = null; + describe("[Storage-Init] Initialize data", () => { + authenticateWeb(__ENV.ADMC_LOGIN, __ENV.ADMC_PASSWORD); + structure = createEmptyStructure(`${schoolName}`, true); + }); + return { head: structure }; +} + +export function testCopyDirectoryToFs(data: InitData) { + describe('[Storage] Pull an uploaded directory back from storage', () => { + authenticateWeb(__ENV.ADMC_LOGIN, __ENV.ADMC_PASSWORD); + + const res = postMassMessagingColumnMapping(data.head.id, data.head.name, teachersCsv); + const ok = check(res, { + "the column mapping should be ok": (r) => r.status === 200, + // The discriminating check: this is the message the service returns when copyDirectoryToFs fails, + // whatever the reason — a rejected signature, a listing that came back empty, a GetObject that 404ed. + "the files should have been copied back from storage": (r) => + r.body != null && (r.body as string).indexOf("Failed to copy import files from storage to FS") < 0, + "no error should be reported at all": (r) => { + const error = r.json("error"); + return error === undefined || error === null; + }, + }); + if (!ok) { + console.error(`Column mapping failed: ${res.status} - ${res.body}`); + return; + } + + // And the CSV really was read from the copied directory: the mapping holds one record per line. + check(res, { + "the mapping should hold the CSV records": (r) => { + const records = r.json("asmRecords"); + return Array.isArray(records) && records.length > 0; + }, + "each record should hold columns": (r) => { + const records = r.json("asmRecords") as any[]; + return Array.isArray(records) && records.every((row) => Array.isArray(row) && row.length > 0); + }, + }); + }); +} diff --git a/tests/src/test/js/it/scenarios/storage/delete.ts b/tests/src/test/js/it/scenarios/storage/delete.ts new file mode 100644 index 0000000000..93d2e76878 --- /dev/null +++ b/tests/src/test/js/it/scenarios/storage/delete.ts @@ -0,0 +1,117 @@ +import { describe } from "https://jslib.k6.io/k6chaijs/4.3.4.0/index.js"; +import { check } from "k6"; + +import { + authenticateWeb, + uploadFile, + downloadFile, +} from '../../../node_modules/edifice-k6-commons/dist/index.js'; +import { + StorageInitData, + initStorageFixture, + deleteDocument, + deleteDocuments, +} from './_utils.ts'; + +/** + * Storage.removeFile and Storage.removeFiles — the DeleteObject requests. + * + * Both sign a request whose only payload is the key in the URI, so they are the shortest path on which a + * wrong canonical URI or a wrong signed host surfaces. No existing scenario deletes a document. + */ + +const maxDuration = __ENV.MAX_DURATION || "5m"; +const schoolName = __ENV.DATA_SCHOOL_NAME || "General - One user - Storage delete"; +const gracefulStop = parseInt(__ENV.GRACEFUL_STOP || "2s"); + +export const options = { + setupTimeout: "1h", + thresholds: { + checks: ["rate == 1.00"], + }, + scenarios: { + testRemoveFile: { + executor: "per-vu-iterations", + exec: "testRemoveFile", + vus: 1, + maxDuration: maxDuration, + gracefulStop, + }, + testRemoveFiles: { + executor: "per-vu-iterations", + exec: "testRemoveFiles", + vus: 1, + maxDuration: maxDuration, + gracefulStop, + }, + } +}; + +const dataRootPath = __ENV.DATA_ROOT_PATH; + +let fileToUpload: ArrayBuffer; +try { + fileToUpload = open(`${dataRootPath}/workspace/small.png`, "b"); +} catch (e) { + fileToUpload = open(`${dataRootPath}/data/workspace/small.png`, "b"); +} + +export function setup(): StorageInitData { + return initStorageFixture(schoolName); +} + +export function testRemoveFile(data: StorageInitData) { + describe('[Storage] Remove one file', () => { + authenticateWeb(data.user.login); + + const uploaded = uploadFile(fileToUpload, "to-delete.png"); + check(downloadFile(uploaded._id), { + "the file should download before deletion": (r) => r.status === 200, + }); + + const deleteRes = deleteDocument(uploaded._id); + const ok = check(deleteRes, { + "delete should be ok": (r) => r.status === 200, + }); + if (!ok) { + console.error(`Delete failed: ${deleteRes.status} - ${deleteRes.body}`); + return; + } + + // The document row is gone, so the route no longer resolves a file id. A DeleteObject that silently + // failed leaves the object in the bucket, which this cannot see — what it does pin is that the delete + // request itself was accepted rather than rejected on its signature. + check(downloadFile(uploaded._id), { + "the file should not download after deletion": (r) => r.status === 404 || r.status === 401, + }); + }); +} + +export function testRemoveFiles(data: StorageInitData) { + describe('[Storage] Remove several files', () => { + authenticateWeb(data.user.login); + + const uploaded = [ + uploadFile(fileToUpload, "batch-1.png"), + uploadFile(fileToUpload, "batch-2.png"), + uploadFile(fileToUpload, "batch-3.png"), + ]; + const ids = uploaded.map((f) => f._id); + + const deleteRes = deleteDocuments(ids); + const ok = check(deleteRes, { + "bulk delete should be ok": (r) => r.status === 200, + "bulk delete should report the three documents": (r) => r.json("number") === 3, + }); + if (!ok) { + console.error(`Bulk delete failed: ${deleteRes.status} - ${deleteRes.body}`); + return; + } + + for (const id of ids) { + check(downloadFile(id), { + "each deleted file should not download": (r) => r.status === 404 || r.status === 401, + }); + } + }); +} diff --git a/tests/src/test/js/it/scenarios/storage/read.ts b/tests/src/test/js/it/scenarios/storage/read.ts new file mode 100644 index 0000000000..af8e6d7057 --- /dev/null +++ b/tests/src/test/js/it/scenarios/storage/read.ts @@ -0,0 +1,104 @@ +import { describe } from "https://jslib.k6.io/k6chaijs/4.3.4.0/index.js"; +import { check } from "k6"; +import crypto from "k6/crypto"; +import encoding from "k6/encoding"; + +import { + authenticateWeb, + uploadFile, +} from '../../../node_modules/edifice-k6-commons/dist/index.js'; +import { + StorageInitData, + initStorageFixture, + getDocumentBase64, + downloadDocumentBinary, +} from './_utils.ts'; + +/** + * Storage.readFile — the GetObject whose whole body is buffered in memory, as opposed to Storage.sendFile + * which streams it to the response. The two take different paths through S3Client, and only sendFile is + * covered today (by workspace/nominal.ts, through the download and thumbnail routes). + * + * The fixture is deliberately a text file, not an image: with the image resizer wired to the bucket, an + * uploaded image is re-encoded on the way in — small.png goes from 201 to 94 bytes — so "what came back + * equals what was sent" would be asserting the optimiser away rather than the read. A text file travels + * untouched, which is what makes the byte comparison below mean something. + * + * Storage.fileStats is deliberately not here: on a workspace document the properties route reads Mongo, not + * the bucket. The only route that reaches fileStats is /archive/export/verify/:exportId, already exercised + * by workspace/export.ts. + */ + +const maxDuration = __ENV.MAX_DURATION || "5m"; +const schoolName = __ENV.DATA_SCHOOL_NAME || "General - One user - Storage read"; +const gracefulStop = parseInt(__ENV.GRACEFUL_STOP || "2s"); + +export const options = { + setupTimeout: "1h", + thresholds: { + checks: ["rate == 1.00"], + }, + scenarios: { + testReadFile: { + executor: "per-vu-iterations", + exec: "testReadFile", + vus: 1, + maxDuration: maxDuration, + gracefulStop, + }, + } +}; + +const dataRootPath = __ENV.DATA_ROOT_PATH; + +let fileToUpload: ArrayBuffer; +try { + fileToUpload = open(`${dataRootPath}/workspace/random_text_file.txt`, "b"); +} catch (e) { + fileToUpload = open(`${dataRootPath}/data/workspace/random_text_file.txt`, "b"); +} + +export function setup(): StorageInitData { + return initStorageFixture(schoolName); +} + +export function testReadFile(data: StorageInitData) { + describe('[Storage] Read a file into memory', () => { + authenticateWeb(data.user.login); + + const uploaded = uploadFile(fileToUpload, "to-read.txt"); + + const base64Res = getDocumentBase64(uploaded._id); + let ok = check(base64Res, { + "base64 read should be ok": (r) => r.status === 200, + "base64 read should return content": (r) => { + const encoded = r.json("base64File"); + return typeof encoded === "string" && encoded.length > 0; + }, + "base64 read should return the document name": (r) => r.json("title") === "to-read.txt", + }); + if (!ok) { + console.error(`Base64 read failed: ${base64Res.status} - ${base64Res.body}`); + return; + } + + // What readFile buffered has to be the file that was uploaded, not a truncated or re-encoded version of + // it: a wrong payload hash on the upload signature would have been rejected, but a partial read would + // not, and only a byte comparison catches that. + const decoded = encoding.b64decode(base64Res.json("base64File") as string, "std", "b"); + check(decoded, { + "the buffered content should have the uploaded size": (d) => + (d as ArrayBuffer).byteLength === fileToUpload.byteLength, + "the buffered content should be the uploaded bytes": (d) => + crypto.sha256(d as ArrayBuffer, "hex") === crypto.sha256(fileToUpload, "hex"), + }); + + // And the streaming path returns the same thing, so readFile and sendFile cannot drift apart. + const streamed = downloadDocumentBinary(uploaded._id); + check(streamed, { + "the streamed content should match the buffered one": (r) => + r.status === 200 && + crypto.sha256(r.body as ArrayBuffer, "hex") === crypto.sha256(decoded as ArrayBuffer, "hex"), + }); + }); +} diff --git a/tests/src/test/js/it/scenarios/storage/special-characters.ts b/tests/src/test/js/it/scenarios/storage/special-characters.ts new file mode 100644 index 0000000000..cc5deb5992 --- /dev/null +++ b/tests/src/test/js/it/scenarios/storage/special-characters.ts @@ -0,0 +1,283 @@ +import { describe } from "https://jslib.k6.io/k6chaijs/4.3.4.0/index.js"; +import { check, fail, sleep } from "k6"; +import crypto from "k6/crypto"; + +import { + authenticateWeb, + uploadFile, + downloadFile, + launchExportOrFail, + downloadExportFile, + parseZip, + getZipTree, +} from '../../../node_modules/edifice-k6-commons/dist/index.js'; +import { + StorageInitData, + initStorageFixture, + createFolderOrFail, + copyDocument, + copiedIds, + downloadDocumentBinary, + verifyExport, +} from './_utils.ts'; + +/** + * The encoding dimension of IMPULS-6155, which no scenario covers today. + * + * Two distinct paths are at stake, and it matters not to confuse them: + * + * - A workspace object key is a generated UUID, so a special character in a file name never reaches + * S3Client.encodeUrlPath on that route. What it does reach is the x-amz-meta-filename header, which is + * quoted-printable encoded and signed, and the Content-Disposition of the download. + * - The archive export names each exported file after the document (S3Storage.writeToFileSystem uses + * alias.getString(id, id)), and an import pushes that tree back with S3Storage.moveFsDirectory. Those + * keys carry real names, and that is where the RFC 3986 encoder replaces URLEncoder — a space used to be + * written as +, storing a literally wrong key, and ~ and * used to yield SignatureDoesNotMatch. + */ + +const maxDuration = __ENV.MAX_DURATION || "10m"; +const schoolName = __ENV.DATA_SCHOOL_NAME || "General - One user - Storage encoding"; +const gracefulStop = parseInt(__ENV.GRACEFUL_STOP || "2s"); +/** + * Five minutes, not thirty. A workspace export of a handful of small files is a matter of seconds; anything + * beyond this is an export that died, and waiting half an hour to be told so is not a test, it is a hang. + * Raise it with EXPORT_TIMEOUT_SECONDS on a slower environment. + */ +const EXPORT_TIMEOUT = parseInt(__ENV.EXPORT_TIMEOUT_SECONDS || "300") * 1000; + +export const options = { + setupTimeout: "1h", + thresholds: { + checks: ["rate == 1.00"], + }, + scenarios: { + testSpecialCharacterNames: { + executor: "per-vu-iterations", + exec: "testSpecialCharacterNames", + vus: 1, + maxDuration: maxDuration, + gracefulStop, + }, + testSpecialCharacterNamesSurviveExport: { + executor: "per-vu-iterations", + exec: "testSpecialCharacterNamesSurviveExport", + vus: 1, + maxDuration: maxDuration, + gracefulStop, + }, + } +}; + +const dataRootPath = __ENV.DATA_ROOT_PATH; + +let fileToUpload: ArrayBuffer; +try { + fileToUpload = open(`${dataRootPath}/workspace/small.png`, "b"); +} catch (e) { + fileToUpload = open(`${dataRootPath}/data/workspace/small.png`, "b"); +} + +/** One name per character URLEncoder and RFC 3986 disagree on, plus non ASCII. */ +const NAMES = [ + "rapport final.png", // space: URLEncoder wrote +, so the stored key was literally wrong + "note~1.png", // tilde: URLEncoder escaped it, the server did not + "etoile*.png", // star: URLEncoder left it raw, the server escaped it + "a+b.png", // plus: has to survive as a literal plus, not decode to a space + "100% sur.png", // percent: the double encoding case + "eleve prive.png", // space again, with the accents stripped from the ASCII name + "élève privé.png", // non ASCII: encoded as UTF-8 percent escapes +]; + +/** + * The size of a k6 response body, whichever shape it came back in: a text response is a string, a binary + * one an ArrayBuffer, and only the latter has byteLength. Reading .length on an ArrayBuffer yields + * undefined, which compares false against anything — a passing download then looks like an empty one. + */ +function bodyLength(body: any): number { + if (body == null) { + return 0; + } + if (typeof body === "string") { + return body.length; + } + if (typeof body.byteLength === "number") { + return body.byteLength; + } + return typeof body.length === "number" ? body.length : 0; +} + +/** + * A pattern matching the entry the export is expected to carry for a stored file name. + * + * The export renames what it cannot put on a file system: a {@code *} comes out as {@code _}. That is + * deliberate, so the assertion allows a substitution on exactly those characters and on nothing else — + * a space, a tilde, a plus, a percent and the accents all have to survive verbatim. + */ +function zipEntryPattern(storedName: string): RegExp { + const escaped = storedName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + // Re-open the classes the export is allowed to rewrite. + const relaxed = escaped.replace(/\\\*/g, "."); + return new RegExp(relaxed + "$"); +} + +/** Printable ASCII only: a name outside that range cannot survive an ISO-8859-1 header. */ +function isAscii(value: string): boolean { + return /^[\x20-\x7E]*$/.test(value); +} + +export function setup(): StorageInitData { + return initStorageFixture(schoolName); +} + +export function testSpecialCharacterNames(data: StorageInitData) { + describe('[Storage] Upload, download and copy files with special characters', () => { + authenticateWeb(data.user.login); + + const folderId = createFolderOrFail("Encoding " + Date.now()); + + for (const name of NAMES) { + const uploaded = uploadFile(fileToUpload, name); + let ok = check(uploaded, { + [`should upload "${name}"`]: (f) => !!f && !!f._id, + }); + if (!ok) { + console.error(`Upload of "${name}" returned ${JSON.stringify(uploaded)}`); + continue; + } + + if (isAscii(name)) { + check(uploaded, { + [`should keep the name of "${name}"`]: (f) => f.metadata.filename === name, + }); + } else if (uploaded.metadata.filename !== name) { + // Not asserted, and not a storage defect: the multipart file name travels in a + // Content-Disposition header, which HTTP defines as ISO-8859-1. k6 writes the raw UTF-8 bytes + // rather than the filename*=UTF-8'' form of RFC 5987, and the server does not read that form + // either, so a non ASCII name comes back mangled. Both ends would have to adopt RFC 5987. + // The file is kept in the list all the same: its bytes still exercise a UTF-8 encoded S3 key. + console.warn(`Non ASCII name does not round-trip: sent "${name}", ` + + `stored "${uploaded.metadata.filename}" — RFC 5987 gap, unrelated to storage.`); + } + + // The download signs the same key again. Content-Disposition is deliberately not asserted: the + // workspace download route serves inline by default and makes no promise about that header. + const downloaded = downloadFile(uploaded._id); + check(downloaded, { + [`should download "${name}"`]: (r) => r.status === 200, + [`should return content for "${name}"`]: (r) => bodyLength(r.body) > 0, + }); + + // The reference for the copy is the stored object, read back, not the local file: an uploaded image + // is re-encoded by the resizer on the way in, so the local bytes are not what sits in the bucket. + const originalBytes = downloadDocumentBinary(uploaded._id); + const expectedDigest = originalBytes.status === 200 + ? crypto.sha256(originalBytes.body as ArrayBuffer, "hex") + : null; + + // And the copy re-signs it as an x-amz-copy-source header value. + const copyRes = copyDocument(uploaded._id, folderId); + ok = check(copyRes, { + [`should copy "${name}"`]: (r) => r.status === 200 && copiedIds(r).length === 1, + }); + if (!ok) { + console.error(`Copy of "${name}" failed: ${copyRes.status} - ${copyRes.body}`); + continue; + } + const copyBytes = downloadDocumentBinary(copiedIds(copyRes)[0]); + check(copyBytes, { + [`the copy of "${name}" should hold the same bytes`]: (r) => + r.status === 200 && expectedDigest != null && + crypto.sha256(r.body as ArrayBuffer, "hex") === expectedDigest, + }); + } + }); +} + +export function testSpecialCharacterNamesSurviveExport(data: StorageInitData) { + describe('[Storage] Export files whose names carry special characters', () => { + authenticateWeb(data.user.login); + + // What the server stored is what the export will name the file after, and for a non ASCII name that + // is not what we sent — see the RFC 5987 note above. Asserting on the sent name would be asserting + // that gap all over again; the round-trip worth pinning here is storage to export. + const storedNames = NAMES.map((name) => { + const uploaded = uploadFile(fileToUpload, name); + return { sent: name, stored: uploaded.metadata.filename }; + }); + + const exportId = launchExportOrFail(["workspace"]); + console.log(`Export launched with id ${exportId}, waiting for it to be ready...`); + const startTime = Date.now(); + let exportReady = false; + let attempts = 0; + let lastStatus = 0; + while (!exportReady && (Date.now() - startTime) < EXPORT_TIMEOUT) { + attempts++; + const verifyRes = verifyExport(exportId); + lastStatus = verifyRes.status; + if (verifyRes.status === 200) { + exportReady = true; + } else if (verifyRes.status === 500) { + fail(`Export failed with status 500. Response: ${verifyRes.body}`); + } else if (verifyRes.status === 404) { + fail(`Export not found with status 404. Response: ${verifyRes.body}`); + } else { + // Every ten attempts, say where we are: an export that died leaves this loop silent otherwise, and + // the reason is in the server log, not here. + if (attempts % 10 === 0) { + console.log(`Export ${exportId} still not ready after ${attempts} attempts ` + + `(last status ${verifyRes.status}); check the server log for an unhandled exception.`); + } + sleep(1); + } + } + check(exportReady, { + "export should be ready within timeout": (ready) => ready, + }); + if (!exportReady) { + console.error(`Export ${exportId} never became ready: ${attempts} attempts, last status ` + + `${lastStatus}, gave up after ${Math.round((Date.now() - startTime) / 1000)}s.`); + return; + } + + const downloadRes = downloadExportFile(exportId); + const ok = check(downloadRes, { + "should download the export": (r) => r.status === 200, + "the export should have content": (r) => bodyLength(r.body) > 0, + }); + if (!ok) { + console.error(`Export download failed: HTTP ${downloadRes.status}, ` + + `${bodyLength(downloadRes.body)} bytes`); + return; + } + + // Every exported file is named after its document, so each of these names became an S3 key on the way + // out. A name that comes back mangled — a space turned into a +, a doubled percent — means the encoder + // and the server disagreed. + const tree = getZipTree(parseZip(downloadRes.body)); + let allFound = true; + for (const { sent, stored } of storedNames) { + if (!isAscii(sent)) { + // Skipped for the same reason the upload name is: the name reaching this zip has been through two + // charset gaps — the RFC 5987 one on upload, then the zip entry encoding, which turns the stored + // U+FFFD into "�". Neither is a storage concern, and the bytes themselves did travel: the + // server log shows the export downloading this object under its UTF-8 encoded S3 key. + console.warn(`Not asserted in the export, non ASCII name: sent "${sent}", stored "${stored}".`); + continue; + } + const pattern = zipEntryPattern(stored); + const found = check(tree, { + [`the export should hold "${sent}"`]: (entries) => + entries.some((entry: string) => pattern.test(entry)), + }); + if (!found) { + allFound = false; + console.error(`Missing from the export: sent "${sent}", stored "${stored}", ` + + `looked for /${pattern.source}/.`); + } + } + if (!allFound) { + console.error(`Export tree was: ${JSON.stringify(tree)}`); + } + }); +} From 3af4df93113410071cb648b4591ff62de8714d1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre-Beno=C3=AEt=20JOUBERT?= Date: Fri, 4 Sep 2026 14:55:08 +0200 Subject: [PATCH 2/2] Feedbacks #1 --- .../test/js/it/scenarios/storage/_utils.ts | 81 ++----------------- .../test/js/it/scenarios/storage/avatar.ts | 2 +- .../src/test/js/it/scenarios/storage/copy.ts | 18 ++--- .../js/it/scenarios/storage/csv-mapping.ts | 2 +- .../test/js/it/scenarios/storage/delete.ts | 6 +- .../src/test/js/it/scenarios/storage/read.ts | 8 +- .../scenarios/storage/special-characters.ts | 19 ++--- tests/src/test/js/pnpm-lock.yaml | 10 +-- 8 files changed, 39 insertions(+), 107 deletions(-) diff --git a/tests/src/test/js/it/scenarios/storage/_utils.ts b/tests/src/test/js/it/scenarios/storage/_utils.ts index 253a0de88f..8145e707ff 100644 --- a/tests/src/test/js/it/scenarios/storage/_utils.ts +++ b/tests/src/test/js/it/scenarios/storage/_utils.ts @@ -1,7 +1,6 @@ import { describe } from "https://jslib.k6.io/k6chaijs/4.3.4.0/index.js"; import http from "k6/http"; import { FormData } from "https://jslib.k6.io/formdata/0.0.2/index.js"; -import { fail } from "k6"; import { BASE_URL, @@ -18,12 +17,14 @@ import { } from '../../../node_modules/edifice-k6-commons/dist/index.js'; /** - * Shared fixture and thin route wrappers for the storage scenarios. + * Shared fixture and the route wrappers that stay local to the storage scenarios. * * These scenarios exercise org.entcore.common.storage.impl.S3Storage through the only surface k6 can reach: - * the workspace and archive routes that call it. edifice-k6-commons covers upload, download and the archive - * flows; the copy, delete and read-back routes have no helper there, so they are wrapped here rather than - * inlined in every scenario. + * the workspace, directory and archive routes that call it. The workspace and archive wrappers now live in + * edifice-k6-commons — createFolderOrFail, copyDocument, copyDocuments, copiedIds, deleteDocument, + * deleteDocuments, getDocumentBase64, the responseType argument of downloadFile and the timeout argument of + * verifyExportFiles. What is left here is the fixture and the two directory routes, which have no helper + * there. */ export type StorageInitData = { @@ -59,64 +60,6 @@ export function initStorageFixture(schoolName: string): StorageInitData { return { head: structure, user }; } -/** POST /workspace/folder — multipart form, 201 with the created folder. */ -export function createFolderOrFail(name: string, parentFolderId?: string): string { - const body: Record = { name }; - if (parentFolderId) { - body.parentFolderId = parentFolderId; - } - const res = http.post(`${BASE_URL}/workspace/folder`, body, { headers: getHeaders() }); - if (res.status !== 201) { - fail(`could not create folder ${name}: ${res.status} - ${res.body}`); - } - return (res.json() as any)._id; -} - -/** POST /workspace/document/copy/:id/:folder — reaches Storage.copyFile through StorageHelper. */ -export function copyDocument(id: string, folderId: string) { - return http.post(`${BASE_URL}/workspace/document/copy/${id}/${folderId}`, null, { headers: getHeaders() }); -} - -/** POST /workspace/documents/copy/:folder — same, in bulk. */ -export function copyDocuments(ids: string[], folderId: string) { - return http.post(`${BASE_URL}/workspace/documents/copy/${folderId}`, JSON.stringify({ ids }), - { headers: getHeaders("application/json") }); -} - -/** DELETE /workspace/document/:id — reaches Storage.removeFile. */ -export function deleteDocument(id: string) { - return http.del(`${BASE_URL}/workspace/document/${id}`, null, { headers: getHeaders() }); -} - -/** DELETE /workspace/documents — reaches Storage.removeFiles with the whole batch. */ -export function deleteDocuments(ids: string[]) { - return http.del(`${BASE_URL}/workspace/documents`, JSON.stringify({ ids }), - { headers: getHeaders("application/json") }); -} - -/** GET /workspace/document/base64/:id — reaches Storage.readFile, which buffers the whole object. */ -export function getDocumentBase64(id: string) { - return http.get(`${BASE_URL}/workspace/document/base64/${id}`, { headers: getHeaders() }); -} - -/** - * GET /workspace/document/:id as raw bytes. downloadFile from the commons returns a text body, which is - * lossy on binary content — a byte for byte comparison needs the binary response type. - */ -export function downloadDocumentBinary(id: string) { - return http.get(`${BASE_URL}/workspace/document/${id}`, - { headers: getHeaders(), responseType: "binary" }); -} - -/** The document ids of a copy response, which returns the created documents as an array. */ -export function copiedIds(res: any): string[] { - const body = res.json(); - if (!Array.isArray(body)) { - return []; - } - return body.filter((d: any) => !!d && !!d._id).map((d: any) => d._id); -} - /** PUT /directory/avatar/:userId — every call goes through cleanAvatarCache, hence findByFilenameEndingWith. */ export function updateAvatar(userId: string, documentId: string) { return http.put(`${BASE_URL}/directory/avatar/${userId}`, @@ -148,15 +91,3 @@ export function postMassMessagingColumnMapping(structureId: string, structureNam headers["Content-Type"] = "multipart/form-data; boundary=" + form.boundary; return http.post(`${BASE_URL}/directory/massmessaging/column/mapping`, form.body(), { headers }); } - -/** - * GET /archive/export/verify/:exportId with an explicit, short timeout. - * - * The commons helper leaves k6 to its 60s default, which turns a stalled export into a poll every 61 - * seconds — silent, and long. An export that is merely not ready answers immediately; one that died - * answers not at all, and that is worth finding out in seconds rather than in minutes. - */ -export function verifyExport(exportId: string, timeout = "10s") { - return http.get(`${BASE_URL}/archive/export/verify/${exportId}`, - { headers: getHeaders(), timeout }); -} diff --git a/tests/src/test/js/it/scenarios/storage/avatar.ts b/tests/src/test/js/it/scenarios/storage/avatar.ts index 4d758fe7a4..05ccb5e58c 100644 --- a/tests/src/test/js/it/scenarios/storage/avatar.ts +++ b/tests/src/test/js/it/scenarios/storage/avatar.ts @@ -56,7 +56,7 @@ export const options = { } }; -const dataRootPath = __ENV.DATA_ROOT_PATH; +const dataRootPath = __ENV.DATA_ROOT_PATH || "../../../../resources/data"; let firstPicture: ArrayBuffer; let secondPicture: ArrayBuffer; diff --git a/tests/src/test/js/it/scenarios/storage/copy.ts b/tests/src/test/js/it/scenarios/storage/copy.ts index af0dacf249..5b209768ff 100644 --- a/tests/src/test/js/it/scenarios/storage/copy.ts +++ b/tests/src/test/js/it/scenarios/storage/copy.ts @@ -5,15 +5,15 @@ import crypto from "k6/crypto"; import { authenticateWeb, uploadFile, -} from '../../../node_modules/edifice-k6-commons/dist/index.js'; -import { - StorageInitData, - initStorageFixture, + downloadFile, createFolderOrFail, copyDocument, copyDocuments, - downloadDocumentBinary, copiedIds, +} from '../../../node_modules/edifice-k6-commons/dist/index.js'; +import { + StorageInitData, + initStorageFixture, } from './_utils.ts'; /** @@ -51,7 +51,7 @@ export const options = { } }; -const dataRootPath = __ENV.DATA_ROOT_PATH; +const dataRootPath = __ENV.DATA_ROOT_PATH || "../../../../resources/data"; let fileToUpload: ArrayBuffer; try { @@ -88,8 +88,8 @@ export function testCopyDocument(data: StorageInitData) { // The point of the scenario: the copy is readable, and holds the very same bytes. A CopyObject that // silently failed, or copied the wrong key, shows up here and nowhere else. - const originalBytes = downloadDocumentBinary(original._id); - const copyBytes = downloadDocumentBinary(copyId); + const originalBytes = downloadFile(original._id, "", "binary"); + const copyBytes = downloadFile(copyId, "", "binary"); ok = check({ originalBytes, copyBytes }, { "original should still download": (r) => r.originalBytes.status === 200, "copy should download": (r) => r.copyBytes.status === 200, @@ -126,7 +126,7 @@ export function testCopyDocumentsInBulk(data: StorageInitData) { // Each copy triggers its own CopyObject: one signature per file, so a header set after the signature // fails on all of them rather than on the first only. for (const copyId of copiedIds(copyRes)) { - const res = downloadDocumentBinary(copyId); + const res = downloadFile(copyId, "", "binary"); check(res, { "each copy should download": (r) => r.status === 200, "each copy should have content": (r) => (r.body as ArrayBuffer).byteLength > 0, diff --git a/tests/src/test/js/it/scenarios/storage/csv-mapping.ts b/tests/src/test/js/it/scenarios/storage/csv-mapping.ts index 046b41ae77..f1313b9b75 100644 --- a/tests/src/test/js/it/scenarios/storage/csv-mapping.ts +++ b/tests/src/test/js/it/scenarios/storage/csv-mapping.ts @@ -42,7 +42,7 @@ export const options = { } }; -const dataRootPath = __ENV.DATA_ROOT_PATH; +const dataRootPath = __ENV.DATA_ROOT_PATH || "../../../../resources/data"; let teachersCsv: ArrayBuffer; try { diff --git a/tests/src/test/js/it/scenarios/storage/delete.ts b/tests/src/test/js/it/scenarios/storage/delete.ts index 93d2e76878..5340a74ab1 100644 --- a/tests/src/test/js/it/scenarios/storage/delete.ts +++ b/tests/src/test/js/it/scenarios/storage/delete.ts @@ -5,12 +5,12 @@ import { authenticateWeb, uploadFile, downloadFile, + deleteDocument, + deleteDocuments, } from '../../../node_modules/edifice-k6-commons/dist/index.js'; import { StorageInitData, initStorageFixture, - deleteDocument, - deleteDocuments, } from './_utils.ts'; /** @@ -47,7 +47,7 @@ export const options = { } }; -const dataRootPath = __ENV.DATA_ROOT_PATH; +const dataRootPath = __ENV.DATA_ROOT_PATH || "../../../../resources/data"; let fileToUpload: ArrayBuffer; try { diff --git a/tests/src/test/js/it/scenarios/storage/read.ts b/tests/src/test/js/it/scenarios/storage/read.ts index af8e6d7057..6a6e1329c3 100644 --- a/tests/src/test/js/it/scenarios/storage/read.ts +++ b/tests/src/test/js/it/scenarios/storage/read.ts @@ -6,12 +6,12 @@ import encoding from "k6/encoding"; import { authenticateWeb, uploadFile, + downloadFile, + getDocumentBase64, } from '../../../node_modules/edifice-k6-commons/dist/index.js'; import { StorageInitData, initStorageFixture, - getDocumentBase64, - downloadDocumentBinary, } from './_utils.ts'; /** @@ -49,7 +49,7 @@ export const options = { } }; -const dataRootPath = __ENV.DATA_ROOT_PATH; +const dataRootPath = __ENV.DATA_ROOT_PATH || "../../../../resources/data"; let fileToUpload: ArrayBuffer; try { @@ -94,7 +94,7 @@ export function testReadFile(data: StorageInitData) { }); // And the streaming path returns the same thing, so readFile and sendFile cannot drift apart. - const streamed = downloadDocumentBinary(uploaded._id); + const streamed = downloadFile(uploaded._id, "", "binary"); check(streamed, { "the streamed content should match the buffered one": (r) => r.status === 200 && diff --git a/tests/src/test/js/it/scenarios/storage/special-characters.ts b/tests/src/test/js/it/scenarios/storage/special-characters.ts index cc5deb5992..8b357b45af 100644 --- a/tests/src/test/js/it/scenarios/storage/special-characters.ts +++ b/tests/src/test/js/it/scenarios/storage/special-characters.ts @@ -6,19 +6,18 @@ import { authenticateWeb, uploadFile, downloadFile, + createFolderOrFail, + copyDocument, + copiedIds, launchExportOrFail, downloadExportFile, + verifyExportFiles, parseZip, getZipTree, } from '../../../node_modules/edifice-k6-commons/dist/index.js'; import { StorageInitData, initStorageFixture, - createFolderOrFail, - copyDocument, - copiedIds, - downloadDocumentBinary, - verifyExport, } from './_utils.ts'; /** @@ -68,7 +67,7 @@ export const options = { } }; -const dataRootPath = __ENV.DATA_ROOT_PATH; +const dataRootPath = __ENV.DATA_ROOT_PATH || "../../../../resources/data"; let fileToUpload: ArrayBuffer; try { @@ -169,7 +168,7 @@ export function testSpecialCharacterNames(data: StorageInitData) { // The reference for the copy is the stored object, read back, not the local file: an uploaded image // is re-encoded by the resizer on the way in, so the local bytes are not what sits in the bucket. - const originalBytes = downloadDocumentBinary(uploaded._id); + const originalBytes = downloadFile(uploaded._id, "", "binary"); const expectedDigest = originalBytes.status === 200 ? crypto.sha256(originalBytes.body as ArrayBuffer, "hex") : null; @@ -183,7 +182,7 @@ export function testSpecialCharacterNames(data: StorageInitData) { console.error(`Copy of "${name}" failed: ${copyRes.status} - ${copyRes.body}`); continue; } - const copyBytes = downloadDocumentBinary(copiedIds(copyRes)[0]); + const copyBytes = downloadFile(copiedIds(copyRes)[0], "", "binary"); check(copyBytes, { [`the copy of "${name}" should hold the same bytes`]: (r) => r.status === 200 && expectedDigest != null && @@ -213,7 +212,9 @@ export function testSpecialCharacterNamesSurviveExport(data: StorageInitData) { let lastStatus = 0; while (!exportReady && (Date.now() - startTime) < EXPORT_TIMEOUT) { attempts++; - const verifyRes = verifyExport(exportId); + // An explicit, short per-request timeout: the k6 default is 60s, so a stalled export would be + // polled once a minute in silence. An export that is merely not ready answers immediately. + const verifyRes = verifyExportFiles(exportId, "10s"); lastStatus = verifyRes.status; if (verifyRes.status === 200) { exportReady = true; diff --git a/tests/src/test/js/pnpm-lock.yaml b/tests/src/test/js/pnpm-lock.yaml index e2f6689a76..278e7d04ea 100644 --- a/tests/src/test/js/pnpm-lock.yaml +++ b/tests/src/test/js/pnpm-lock.yaml @@ -12,20 +12,20 @@ importers: specifier: ^0.54.2 version: 0.54.2 edifice-k6-commons: - specifier: 2.1.6-develop-b2school-3 - version: 2.1.6-develop-b2school-3 + specifier: develop + version: 2.1.14-develop.2 packages: '@types/k6@0.54.2': resolution: {integrity: sha512-B5LPxeQm97JnUTpoKNE1UX9jFp+JiJCAXgZOa2P7aChxVoPQXKfWMzK+739xHq3lPkKj1aV+HeOxkP56g/oWBg==} - edifice-k6-commons@2.1.6-develop-b2school-3: - resolution: {integrity: sha512-H4qJMdtIdR025qWy45/T5IsN2SJU1N/GmLXiSqXfbt96rylJoeA7YysZLJAU6vW3kwJBHYjenn5rwCHWkwVMlA==} + edifice-k6-commons@2.1.14-develop.2: + resolution: {integrity: sha512-smomxKkRG/2A9nN5UVz6c0QKl7vTj/V/DQEQnZYA1l2QANOegVFQcEP74LvoBcqsXjDMvSdx7n6g7SsUpIkXpg==} engines: {node: '>=18'} snapshots: '@types/k6@0.54.2': {} - edifice-k6-commons@2.1.6-develop-b2school-3: {} + edifice-k6-commons@2.1.14-develop.2: {}