From aa20582762d94017a1f5350209abbbafe4138775 Mon Sep 17 00:00:00 2001 From: yyh Date: Tue, 30 Jun 2026 17:10:55 +0800 Subject: [PATCH 001/185] test(e2e): add agent v2 core coverage --- e2e/features/agent-v2/access-point.feature | 16 +++ e2e/features/agent-v2/build-draft.feature | 16 +++ .../agent-v2/configure-persistence.feature | 10 ++ .../agent-v2/configure-validation.feature | 9 ++ e2e/features/agent-v2/publish.feature | 9 ++ .../agent-v2/access-point.steps.ts | 79 ++++++++++++++ .../agent-v2/configure.steps.ts | 101 ++++++++++++++++-- .../common/navigation.steps.ts | 4 + e2e/features/support/world.ts | 2 + e2e/support/agent.ts | 43 +++++++- 10 files changed, 277 insertions(+), 12 deletions(-) create mode 100644 e2e/features/agent-v2/access-point.feature create mode 100644 e2e/features/agent-v2/build-draft.feature create mode 100644 e2e/features/agent-v2/configure-persistence.feature create mode 100644 e2e/features/agent-v2/configure-validation.feature create mode 100644 e2e/features/agent-v2/publish.feature create mode 100644 e2e/features/step-definitions/agent-v2/access-point.steps.ts diff --git a/e2e/features/agent-v2/access-point.feature b/e2e/features/agent-v2/access-point.feature new file mode 100644 index 00000000000000..0e2f4c12fda0fe --- /dev/null +++ b/e2e/features/agent-v2/access-point.feature @@ -0,0 +1,16 @@ +@agent-v2 @authenticated @access-point @core +Feature: Agent v2 Access Point + Scenario: Access Point shows the available Agent v2 access surfaces + Given I am signed in as the default E2E admin + And an Agent v2 test agent has been created via API + When I open the Agent v2 Access Point page + Then I should see the Agent v2 Access Point overview + + Scenario: Backend service API shows endpoint, key management, and API reference entry + Given I am signed in as the default E2E admin + And an Agent v2 test agent has been created via API + And Agent v2 Backend service API access has been enabled via API + When I open the Agent v2 Access Point page + Then I should see the Agent v2 Backend service API endpoint + And I should see the Agent v2 API Reference entry + And I should be able to open Agent v2 API key management without exposing a secret by default diff --git a/e2e/features/agent-v2/build-draft.feature b/e2e/features/agent-v2/build-draft.feature new file mode 100644 index 00000000000000..ab4b2d72bd97d5 --- /dev/null +++ b/e2e/features/agent-v2/build-draft.feature @@ -0,0 +1,16 @@ +@agent-v2 @authenticated @build @core +Feature: Agent v2 build draft + Scenario: Discarding a Build draft keeps the original Agent configuration + Given I am signed in as the default E2E admin + And an Agent v2 test agent has been created via API + And the Agent v2 composer draft uses the normal E2E prompt + And an Agent v2 Build draft uses the updated E2E prompt + When I open the Agent v2 configure page + Then I should see the Agent v2 Build draft pending changes + And I should see the updated E2E prompt in the Agent v2 prompt editor + When I discard the Agent v2 Build draft + Then I should see the normal E2E prompt in the Agent v2 prompt editor + And the Agent v2 Build draft should no longer be active + When I refresh the current page + Then I should see the normal E2E prompt in the Agent v2 prompt editor + And the Agent v2 Build draft should no longer be active diff --git a/e2e/features/agent-v2/configure-persistence.feature b/e2e/features/agent-v2/configure-persistence.feature new file mode 100644 index 00000000000000..0dc4edf5d91fb8 --- /dev/null +++ b/e2e/features/agent-v2/configure-persistence.feature @@ -0,0 +1,10 @@ +@agent-v2 @authenticated @core +Feature: Agent v2 configure persistence + Scenario: Persisted Agent v2 instructions remain visible after refresh + Given I am signed in as the default E2E admin + And an Agent v2 test agent has been created via API + And the Agent v2 composer draft uses the normal E2E prompt + When I open the Agent v2 configure page + Then I should see the normal E2E prompt in the Agent v2 prompt editor + When I refresh the current page + Then I should see the normal E2E prompt in the Agent v2 prompt editor diff --git a/e2e/features/agent-v2/configure-validation.feature b/e2e/features/agent-v2/configure-validation.feature new file mode 100644 index 00000000000000..e053b38fe41e8f --- /dev/null +++ b/e2e/features/agent-v2/configure-validation.feature @@ -0,0 +1,9 @@ +@agent-v2 @authenticated @core +Feature: Agent v2 configure validation + Scenario: Preview is unavailable until a required model is configured + Given I am signed in as the default E2E admin + And an Agent v2 test agent has been created via API + And the Agent v2 composer draft uses the normal E2E prompt + When I open the Agent v2 configure page + Then Agent v2 Preview should be unavailable until a model is configured + And I should see the normal E2E prompt in the Agent v2 prompt editor diff --git a/e2e/features/agent-v2/publish.feature b/e2e/features/agent-v2/publish.feature new file mode 100644 index 00000000000000..78bd2218356a15 --- /dev/null +++ b/e2e/features/agent-v2/publish.feature @@ -0,0 +1,9 @@ +@agent-v2 @authenticated @publish @core +Feature: Agent v2 publish + Scenario: Publish a configured Agent v2 draft + Given I am signed in as the default E2E admin + And an Agent v2 test agent has been created via API + And the Agent v2 composer draft uses the normal E2E prompt + When I open the Agent v2 configure page + And I publish the Agent v2 draft + Then the Agent v2 draft should be published and up to date diff --git a/e2e/features/step-definitions/agent-v2/access-point.steps.ts b/e2e/features/step-definitions/agent-v2/access-point.steps.ts new file mode 100644 index 00000000000000..9591911db22b4b --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/access-point.steps.ts @@ -0,0 +1,79 @@ +import type { DifyWorld } from '../../support/world' +import { Given, Then, When } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { getAgentAccessPath, setAgentApiAccess } from '../../../support/agent' + +const getCurrentAgentId = (world: DifyWorld) => { + const agentId = world.createdAgentIds.at(-1) + if (!agentId) + throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') + + return agentId +} + +Given( + 'Agent v2 Backend service API access has been enabled via API', + async function (this: DifyWorld) { + const apiAccess = await setAgentApiAccess(getCurrentAgentId(this), true) + + this.lastAgentServiceApiBaseURL = apiAccess.service_api_base_url + }, +) + +When('I open the Agent v2 Access Point page', async function (this: DifyWorld) { + await this.getPage().goto(getAgentAccessPath(getCurrentAgentId(this))) +}) + +Then('I should see the Agent v2 Access Point overview', async function (this: DifyWorld) { + const page = this.getPage() + const accessRegion = page.getByRole('region', { name: 'Access Point' }) + + await expect(accessRegion).toBeVisible({ timeout: 30_000 }) + await expect(accessRegion.getByRole('heading', { name: 'Access Point' })).toBeVisible() + await expect(accessRegion.getByRole('heading', { name: 'Web app' })).toBeVisible() + await expect(accessRegion.getByRole('heading', { name: 'Backend service API' })).toBeVisible() + await expect(accessRegion.getByRole('heading', { name: 'Workflow access' })).toBeVisible() + await expect(accessRegion.getByRole('columnheader', { name: 'Name' })).toBeVisible() + await expect(accessRegion.getByRole('columnheader', { name: 'Version' })).toBeVisible() + await expect(accessRegion.getByRole('columnheader', { name: 'Nodes' })).toBeVisible() + await expect(accessRegion.getByRole('columnheader', { name: 'Last updated' })).toBeVisible() + await expect(accessRegion.getByRole('columnheader', { name: 'Actions' })).toBeVisible() + await expect(accessRegion.getByText('No workflow references yet.')).toBeVisible() +}) + +Then('I should see the Agent v2 Backend service API endpoint', async function (this: DifyWorld) { + const page = this.getPage() + + if (!this.lastAgentServiceApiBaseURL) + throw new Error('No Agent v2 service API endpoint found. Enable Backend service API first.') + + await expect(page.getByRole('heading', { name: 'Backend service API' })).toBeVisible({ + timeout: 30_000, + }) + await expect(page.getByText('Service API Endpoint')).toBeVisible() + await expect(page.getByText(this.lastAgentServiceApiBaseURL)).toBeVisible() + await expect(page.getByLabel('Copy service API endpoint')).toBeEnabled() +}) + +Then( + 'I should be able to open Agent v2 API key management without exposing a secret by default', + async function (this: DifyWorld) { + const page = this.getPage() + + await page.getByRole('button', { name: /^API Key\b/ }).click() + const dialog = page.getByRole('dialog', { name: /API Secret key/i }) + + await expect(dialog).toBeVisible() + await expect(dialog.getByText('Secret Key', { exact: true })).toBeVisible() + await expect(dialog.getByText('CREATED', { exact: true })).toBeVisible() + await expect(dialog.getByText('LAST USED', { exact: true })).toBeVisible() + await expect(dialog.getByText('No data', { exact: true })).toBeVisible() + await expect(dialog.getByRole('button', { name: 'Create new Secret key' })).toBeVisible() + await expect(dialog.getByText(/^app-/)).not.toBeVisible() + await expect(page.getByRole('dialog', { name: 'Internal Server Error' })).not.toBeVisible() + }, +) + +Then('I should see the Agent v2 API Reference entry', async function (this: DifyWorld) { + await expect(this.getPage().getByRole('link', { name: 'API Reference' })).toBeVisible() +}) diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index cbb49fef6851ce..acaaf7e0bb02b7 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -4,9 +4,23 @@ import { expect } from '@playwright/test' import { createTestAgent, getAgentConfigurePath, + getTestAgent, + normalAgentPrompt, + normalAgentSoulConfig, + saveAgentBuildDraft, saveAgentComposerDraft, + updatedAgentPrompt, + updatedAgentSoulConfig, } from '../../../support/agent' +const getCurrentAgentId = (world: DifyWorld) => { + const agentId = world.createdAgentIds.at(-1) + if (!agentId) + throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') + + return agentId +} + Given('an Agent v2 test agent has been created via API', async function (this: DifyWorld) { const agent = await createTestAgent() this.createdAgentIds.push(agent.id) @@ -15,25 +29,37 @@ Given('an Agent v2 test agent has been created via API', async function (this: D }) Given('a minimal Agent v2 composer draft has been synced', async function (this: DifyWorld) { - const agentId = this.createdAgentIds.at(-1) - if (!agentId) - throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') + const agentId = getCurrentAgentId(this) await saveAgentComposerDraft(agentId) }) +Given('the Agent v2 composer draft uses the normal E2E prompt', async function (this: DifyWorld) { + await saveAgentComposerDraft(getCurrentAgentId(this), normalAgentSoulConfig) +}) + +Given('an Agent v2 Build draft uses the updated E2E prompt', async function (this: DifyWorld) { + await saveAgentBuildDraft(getCurrentAgentId(this), updatedAgentSoulConfig) +}) + When('I open the Agent v2 configure page', async function (this: DifyWorld) { - const agentId = this.createdAgentIds.at(-1) - if (!agentId) - throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') + await this.getPage().goto(getAgentConfigurePath(getCurrentAgentId(this))) +}) - await this.getPage().goto(getAgentConfigurePath(agentId)) +When('I discard the Agent v2 Build draft', async function (this: DifyWorld) { + await this.getPage().getByRole('button', { name: 'Discard' }).click() +}) + +When('I publish the Agent v2 draft', async function (this: DifyWorld) { + const page = this.getPage() + const publishButton = page.getByRole('button', { name: /^Publish(?: update)?$/ }) + + await expect(publishButton).toBeEnabled({ timeout: 30_000 }) + await publishButton.click() }) Then('I should be on the Agent v2 configure page', async function (this: DifyWorld) { - const agentId = this.createdAgentIds.at(-1) - if (!agentId) - throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') + const agentId = getCurrentAgentId(this) await expect(this.getPage()).toHaveURL( new RegExp(`/roster/agent/${agentId}/configure(?:\\?.*)?$`), @@ -47,3 +73,58 @@ Then('I should see the Agent v2 configure workspace', async function (this: Dify await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible() await expect(page.getByText(this.lastCreatedAgentName!)).toBeVisible() }) + +Then( + 'I should see the normal E2E prompt in the Agent v2 prompt editor', + async function (this: DifyWorld) { + const page = this.getPage() + + await expect(page.getByRole('heading', { name: 'Prompt' })).toBeVisible({ timeout: 30_000 }) + await expect(page.getByText(normalAgentPrompt)).toBeVisible() + }, +) + +Then( + 'I should see the updated E2E prompt in the Agent v2 prompt editor', + async function (this: DifyWorld) { + const page = this.getPage() + + await expect(page.getByRole('heading', { name: 'Prompt' })).toBeVisible({ timeout: 30_000 }) + await expect(page.getByText(updatedAgentPrompt)).toBeVisible() + }, +) + +Then( + 'Agent v2 Preview should be unavailable until a model is configured', + async function (this: DifyWorld) { + const page = this.getPage() + + await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) + await expect(page.getByRole('button', { name: /^Preview$/i })).toBeDisabled() + }, +) + +Then('I should see the Agent v2 Build draft pending changes', async function (this: DifyWorld) { + const page = this.getPage() + + await expect(page.getByText('Build draft')).toBeVisible({ timeout: 30_000 }) + await expect(page.getByRole('button', { name: 'Apply' })).toBeVisible() + await expect(page.getByRole('button', { name: 'Discard' })).toBeVisible() +}) + +Then('the Agent v2 Build draft should no longer be active', async function (this: DifyWorld) { + const page = this.getPage() + + await expect(page.getByText('Build draft')).not.toBeVisible() + await expect(page.getByRole('button', { name: 'Apply' })).not.toBeVisible() + await expect(page.getByRole('button', { name: 'Discard' })).not.toBeVisible() +}) + +Then('the Agent v2 draft should be published and up to date', async function (this: DifyWorld) { + const page = this.getPage() + const agentId = getCurrentAgentId(this) + + await expect(page.getByRole('button', { name: 'Published' })).toBeVisible({ timeout: 30_000 }) + await expect(page.getByText('Up to date')).toBeVisible() + await expect.poll(async () => (await getTestAgent(agentId)).active_config_is_published).toBe(true) +}) diff --git a/e2e/features/step-definitions/common/navigation.steps.ts b/e2e/features/step-definitions/common/navigation.steps.ts index 02b860b372b779..a558d96f937c22 100644 --- a/e2e/features/step-definitions/common/navigation.steps.ts +++ b/e2e/features/step-definitions/common/navigation.steps.ts @@ -7,6 +7,10 @@ When('I open the apps console', async function (this: DifyWorld) { await this.getPage().goto('/apps') }) +When('I refresh the current page', async function (this: DifyWorld) { + await this.getPage().reload() +}) + Then('I should stay on the apps console', async function (this: DifyWorld) { await waitForAppsConsole(this.getPage()) }) diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index 6faf740a7c9c34..4ba1ec2d2482df 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -15,6 +15,7 @@ export class DifyWorld extends World { lastCreatedAppName: string | undefined lastCreatedAgentName: string | undefined lastCreatedAgentRole: string | undefined + lastAgentServiceApiBaseURL: string | undefined createdAppIds: string[] = [] createdAgentIds: string[] = [] capturedDownloads: Download[] = [] @@ -31,6 +32,7 @@ export class DifyWorld extends World { this.lastCreatedAppName = undefined this.lastCreatedAgentName = undefined this.lastCreatedAgentRole = undefined + this.lastAgentServiceApiBaseURL = undefined this.createdAppIds = [] this.createdAgentIds = [] this.capturedDownloads = [] diff --git a/e2e/support/agent.ts b/e2e/support/agent.ts index 445c10b5cfc462..168f94eccf09b9 100644 --- a/e2e/support/agent.ts +++ b/e2e/support/agent.ts @@ -1,6 +1,7 @@ import { createApiContext, expectApiResponseOK, setAppSiteEnabled } from './api' export type AgentSeed = { + active_config_is_published?: boolean app_id?: string backing_app_id?: string description?: string @@ -29,10 +30,9 @@ export type AgentBuildDraftResponse = { export type AgentApiAccess = { api_key_count: number - api_reference_url: string - endpoint: string enabled: boolean files_upload_endpoint: string + service_api_base_url: string } export type AgentApiKey = { @@ -46,6 +46,24 @@ export const defaultAgentSoulConfig: AgentSoulConfig = { }, } +export const normalAgentPrompt + = 'You are a Dify Agent E2E test assistant. Reply briefly to every user message, and always include AGENT_E2E_PASS in your response.' + +export const updatedAgentPrompt + = 'You are a Dify Agent E2E test assistant. Every response must start with E2E_AGENT_UPDATED.' + +export const normalAgentSoulConfig: AgentSoulConfig = { + prompt: { + system_prompt: normalAgentPrompt, + }, +} + +export const updatedAgentSoulConfig: AgentSoulConfig = { + prompt: { + system_prompt: updatedAgentPrompt, + }, +} + export const getAgentConfigurePath = (agentId: string) => `/roster/agent/${agentId}/configure` export const getAgentAccessPath = (agentId: string) => `/roster/agent/${agentId}/access` @@ -136,6 +154,27 @@ export async function checkoutAgentBuildDraft(agentId: string): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.put(`/console/api/agent/${agentId}/build-draft`, { + data: { + agent_soul: agentSoul, + save_strategy: 'save_to_current_version', + variant: 'agent_app', + }, + }) + await expectApiResponseOK(response, `Save Agent v2 build draft for ${agentId}`) + return (await response.json()) as AgentBuildDraftResponse + } + finally { + await ctx.dispose() + } +} + export async function discardAgentBuildDraft(agentId: string): Promise { const ctx = await createApiContext() try { From 704f007c2f5ea3a30b998a6d7d65084760c42eac Mon Sep 17 00:00:00 2001 From: yyh Date: Tue, 30 Jun 2026 17:50:42 +0800 Subject: [PATCH 002/185] test(e2e): cover agent v2 service api access flow --- e2e/features/agent-v2/access-point.feature | 18 ++- .../agent-v2/access-point.steps.ts | 128 +++++++++++++++--- .../agent-v2/configure.steps.ts | 16 +++ e2e/features/support/world.ts | 4 + 4 files changed, 146 insertions(+), 20 deletions(-) diff --git a/e2e/features/agent-v2/access-point.feature b/e2e/features/agent-v2/access-point.feature index 0e2f4c12fda0fe..62fcc076a1e3e1 100644 --- a/e2e/features/agent-v2/access-point.feature +++ b/e2e/features/agent-v2/access-point.feature @@ -6,11 +6,21 @@ Feature: Agent v2 Access Point When I open the Agent v2 Access Point page Then I should see the Agent v2 Access Point overview - Scenario: Backend service API shows endpoint, key management, and API reference entry + Scenario: Backend service API supports endpoint copy, key creation, and API reference navigation Given I am signed in as the default E2E admin And an Agent v2 test agent has been created via API And Agent v2 Backend service API access has been enabled via API - When I open the Agent v2 Access Point page + When I open the Agent v2 configure page from the Agent Roster + And I switch to the Agent v2 Access Point section Then I should see the Agent v2 Backend service API endpoint - And I should see the Agent v2 API Reference entry - And I should be able to open Agent v2 API key management without exposing a secret by default + When I copy the Agent v2 Backend service API endpoint + Then the Agent v2 Backend service API endpoint should show it was copied + When I open Agent v2 API key management + Then Agent v2 API keys should not expose a secret by default + When I create a new Agent v2 API key + Then I should see the newly generated Agent v2 API key once + When I close the newly generated Agent v2 API key + Then the Agent v2 API key list should not expose the full generated secret + When I close Agent v2 API key management + And I open the Agent v2 API Reference + Then the Agent v2 API Reference should open in a new tab diff --git a/e2e/features/step-definitions/agent-v2/access-point.steps.ts b/e2e/features/step-definitions/agent-v2/access-point.steps.ts index 9591911db22b4b..405f8f06fca8ba 100644 --- a/e2e/features/step-definitions/agent-v2/access-point.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point.steps.ts @@ -24,6 +24,15 @@ When('I open the Agent v2 Access Point page', async function (this: DifyWorld) { await this.getPage().goto(getAgentAccessPath(getCurrentAgentId(this))) }) +When('I switch to the Agent v2 Access Point section', async function (this: DifyWorld) { + const page = this.getPage() + const agentId = getCurrentAgentId(this) + + await page.getByRole('link', { name: 'Access Point' }).click() + await expect(page).toHaveURL(new RegExp(`/roster/agent/${agentId}/access(?:\\?.*)?$`)) + await expect(page.getByRole('region', { name: 'Access Point' })).toBeVisible() +}) + Then('I should see the Agent v2 Access Point overview', async function (this: DifyWorld) { const page = this.getPage() const accessRegion = page.getByRole('region', { name: 'Access Point' }) @@ -55,25 +64,112 @@ Then('I should see the Agent v2 Backend service API endpoint', async function (t await expect(page.getByLabel('Copy service API endpoint')).toBeEnabled() }) +When('I copy the Agent v2 Backend service API endpoint', async function (this: DifyWorld) { + await this.getPage().getByLabel('Copy service API endpoint').click() +}) + +Then( + 'the Agent v2 Backend service API endpoint should show it was copied', + async function (this: DifyWorld) { + await expect(this.getPage().getByLabel('Copied')).toBeVisible() + }, +) + +When('I open Agent v2 API key management', async function (this: DifyWorld) { + await this.getPage() + .getByRole('button', { name: /^API Key\b/ }) + .click() +}) + +Then('Agent v2 API keys should not expose a secret by default', async function (this: DifyWorld) { + const page = this.getPage() + const dialog = page.getByRole('dialog', { name: /API Secret key/i }) + + await expect(dialog).toBeVisible() + await expect(dialog.getByText('Secret Key', { exact: true })).toBeVisible() + await expect(dialog.getByText('CREATED', { exact: true })).toBeVisible() + await expect(dialog.getByText('LAST USED', { exact: true })).toBeVisible() + await expect(dialog.getByText('No data', { exact: true })).toBeVisible() + await expect(dialog.getByRole('button', { name: 'Create new Secret key' })).toBeVisible() + await expect(dialog.getByText(/^app-/)).not.toBeVisible() + await expect(page.getByRole('dialog', { name: 'Internal Server Error' })).not.toBeVisible() +}) + +When('I create a new Agent v2 API key', async function (this: DifyWorld) { + const dialog = this.getPage().getByRole('dialog', { name: /API Secret key/i }) + + await dialog.getByRole('button', { name: 'Create new Secret key' }).click() +}) + +Then('I should see the newly generated Agent v2 API key once', async function (this: DifyWorld) { + const generatedKeyDialog = this.getPage() + .getByRole('dialog', { name: /API Secret key/i }) + .last() + const generatedKey = generatedKeyDialog.getByText(/^app-/) + + await expect(generatedKeyDialog).toBeVisible() + await expect( + generatedKeyDialog.getByText('Keep this key in a secure and accessible place.'), + ).toBeVisible() + await expect(generatedKey).toBeVisible() + await expect(generatedKeyDialog.getByLabel('Copy')).toBeVisible() + + this.lastGeneratedAgentApiKey = (await generatedKey.textContent())?.trim() + if (!this.lastGeneratedAgentApiKey) + throw new Error('Generated Agent v2 API key was empty.') +}) + +When('I close the newly generated Agent v2 API key', async function (this: DifyWorld) { + const page = this.getPage() + const generatedKeyDialog = page.getByRole('dialog', { name: /API Secret key/i }).last() + + await generatedKeyDialog.getByRole('button', { name: 'OK' }).click() + await expect(page.getByText('Keep this key in a secure and accessible place.')).not.toBeVisible() +}) + Then( - 'I should be able to open Agent v2 API key management without exposing a secret by default', + 'the Agent v2 API key list should not expose the full generated secret', async function (this: DifyWorld) { - const page = this.getPage() - - await page.getByRole('button', { name: /^API Key\b/ }).click() - const dialog = page.getByRole('dialog', { name: /API Secret key/i }) - - await expect(dialog).toBeVisible() - await expect(dialog.getByText('Secret Key', { exact: true })).toBeVisible() - await expect(dialog.getByText('CREATED', { exact: true })).toBeVisible() - await expect(dialog.getByText('LAST USED', { exact: true })).toBeVisible() - await expect(dialog.getByText('No data', { exact: true })).toBeVisible() - await expect(dialog.getByRole('button', { name: 'Create new Secret key' })).toBeVisible() - await expect(dialog.getByText(/^app-/)).not.toBeVisible() - await expect(page.getByRole('dialog', { name: 'Internal Server Error' })).not.toBeVisible() + const fullSecret = this.lastGeneratedAgentApiKey + if (!fullSecret) + throw new Error('No generated Agent v2 API key found.') + + const apiKeyDialog = this.getPage().getByRole('dialog', { name: /API Secret key/i }) + + await expect(apiKeyDialog).toBeVisible() + await expect(apiKeyDialog.getByText(fullSecret, { exact: true })).not.toBeVisible() + await expect(apiKeyDialog.getByText(/^app-/)).not.toBeVisible() + await expect(apiKeyDialog.getByLabel('Copy')).toBeVisible() }, ) -Then('I should see the Agent v2 API Reference entry', async function (this: DifyWorld) { - await expect(this.getPage().getByRole('link', { name: 'API Reference' })).toBeVisible() +When('I close Agent v2 API key management', async function (this: DifyWorld) { + const apiKeyDialog = this.getPage().getByRole('dialog', { name: /API Secret key/i }) + + await apiKeyDialog.getByLabel('Close').click() + await expect(apiKeyDialog).not.toBeVisible() +}) + +When('I open the Agent v2 API Reference', async function (this: DifyWorld) { + const page = this.getPage() + const apiReferenceLink = page.getByRole('link', { name: 'API Reference' }) + + await expect(apiReferenceLink).toBeVisible() + + const [apiReferencePage] = await Promise.all([ + page.waitForEvent('popup'), + apiReferenceLink.click(), + ]) + + this.lastAgentApiReferencePage = apiReferencePage +}) + +Then('the Agent v2 API Reference should open in a new tab', async function (this: DifyWorld) { + const apiReferencePage = this.lastAgentApiReferencePage + if (!apiReferencePage) + throw new Error('No Agent v2 API Reference page was opened.') + + await expect(apiReferencePage).toHaveURL(/developing-with-apis/) + await apiReferencePage.close() + this.lastAgentApiReferencePage = undefined }) diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index acaaf7e0bb02b7..df60347c6b555f 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -46,6 +46,22 @@ When('I open the Agent v2 configure page', async function (this: DifyWorld) { await this.getPage().goto(getAgentConfigurePath(getCurrentAgentId(this))) }) +When( + 'I open the Agent v2 configure page from the Agent Roster', + async function (this: DifyWorld) { + const page = this.getPage() + const agentId = getCurrentAgentId(this) + const agentName = this.lastCreatedAgentName + if (!agentName) + throw new Error('No Agent v2 name found. Create an Agent v2 test agent first.') + + await page.goto('/roster') + await page.getByRole('link', { name: agentName }).click() + await expect(page).toHaveURL(new RegExp(`/roster/agent/${agentId}/configure(?:\\?.*)?$`)) + await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) + }, +) + When('I discard the Agent v2 Build draft', async function (this: DifyWorld) { await this.getPage().getByRole('button', { name: 'Discard' }).click() }) diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index 4ba1ec2d2482df..e79a58281e5eaf 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -16,6 +16,8 @@ export class DifyWorld extends World { lastCreatedAgentName: string | undefined lastCreatedAgentRole: string | undefined lastAgentServiceApiBaseURL: string | undefined + lastGeneratedAgentApiKey: string | undefined + lastAgentApiReferencePage: Page | undefined createdAppIds: string[] = [] createdAgentIds: string[] = [] capturedDownloads: Download[] = [] @@ -33,6 +35,8 @@ export class DifyWorld extends World { this.lastCreatedAgentName = undefined this.lastCreatedAgentRole = undefined this.lastAgentServiceApiBaseURL = undefined + this.lastGeneratedAgentApiKey = undefined + this.lastAgentApiReferencePage = undefined this.createdAppIds = [] this.createdAgentIds = [] this.capturedDownloads = [] From 5f10accb442f882ac992b63bec7776f0c28c0866 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 11:39:31 +0800 Subject: [PATCH 003/185] test(e2e): centralize resource naming --- .../step-definitions/apps/create-app.steps.ts | 3 ++- .../step-definitions/apps/duplicate-app.steps.ts | 3 ++- .../step-definitions/apps/share-app.steps.ts | 3 ++- .../apps/switch-app-mode.steps.ts | 3 ++- e2e/features/step-definitions/common/app.steps.ts | 3 ++- e2e/support/agent.ts | 15 +++++++++------ e2e/support/api.ts | 6 +++++- e2e/support/naming.ts | 4 ++++ 8 files changed, 28 insertions(+), 12 deletions(-) create mode 100644 e2e/support/naming.ts diff --git a/e2e/features/step-definitions/apps/create-app.steps.ts b/e2e/features/step-definitions/apps/create-app.steps.ts index d6a5eb21d54663..d6fd5bd3729354 100644 --- a/e2e/features/step-definitions/apps/create-app.steps.ts +++ b/e2e/features/step-definitions/apps/create-app.steps.ts @@ -2,13 +2,14 @@ import type { DifyWorld } from '../../support/world' import { Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { openBlankAppCreation } from '../../../support/apps' +import { createE2EResourceName } from '../../../support/naming' When('I start creating a blank app', async function (this: DifyWorld) { await openBlankAppCreation(this.getPage()) }) When('I enter a unique E2E app name', async function (this: DifyWorld) { - const appName = `E2E App ${Date.now()}` + const appName = createE2EResourceName('App') this.lastCreatedAppName = appName await this.getPage().getByPlaceholder('Give your app a name').fill(appName) }) diff --git a/e2e/features/step-definitions/apps/duplicate-app.steps.ts b/e2e/features/step-definitions/apps/duplicate-app.steps.ts index 5e998b3603ee8a..aa8c06f5df8f02 100644 --- a/e2e/features/step-definitions/apps/duplicate-app.steps.ts +++ b/e2e/features/step-definitions/apps/duplicate-app.steps.ts @@ -2,9 +2,10 @@ import type { DifyWorld } from '../../support/world' import { Given, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { createTestApp } from '../../../support/api' +import { createE2EResourceName } from '../../../support/naming' Given('there is an existing E2E app available for testing', async function (this: DifyWorld) { - const name = `E2E Test App ${Date.now()}` + const name = createE2EResourceName('App', 'Test') const app = await createTestApp(name, 'completion') this.lastCreatedAppName = app.name this.createdAppIds.push(app.id) diff --git a/e2e/features/step-definitions/apps/share-app.steps.ts b/e2e/features/step-definitions/apps/share-app.steps.ts index c7acc91ebe5d1e..20d9c05522e78d 100644 --- a/e2e/features/step-definitions/apps/share-app.steps.ts +++ b/e2e/features/step-definitions/apps/share-app.steps.ts @@ -7,6 +7,7 @@ import { publishWorkflowApp, syncRunnableWorkflowDraft, } from '../../../support/api' +import { createE2EResourceName } from '../../../support/naming' When('I enable the Web App share', async function (this: DifyWorld) { const page = this.getPage() @@ -27,7 +28,7 @@ Then('the Web App should be in service', async function (this: DifyWorld) { }) Given('a workflow app has been published and shared via API', async function (this: DifyWorld) { - const app = await createTestApp(`E2E Share ${Date.now()}`, 'workflow') + const app = await createTestApp(createE2EResourceName('App', 'Share'), 'workflow') this.createdAppIds.push(app.id) this.lastCreatedAppName = app.name await syncRunnableWorkflowDraft(app.id) diff --git a/e2e/features/step-definitions/apps/switch-app-mode.steps.ts b/e2e/features/step-definitions/apps/switch-app-mode.steps.ts index 55ad1ab02c93dc..cfaf21a66bde54 100644 --- a/e2e/features/step-definitions/apps/switch-app-mode.steps.ts +++ b/e2e/features/step-definitions/apps/switch-app-mode.steps.ts @@ -2,11 +2,12 @@ import type { DifyWorld } from '../../support/world' import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { createTestApp } from '../../../support/api' +import { createE2EResourceName } from '../../../support/naming' Given( 'there is an existing E2E completion app available for testing', async function (this: DifyWorld) { - const name = `E2E Test App ${Date.now()}` + const name = createE2EResourceName('App', 'Test') const app = await createTestApp(name, 'completion') this.lastCreatedAppName = app.name this.createdAppIds.push(app.id) diff --git a/e2e/features/step-definitions/common/app.steps.ts b/e2e/features/step-definitions/common/app.steps.ts index 6deca22c609ab8..1fa50cbf0d1678 100644 --- a/e2e/features/step-definitions/common/app.steps.ts +++ b/e2e/features/step-definitions/common/app.steps.ts @@ -3,9 +3,10 @@ import { Given, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { createTestApp, syncMinimalWorkflowDraft } from '../../../support/api' import { waitForAppsConsole } from '../../../support/apps' +import { createE2EResourceName } from '../../../support/naming' Given('a {string} app has been created via API', async function (this: DifyWorld, mode: string) { - const app = await createTestApp(`E2E ${Date.now()}`, mode) + const app = await createTestApp(createE2EResourceName('App', mode), mode) this.createdAppIds.push(app.id) this.lastCreatedAppName = app.name }) diff --git a/e2e/support/agent.ts b/e2e/support/agent.ts index 168f94eccf09b9..9372f8453dec2a 100644 --- a/e2e/support/agent.ts +++ b/e2e/support/agent.ts @@ -1,4 +1,5 @@ import { createApiContext, expectApiResponseOK, setAppSiteEnabled } from './api' +import { createE2EResourceName } from './naming' export type AgentSeed = { active_config_is_published?: boolean @@ -40,6 +41,12 @@ export type AgentApiKey = { token?: string } +export type CreateTestAgentOptions = { + description?: string + name?: string + role?: string +} + export const defaultAgentSoulConfig: AgentSoulConfig = { prompt: { system_prompt: 'You are a Dify Agent E2E test assistant.', @@ -69,13 +76,9 @@ export const getAgentAccessPath = (agentId: string) => `/roster/agent/${agentId} export async function createTestAgent({ description = 'Created by Dify E2E.', - name = `E2E Agent ${Date.now()}`, + name = createE2EResourceName('Agent'), role = 'E2E test assistant', -}: { - description?: string - name?: string - role?: string -} = {}): Promise { +}: CreateTestAgentOptions = {}): Promise { const ctx = await createApiContext() try { const response = await ctx.post('/console/api/agent', { diff --git a/e2e/support/api.ts b/e2e/support/api.ts index 81773dea595719..f03de5f349b5fb 100644 --- a/e2e/support/api.ts +++ b/e2e/support/api.ts @@ -3,6 +3,7 @@ import { readFile } from 'node:fs/promises' import { request } from '@playwright/test' import { authStatePath } from '../fixtures/auth' import { apiURL } from '../test-env' +import { createE2EResourceName } from './naming' type StorageState = { cookies: Array<{ name: string, value: string }> @@ -32,7 +33,10 @@ export type AppSeed = { name: string } -export async function createTestApp(name: string, mode = 'workflow'): Promise { +export async function createTestApp( + name = createE2EResourceName('App'), + mode = 'workflow', +): Promise { const ctx = await createApiContext() try { const response = await ctx.post('/console/api/apps', { diff --git a/e2e/support/naming.ts b/e2e/support/naming.ts new file mode 100644 index 00000000000000..7a973c204af035 --- /dev/null +++ b/e2e/support/naming.ts @@ -0,0 +1,4 @@ +export const createE2EResourceName = (resource: string, qualifier?: string) => { + const nonce = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}` + return ['E2E', qualifier, resource, nonce].filter(Boolean).join(' ') +} From b9b1c683f8575413093b6ca1d070bf7a9dbfa6e5 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 11:39:49 +0800 Subject: [PATCH 004/185] test(e2e): add basic configured agent fixture --- .../step-definitions/agent-v2/configure.steps.ts | 11 +++++++++++ e2e/support/agent.ts | 12 ++++++++++++ 2 files changed, 23 insertions(+) diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index df60347c6b555f..80887dcd96968d 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -2,6 +2,7 @@ import type { DifyWorld } from '../../support/world' import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { + createConfiguredTestAgent, createTestAgent, getAgentConfigurePath, getTestAgent, @@ -28,6 +29,16 @@ Given('an Agent v2 test agent has been created via API', async function (this: D this.lastCreatedAgentRole = agent.role }) +Given( + 'a basic configured Agent v2 test agent has been created via API', + async function (this: DifyWorld) { + const agent = await createConfiguredTestAgent() + this.createdAgentIds.push(agent.id) + this.lastCreatedAgentName = agent.name + this.lastCreatedAgentRole = agent.role + }, +) + Given('a minimal Agent v2 composer draft has been synced', async function (this: DifyWorld) { const agentId = getCurrentAgentId(this) diff --git a/e2e/support/agent.ts b/e2e/support/agent.ts index 9372f8453dec2a..95baf67fdf06f5 100644 --- a/e2e/support/agent.ts +++ b/e2e/support/agent.ts @@ -99,6 +99,18 @@ export async function createTestAgent({ } } +export async function createConfiguredTestAgent({ + agentSoul = normalAgentSoulConfig, + seed, +}: { + agentSoul?: AgentSoulConfig + seed?: CreateTestAgentOptions +} = {}): Promise { + const agent = await createTestAgent(seed) + await saveAgentComposerDraft(agent.id, agentSoul) + return agent +} + export async function getTestAgent(agentId: string): Promise { const ctx = await createApiContext() try { From 7a5800e4e6ce62e0263ce1698befc47131eb260b Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 11:42:16 +0800 Subject: [PATCH 005/185] test(e2e): add agent builder file fixtures --- e2e/.gitignore | 1 + .../agent-build-instruction.txt | 3 + .../test-materials/agent-empty-file.txt | 0 e2e/fixtures/test-materials/agent-invalid.env | 4 ++ .../test-materials/agent-knowledge-source.txt | 2 + .../test-materials/agent-small-file.txt | 2 + ...ilename-\344\270\255\346\226\207 @#$%.txt" | 2 + .../test-materials/agent-unsupported-file.exe | 1 + e2e/fixtures/test-materials/agent-valid.env | 2 + .../count_batch_5_valid_files/file-1.txt | 1 + .../count_batch_5_valid_files/file-2.txt | 1 + .../count_batch_5_valid_files/file-3.txt | 1 + .../count_batch_5_valid_files/file-4.txt | 1 + .../count_batch_5_valid_files/file-5.txt | 1 + .../count_batch_6_valid_files/file-1.txt | 1 + .../count_batch_6_valid_files/file-2.txt | 1 + .../count_batch_6_valid_files/file-3.txt | 1 + .../count_batch_6_valid_files/file-4.txt | 1 + .../count_batch_6_valid_files/file-5.txt | 1 + .../count_batch_6_valid_files/file-6.txt | 1 + .../count_total_50_valid_files/file-01.txt | 1 + .../count_total_50_valid_files/file-02.txt | 1 + .../count_total_50_valid_files/file-03.txt | 1 + .../count_total_50_valid_files/file-04.txt | 1 + .../count_total_50_valid_files/file-05.txt | 1 + .../count_total_50_valid_files/file-06.txt | 1 + .../count_total_50_valid_files/file-07.txt | 1 + .../count_total_50_valid_files/file-08.txt | 1 + .../count_total_50_valid_files/file-09.txt | 1 + .../count_total_50_valid_files/file-10.txt | 1 + .../count_total_50_valid_files/file-11.txt | 1 + .../count_total_50_valid_files/file-12.txt | 1 + .../count_total_50_valid_files/file-13.txt | 1 + .../count_total_50_valid_files/file-14.txt | 1 + .../count_total_50_valid_files/file-15.txt | 1 + .../count_total_50_valid_files/file-16.txt | 1 + .../count_total_50_valid_files/file-17.txt | 1 + .../count_total_50_valid_files/file-18.txt | 1 + .../count_total_50_valid_files/file-19.txt | 1 + .../count_total_50_valid_files/file-20.txt | 1 + .../count_total_50_valid_files/file-21.txt | 1 + .../count_total_50_valid_files/file-22.txt | 1 + .../count_total_50_valid_files/file-23.txt | 1 + .../count_total_50_valid_files/file-24.txt | 1 + .../count_total_50_valid_files/file-25.txt | 1 + .../count_total_50_valid_files/file-26.txt | 1 + .../count_total_50_valid_files/file-27.txt | 1 + .../count_total_50_valid_files/file-28.txt | 1 + .../count_total_50_valid_files/file-29.txt | 1 + .../count_total_50_valid_files/file-30.txt | 1 + .../count_total_50_valid_files/file-31.txt | 1 + .../count_total_50_valid_files/file-32.txt | 1 + .../count_total_50_valid_files/file-33.txt | 1 + .../count_total_50_valid_files/file-34.txt | 1 + .../count_total_50_valid_files/file-35.txt | 1 + .../count_total_50_valid_files/file-36.txt | 1 + .../count_total_50_valid_files/file-37.txt | 1 + .../count_total_50_valid_files/file-38.txt | 1 + .../count_total_50_valid_files/file-39.txt | 1 + .../count_total_50_valid_files/file-40.txt | 1 + .../count_total_50_valid_files/file-41.txt | 1 + .../count_total_50_valid_files/file-42.txt | 1 + .../count_total_50_valid_files/file-43.txt | 1 + .../count_total_50_valid_files/file-44.txt | 1 + .../count_total_50_valid_files/file-45.txt | 1 + .../count_total_50_valid_files/file-46.txt | 1 + .../count_total_50_valid_files/file-47.txt | 1 + .../count_total_50_valid_files/file-48.txt | 1 + .../count_total_50_valid_files/file-49.txt | 1 + .../count_total_50_valid_files/file-50.txt | 1 + .../file-01.txt | 1 + .../test-materials/e2e-summary.SKILL.md | 5 ++ .../file_tree_fixture/assets/sample.csv | 3 + ...55\346\226\207\350\257\264\346\230\216.md" | 3 + .../file_tree_fixture/public/index.html | 2 + .../file_tree_fixture/src/main.txt | 1 + .../file_tree_fixture/web-game/README.md | 3 + e2e/support/test-materials.ts | 68 +++++++++++++++++++ 78 files changed, 164 insertions(+) create mode 100644 e2e/fixtures/test-materials/agent-build-instruction.txt create mode 100644 e2e/fixtures/test-materials/agent-empty-file.txt create mode 100644 e2e/fixtures/test-materials/agent-invalid.env create mode 100644 e2e/fixtures/test-materials/agent-knowledge-source.txt create mode 100644 e2e/fixtures/test-materials/agent-small-file.txt create mode 100644 "e2e/fixtures/test-materials/agent-special-filename-\344\270\255\346\226\207 @#$%.txt" create mode 100644 e2e/fixtures/test-materials/agent-unsupported-file.exe create mode 100644 e2e/fixtures/test-materials/agent-valid.env create mode 100644 e2e/fixtures/test-materials/count_batch_5_valid_files/file-1.txt create mode 100644 e2e/fixtures/test-materials/count_batch_5_valid_files/file-2.txt create mode 100644 e2e/fixtures/test-materials/count_batch_5_valid_files/file-3.txt create mode 100644 e2e/fixtures/test-materials/count_batch_5_valid_files/file-4.txt create mode 100644 e2e/fixtures/test-materials/count_batch_5_valid_files/file-5.txt create mode 100644 e2e/fixtures/test-materials/count_batch_6_valid_files/file-1.txt create mode 100644 e2e/fixtures/test-materials/count_batch_6_valid_files/file-2.txt create mode 100644 e2e/fixtures/test-materials/count_batch_6_valid_files/file-3.txt create mode 100644 e2e/fixtures/test-materials/count_batch_6_valid_files/file-4.txt create mode 100644 e2e/fixtures/test-materials/count_batch_6_valid_files/file-5.txt create mode 100644 e2e/fixtures/test-materials/count_batch_6_valid_files/file-6.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-01.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-02.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-03.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-04.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-05.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-06.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-07.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-08.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-09.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-10.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-11.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-12.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-13.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-14.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-15.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-16.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-17.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-18.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-19.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-20.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-21.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-22.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-23.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-24.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-25.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-26.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-27.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-28.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-29.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-30.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-31.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-32.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-33.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-34.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-35.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-36.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-37.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-38.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-39.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-40.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-41.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-42.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-43.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-44.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-45.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-46.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-47.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-48.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-49.txt create mode 100644 e2e/fixtures/test-materials/count_total_50_valid_files/file-50.txt create mode 100644 e2e/fixtures/test-materials/count_total_extra_1_valid_file/file-01.txt create mode 100644 e2e/fixtures/test-materials/e2e-summary.SKILL.md create mode 100644 e2e/fixtures/test-materials/file_tree_fixture/assets/sample.csv create mode 100644 "e2e/fixtures/test-materials/file_tree_fixture/docs/\344\270\255\346\226\207\350\257\264\346\230\216.md" create mode 100644 e2e/fixtures/test-materials/file_tree_fixture/public/index.html create mode 100644 e2e/fixtures/test-materials/file_tree_fixture/src/main.txt create mode 100644 e2e/fixtures/test-materials/file_tree_fixture/web-game/README.md create mode 100644 e2e/support/test-materials.ts diff --git a/e2e/.gitignore b/e2e/.gitignore index 96c1e0f3a18978..94517c8409d24d 100644 --- a/e2e/.gitignore +++ b/e2e/.gitignore @@ -4,3 +4,4 @@ playwright-report/ test-results/ cucumber-report/ .logs/ +.generated-test-materials/ diff --git a/e2e/fixtures/test-materials/agent-build-instruction.txt b/e2e/fixtures/test-materials/agent-build-instruction.txt new file mode 100644 index 00000000000000..45a36462d2f555 --- /dev/null +++ b/e2e/fixtures/test-materials/agent-build-instruction.txt @@ -0,0 +1,3 @@ +Ask the Agent to use E2E Summary Skill to summarize user input. +When replacing JSON strings, use JSON Process / JSON Replace. +After applying, include these capabilities in the Agent instructions. diff --git a/e2e/fixtures/test-materials/agent-empty-file.txt b/e2e/fixtures/test-materials/agent-empty-file.txt new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/e2e/fixtures/test-materials/agent-invalid.env b/e2e/fixtures/test-materials/agent-invalid.env new file mode 100644 index 00000000000000..5cf1e3a3bdbfeb --- /dev/null +++ b/e2e/fixtures/test-materials/agent-invalid.env @@ -0,0 +1,4 @@ +E2E_AGENT_FLAG=enabled +INVALID ENV LINE WITHOUT EQUALS +=missing_key +E2E_AGENT_AFTER_INVALID=still-valid diff --git a/e2e/fixtures/test-materials/agent-knowledge-source.txt b/e2e/fixtures/test-materials/agent-knowledge-source.txt new file mode 100644 index 00000000000000..2275246ac5b5c3 --- /dev/null +++ b/e2e/fixtures/test-materials/agent-knowledge-source.txt @@ -0,0 +1,2 @@ +Dify Agent E2E knowledge source. +Dify Agent E2E test passphrase is AGENT_KNOWLEDGE_PASS. diff --git a/e2e/fixtures/test-materials/agent-small-file.txt b/e2e/fixtures/test-materials/agent-small-file.txt new file mode 100644 index 00000000000000..13ec36cc2437fb --- /dev/null +++ b/e2e/fixtures/test-materials/agent-small-file.txt @@ -0,0 +1,2 @@ +Dify Agent E2E small file fixture. +Expected token: AGENT_FILE_PASS diff --git "a/e2e/fixtures/test-materials/agent-special-filename-\344\270\255\346\226\207 @#$%.txt" "b/e2e/fixtures/test-materials/agent-special-filename-\344\270\255\346\226\207 @#$%.txt" new file mode 100644 index 00000000000000..c58409d856042b --- /dev/null +++ "b/e2e/fixtures/test-materials/agent-special-filename-\344\270\255\346\226\207 @#$%.txt" @@ -0,0 +1,2 @@ +Special filename fixture. +Expected token: AGENT_SPECIAL_FILENAME_PASS diff --git a/e2e/fixtures/test-materials/agent-unsupported-file.exe b/e2e/fixtures/test-materials/agent-unsupported-file.exe new file mode 100644 index 00000000000000..e45fe0473da183 --- /dev/null +++ b/e2e/fixtures/test-materials/agent-unsupported-file.exe @@ -0,0 +1 @@ +This file intentionally uses an unsupported extension for upload validation. diff --git a/e2e/fixtures/test-materials/agent-valid.env b/e2e/fixtures/test-materials/agent-valid.env new file mode 100644 index 00000000000000..cd3ac0a0009717 --- /dev/null +++ b/e2e/fixtures/test-materials/agent-valid.env @@ -0,0 +1,2 @@ +E2E_AGENT_FLAG=enabled +E2E_AGENT_MODE=plain diff --git a/e2e/fixtures/test-materials/count_batch_5_valid_files/file-1.txt b/e2e/fixtures/test-materials/count_batch_5_valid_files/file-1.txt new file mode 100644 index 00000000000000..a64cd23552ddd3 --- /dev/null +++ b/e2e/fixtures/test-materials/count_batch_5_valid_files/file-1.txt @@ -0,0 +1 @@ +Batch 5 valid file 1 token E2E_BATCH_5_1 diff --git a/e2e/fixtures/test-materials/count_batch_5_valid_files/file-2.txt b/e2e/fixtures/test-materials/count_batch_5_valid_files/file-2.txt new file mode 100644 index 00000000000000..1a65ed90e76900 --- /dev/null +++ b/e2e/fixtures/test-materials/count_batch_5_valid_files/file-2.txt @@ -0,0 +1 @@ +Batch 5 valid file 2 token E2E_BATCH_5_2 diff --git a/e2e/fixtures/test-materials/count_batch_5_valid_files/file-3.txt b/e2e/fixtures/test-materials/count_batch_5_valid_files/file-3.txt new file mode 100644 index 00000000000000..c5c00a2477e4b0 --- /dev/null +++ b/e2e/fixtures/test-materials/count_batch_5_valid_files/file-3.txt @@ -0,0 +1 @@ +Batch 5 valid file 3 token E2E_BATCH_5_3 diff --git a/e2e/fixtures/test-materials/count_batch_5_valid_files/file-4.txt b/e2e/fixtures/test-materials/count_batch_5_valid_files/file-4.txt new file mode 100644 index 00000000000000..120e4ee335c89c --- /dev/null +++ b/e2e/fixtures/test-materials/count_batch_5_valid_files/file-4.txt @@ -0,0 +1 @@ +Batch 5 valid file 4 token E2E_BATCH_5_4 diff --git a/e2e/fixtures/test-materials/count_batch_5_valid_files/file-5.txt b/e2e/fixtures/test-materials/count_batch_5_valid_files/file-5.txt new file mode 100644 index 00000000000000..4efaff6d828fa3 --- /dev/null +++ b/e2e/fixtures/test-materials/count_batch_5_valid_files/file-5.txt @@ -0,0 +1 @@ +Batch 5 valid file 5 token E2E_BATCH_5_5 diff --git a/e2e/fixtures/test-materials/count_batch_6_valid_files/file-1.txt b/e2e/fixtures/test-materials/count_batch_6_valid_files/file-1.txt new file mode 100644 index 00000000000000..6ce029a426b856 --- /dev/null +++ b/e2e/fixtures/test-materials/count_batch_6_valid_files/file-1.txt @@ -0,0 +1 @@ +Batch 6 valid file 1 token E2E_BATCH_6_1 diff --git a/e2e/fixtures/test-materials/count_batch_6_valid_files/file-2.txt b/e2e/fixtures/test-materials/count_batch_6_valid_files/file-2.txt new file mode 100644 index 00000000000000..08620c1994fa95 --- /dev/null +++ b/e2e/fixtures/test-materials/count_batch_6_valid_files/file-2.txt @@ -0,0 +1 @@ +Batch 6 valid file 2 token E2E_BATCH_6_2 diff --git a/e2e/fixtures/test-materials/count_batch_6_valid_files/file-3.txt b/e2e/fixtures/test-materials/count_batch_6_valid_files/file-3.txt new file mode 100644 index 00000000000000..73deeefa755fb0 --- /dev/null +++ b/e2e/fixtures/test-materials/count_batch_6_valid_files/file-3.txt @@ -0,0 +1 @@ +Batch 6 valid file 3 token E2E_BATCH_6_3 diff --git a/e2e/fixtures/test-materials/count_batch_6_valid_files/file-4.txt b/e2e/fixtures/test-materials/count_batch_6_valid_files/file-4.txt new file mode 100644 index 00000000000000..5b62b0d70752af --- /dev/null +++ b/e2e/fixtures/test-materials/count_batch_6_valid_files/file-4.txt @@ -0,0 +1 @@ +Batch 6 valid file 4 token E2E_BATCH_6_4 diff --git a/e2e/fixtures/test-materials/count_batch_6_valid_files/file-5.txt b/e2e/fixtures/test-materials/count_batch_6_valid_files/file-5.txt new file mode 100644 index 00000000000000..c0192f87bb4f45 --- /dev/null +++ b/e2e/fixtures/test-materials/count_batch_6_valid_files/file-5.txt @@ -0,0 +1 @@ +Batch 6 valid file 5 token E2E_BATCH_6_5 diff --git a/e2e/fixtures/test-materials/count_batch_6_valid_files/file-6.txt b/e2e/fixtures/test-materials/count_batch_6_valid_files/file-6.txt new file mode 100644 index 00000000000000..e3505c43d595a8 --- /dev/null +++ b/e2e/fixtures/test-materials/count_batch_6_valid_files/file-6.txt @@ -0,0 +1 @@ +Batch 6 valid file 6 token E2E_BATCH_6_6 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-01.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-01.txt new file mode 100644 index 00000000000000..db1a55a0021c4f --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-01.txt @@ -0,0 +1 @@ +Total 50 valid file 01 token E2E_TOTAL_50_01 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-02.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-02.txt new file mode 100644 index 00000000000000..bdcd785ab8f835 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-02.txt @@ -0,0 +1 @@ +Total 50 valid file 02 token E2E_TOTAL_50_02 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-03.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-03.txt new file mode 100644 index 00000000000000..cf00c92b0a36f6 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-03.txt @@ -0,0 +1 @@ +Total 50 valid file 03 token E2E_TOTAL_50_03 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-04.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-04.txt new file mode 100644 index 00000000000000..01522864989af4 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-04.txt @@ -0,0 +1 @@ +Total 50 valid file 04 token E2E_TOTAL_50_04 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-05.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-05.txt new file mode 100644 index 00000000000000..22a118ddd0fa20 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-05.txt @@ -0,0 +1 @@ +Total 50 valid file 05 token E2E_TOTAL_50_05 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-06.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-06.txt new file mode 100644 index 00000000000000..3cee5574426485 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-06.txt @@ -0,0 +1 @@ +Total 50 valid file 06 token E2E_TOTAL_50_06 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-07.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-07.txt new file mode 100644 index 00000000000000..381bf4d24d5b86 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-07.txt @@ -0,0 +1 @@ +Total 50 valid file 07 token E2E_TOTAL_50_07 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-08.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-08.txt new file mode 100644 index 00000000000000..8cc4d21fc8698c --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-08.txt @@ -0,0 +1 @@ +Total 50 valid file 08 token E2E_TOTAL_50_08 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-09.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-09.txt new file mode 100644 index 00000000000000..490cc4b779c9cf --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-09.txt @@ -0,0 +1 @@ +Total 50 valid file 09 token E2E_TOTAL_50_09 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-10.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-10.txt new file mode 100644 index 00000000000000..fe4089625f598f --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-10.txt @@ -0,0 +1 @@ +Total 50 valid file 10 token E2E_TOTAL_50_10 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-11.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-11.txt new file mode 100644 index 00000000000000..b5a9ae2ea3f92c --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-11.txt @@ -0,0 +1 @@ +Total 50 valid file 11 token E2E_TOTAL_50_11 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-12.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-12.txt new file mode 100644 index 00000000000000..33e0f0b77c0de3 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-12.txt @@ -0,0 +1 @@ +Total 50 valid file 12 token E2E_TOTAL_50_12 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-13.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-13.txt new file mode 100644 index 00000000000000..0bf532f9a32fd0 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-13.txt @@ -0,0 +1 @@ +Total 50 valid file 13 token E2E_TOTAL_50_13 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-14.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-14.txt new file mode 100644 index 00000000000000..60b3cb0d7517de --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-14.txt @@ -0,0 +1 @@ +Total 50 valid file 14 token E2E_TOTAL_50_14 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-15.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-15.txt new file mode 100644 index 00000000000000..15c0e3769bef02 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-15.txt @@ -0,0 +1 @@ +Total 50 valid file 15 token E2E_TOTAL_50_15 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-16.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-16.txt new file mode 100644 index 00000000000000..6c80e6e66bb553 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-16.txt @@ -0,0 +1 @@ +Total 50 valid file 16 token E2E_TOTAL_50_16 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-17.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-17.txt new file mode 100644 index 00000000000000..9ac689f667afb8 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-17.txt @@ -0,0 +1 @@ +Total 50 valid file 17 token E2E_TOTAL_50_17 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-18.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-18.txt new file mode 100644 index 00000000000000..3a9c6725090685 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-18.txt @@ -0,0 +1 @@ +Total 50 valid file 18 token E2E_TOTAL_50_18 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-19.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-19.txt new file mode 100644 index 00000000000000..35d544714c8c1e --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-19.txt @@ -0,0 +1 @@ +Total 50 valid file 19 token E2E_TOTAL_50_19 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-20.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-20.txt new file mode 100644 index 00000000000000..cc53594e52fa83 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-20.txt @@ -0,0 +1 @@ +Total 50 valid file 20 token E2E_TOTAL_50_20 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-21.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-21.txt new file mode 100644 index 00000000000000..2e0af50430ca14 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-21.txt @@ -0,0 +1 @@ +Total 50 valid file 21 token E2E_TOTAL_50_21 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-22.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-22.txt new file mode 100644 index 00000000000000..fe09be37a12ee2 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-22.txt @@ -0,0 +1 @@ +Total 50 valid file 22 token E2E_TOTAL_50_22 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-23.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-23.txt new file mode 100644 index 00000000000000..70bb91ee326670 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-23.txt @@ -0,0 +1 @@ +Total 50 valid file 23 token E2E_TOTAL_50_23 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-24.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-24.txt new file mode 100644 index 00000000000000..a8ba2e60d615fa --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-24.txt @@ -0,0 +1 @@ +Total 50 valid file 24 token E2E_TOTAL_50_24 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-25.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-25.txt new file mode 100644 index 00000000000000..ddfe2afed467e1 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-25.txt @@ -0,0 +1 @@ +Total 50 valid file 25 token E2E_TOTAL_50_25 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-26.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-26.txt new file mode 100644 index 00000000000000..1a56d928a9a96d --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-26.txt @@ -0,0 +1 @@ +Total 50 valid file 26 token E2E_TOTAL_50_26 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-27.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-27.txt new file mode 100644 index 00000000000000..a8182ad2e71069 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-27.txt @@ -0,0 +1 @@ +Total 50 valid file 27 token E2E_TOTAL_50_27 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-28.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-28.txt new file mode 100644 index 00000000000000..a6f78f011db032 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-28.txt @@ -0,0 +1 @@ +Total 50 valid file 28 token E2E_TOTAL_50_28 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-29.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-29.txt new file mode 100644 index 00000000000000..7a76d201194786 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-29.txt @@ -0,0 +1 @@ +Total 50 valid file 29 token E2E_TOTAL_50_29 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-30.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-30.txt new file mode 100644 index 00000000000000..5a37417ca8c604 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-30.txt @@ -0,0 +1 @@ +Total 50 valid file 30 token E2E_TOTAL_50_30 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-31.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-31.txt new file mode 100644 index 00000000000000..1baa8eb8c7b0d9 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-31.txt @@ -0,0 +1 @@ +Total 50 valid file 31 token E2E_TOTAL_50_31 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-32.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-32.txt new file mode 100644 index 00000000000000..e5ff1a42437b42 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-32.txt @@ -0,0 +1 @@ +Total 50 valid file 32 token E2E_TOTAL_50_32 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-33.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-33.txt new file mode 100644 index 00000000000000..dcbc2995748573 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-33.txt @@ -0,0 +1 @@ +Total 50 valid file 33 token E2E_TOTAL_50_33 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-34.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-34.txt new file mode 100644 index 00000000000000..fefc133eed1629 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-34.txt @@ -0,0 +1 @@ +Total 50 valid file 34 token E2E_TOTAL_50_34 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-35.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-35.txt new file mode 100644 index 00000000000000..1c1a93172cb00f --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-35.txt @@ -0,0 +1 @@ +Total 50 valid file 35 token E2E_TOTAL_50_35 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-36.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-36.txt new file mode 100644 index 00000000000000..d7e0bab1aaf420 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-36.txt @@ -0,0 +1 @@ +Total 50 valid file 36 token E2E_TOTAL_50_36 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-37.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-37.txt new file mode 100644 index 00000000000000..6a78127243fcf5 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-37.txt @@ -0,0 +1 @@ +Total 50 valid file 37 token E2E_TOTAL_50_37 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-38.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-38.txt new file mode 100644 index 00000000000000..fc5ef9123db251 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-38.txt @@ -0,0 +1 @@ +Total 50 valid file 38 token E2E_TOTAL_50_38 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-39.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-39.txt new file mode 100644 index 00000000000000..37765df65e5a58 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-39.txt @@ -0,0 +1 @@ +Total 50 valid file 39 token E2E_TOTAL_50_39 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-40.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-40.txt new file mode 100644 index 00000000000000..74de5e2ed4e726 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-40.txt @@ -0,0 +1 @@ +Total 50 valid file 40 token E2E_TOTAL_50_40 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-41.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-41.txt new file mode 100644 index 00000000000000..13d3dee7c514c5 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-41.txt @@ -0,0 +1 @@ +Total 50 valid file 41 token E2E_TOTAL_50_41 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-42.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-42.txt new file mode 100644 index 00000000000000..befc45f23878e4 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-42.txt @@ -0,0 +1 @@ +Total 50 valid file 42 token E2E_TOTAL_50_42 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-43.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-43.txt new file mode 100644 index 00000000000000..521ad583a5a7cd --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-43.txt @@ -0,0 +1 @@ +Total 50 valid file 43 token E2E_TOTAL_50_43 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-44.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-44.txt new file mode 100644 index 00000000000000..e4b315b675f542 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-44.txt @@ -0,0 +1 @@ +Total 50 valid file 44 token E2E_TOTAL_50_44 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-45.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-45.txt new file mode 100644 index 00000000000000..c97738e4fac253 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-45.txt @@ -0,0 +1 @@ +Total 50 valid file 45 token E2E_TOTAL_50_45 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-46.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-46.txt new file mode 100644 index 00000000000000..22b73767b42e92 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-46.txt @@ -0,0 +1 @@ +Total 50 valid file 46 token E2E_TOTAL_50_46 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-47.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-47.txt new file mode 100644 index 00000000000000..b327026efd0363 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-47.txt @@ -0,0 +1 @@ +Total 50 valid file 47 token E2E_TOTAL_50_47 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-48.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-48.txt new file mode 100644 index 00000000000000..5458e0cf222a82 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-48.txt @@ -0,0 +1 @@ +Total 50 valid file 48 token E2E_TOTAL_50_48 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-49.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-49.txt new file mode 100644 index 00000000000000..90bf454dd5feba --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-49.txt @@ -0,0 +1 @@ +Total 50 valid file 49 token E2E_TOTAL_50_49 diff --git a/e2e/fixtures/test-materials/count_total_50_valid_files/file-50.txt b/e2e/fixtures/test-materials/count_total_50_valid_files/file-50.txt new file mode 100644 index 00000000000000..b5760aed272798 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_50_valid_files/file-50.txt @@ -0,0 +1 @@ +Total 50 valid file 50 token E2E_TOTAL_50_50 diff --git a/e2e/fixtures/test-materials/count_total_extra_1_valid_file/file-01.txt b/e2e/fixtures/test-materials/count_total_extra_1_valid_file/file-01.txt new file mode 100644 index 00000000000000..4df2e230c05a29 --- /dev/null +++ b/e2e/fixtures/test-materials/count_total_extra_1_valid_file/file-01.txt @@ -0,0 +1 @@ +Total extra valid file token E2E_TOTAL_EXTRA_1 diff --git a/e2e/fixtures/test-materials/e2e-summary.SKILL.md b/e2e/fixtures/test-materials/e2e-summary.SKILL.md new file mode 100644 index 00000000000000..948f6bb143b491 --- /dev/null +++ b/e2e/fixtures/test-materials/e2e-summary.SKILL.md @@ -0,0 +1,5 @@ +# E2E Summary Skill + +Summarize the user input in one concise paragraph. + +The summary must include E2E_SUMMARY_SKILL_PASS. diff --git a/e2e/fixtures/test-materials/file_tree_fixture/assets/sample.csv b/e2e/fixtures/test-materials/file_tree_fixture/assets/sample.csv new file mode 100644 index 00000000000000..bed40b0fc098c5 --- /dev/null +++ b/e2e/fixtures/test-materials/file_tree_fixture/assets/sample.csv @@ -0,0 +1,3 @@ +name,value +alpha,1 +beta,2 diff --git "a/e2e/fixtures/test-materials/file_tree_fixture/docs/\344\270\255\346\226\207\350\257\264\346\230\216.md" "b/e2e/fixtures/test-materials/file_tree_fixture/docs/\344\270\255\346\226\207\350\257\264\346\230\216.md" new file mode 100644 index 00000000000000..9d82b4cabd154e --- /dev/null +++ "b/e2e/fixtures/test-materials/file_tree_fixture/docs/\344\270\255\346\226\207\350\257\264\346\230\216.md" @@ -0,0 +1,3 @@ +# 中文说明 + +文件树中文说明 token: E2E_FILE_TREE_ZH diff --git a/e2e/fixtures/test-materials/file_tree_fixture/public/index.html b/e2e/fixtures/test-materials/file_tree_fixture/public/index.html new file mode 100644 index 00000000000000..1e053518f4b564 --- /dev/null +++ b/e2e/fixtures/test-materials/file_tree_fixture/public/index.html @@ -0,0 +1,2 @@ + +E2E_FILE_TREE_INDEX diff --git a/e2e/fixtures/test-materials/file_tree_fixture/src/main.txt b/e2e/fixtures/test-materials/file_tree_fixture/src/main.txt new file mode 100644 index 00000000000000..2f52ebf5950efd --- /dev/null +++ b/e2e/fixtures/test-materials/file_tree_fixture/src/main.txt @@ -0,0 +1 @@ +Main source fixture token: E2E_FILE_TREE_MAIN diff --git a/e2e/fixtures/test-materials/file_tree_fixture/web-game/README.md b/e2e/fixtures/test-materials/file_tree_fixture/web-game/README.md new file mode 100644 index 00000000000000..3581e8ee127e3a --- /dev/null +++ b/e2e/fixtures/test-materials/file_tree_fixture/web-game/README.md @@ -0,0 +1,3 @@ +# Web Game Fixture + +Expected token: E2E_FILE_TREE_README diff --git a/e2e/support/test-materials.ts b/e2e/support/test-materials.ts new file mode 100644 index 00000000000000..ccefedd7fc7842 --- /dev/null +++ b/e2e/support/test-materials.ts @@ -0,0 +1,68 @@ +import { Buffer } from 'node:buffer' +import { mkdir, writeFile } from 'node:fs/promises' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +export const testMaterialsDir = fileURLToPath( + new URL('../fixtures/test-materials', import.meta.url), +) +export const generatedTestMaterialsDir = fileURLToPath( + new URL('../.generated-test-materials', import.meta.url), +) + +export const getTestMaterialPath = (fileName: string) => path.join(testMaterialsDir, fileName) + +export const agentBuilderTestMaterials = { + smallFile: 'agent-small-file.txt', + knowledgeSource: 'agent-knowledge-source.txt', + emptyFile: 'agent-empty-file.txt', + unsupportedFile: 'agent-unsupported-file.exe', + specialFilename: 'agent-special-filename-中文 @#$%.txt', + validEnv: 'agent-valid.env', + invalidEnv: 'agent-invalid.env', + buildInstruction: 'agent-build-instruction.txt', + summarySkill: 'e2e-summary.SKILL.md', + fileTreeFixture: 'file_tree_fixture', + countBatch5: 'count_batch_5_valid_files', + countBatch6: 'count_batch_6_valid_files', + countTotal50: 'count_total_50_valid_files', + countTotalExtra1: 'count_total_extra_1_valid_file', +} as const + +export const getAgentBuilderTestMaterialPath = ( + material: keyof typeof agentBuilderTestMaterials, +) => getTestMaterialPath(agentBuilderTestMaterials[material]) + +export async function getGeneratedTextMaterialPath({ + fileName, + sizeBytes, + seedText, +}: { + fileName: string + sizeBytes: number + seedText: string +}) { + await mkdir(generatedTestMaterialsDir, { recursive: true }) + + const targetPath = path.join(generatedTestMaterialsDir, fileName) + const chunk = `${seedText}\n` + const repeatCount = Math.ceil(sizeBytes / Buffer.byteLength(chunk)) + const contents = chunk.repeat(repeatCount).slice(0, sizeBytes) + await writeFile(targetPath, contents) + + return targetPath +} + +export const getTooLargeAgentFilePath = () => + getGeneratedTextMaterialPath({ + fileName: 'agent-too-large-file.txt', + sizeBytes: 16 * 1024 * 1024, + seedText: 'E2E_TOO_LARGE_FILE_FIXTURE', + }) + +export const getSlowUploadAgentFilePath = () => + getGeneratedTextMaterialPath({ + fileName: 'agent-slow-upload-file.txt', + sizeBytes: 2 * 1024 * 1024, + seedText: 'E2E_SLOW_UPLOAD_FILE_FIXTURE', + }) From 16837cfcefb43041d6f499e1b0975e5b0ddc1da3 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 11:42:28 +0800 Subject: [PATCH 006/185] test(e2e): add agent builder preflight resources --- e2e/support/agent-builder-resources.ts | 41 +++++++++++++++++++ e2e/support/preflight.ts | 55 ++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 e2e/support/agent-builder-resources.ts create mode 100644 e2e/support/preflight.ts diff --git a/e2e/support/agent-builder-resources.ts b/e2e/support/agent-builder-resources.ts new file mode 100644 index 00000000000000..fa6b0376c233a6 --- /dev/null +++ b/e2e/support/agent-builder-resources.ts @@ -0,0 +1,41 @@ +export const agentBuilderPreseededResources = { + stableChatModel: 'E2E Stable Chat Model', + summarySkill: 'E2E Summary Skill', + jsonReplaceTool: 'JSON Process / JSON Replace', + tavilySearchTool: 'Tavily / Tavily Search', + agentKnowledgeBase: 'E2E Agent Knowledge Base', + indexingKnowledgeBase: 'E2E Agent Knowledge Base Indexing', + brokenModelProvider: 'E2E Broken Model Provider', + brokenModel: 'e2e-broken-model', + fullConfigAgent: 'E2E New Agent Builder Full Config', + toolStatesAgent: 'E2E New Agent Builder Tool States', + fileTreeAgent: 'E2E Agent With File Tree', + dualRetrievalAgent: 'E2E Agent With Dual Retrieval', + publishedWebAppAgent: 'E2E Agent Published Web App', + backendApiEnabledAgent: 'E2E Agent Backend API Enabled', + workflowReferenceAgent: 'E2E Agent With Workflow Reference', + referenceWorkflow: 'E2E Agent Reference Workflow', + backendApiKey: 'E2E Backend API Key', +} as const + +export const agentBuilderFixedInputs = { + tavilyInvalidApiKey: 'E2E_INVALID_TAVILY_API_KEY_DO_NOT_USE', + missingSkillSearch: 'E2E_NOT_EXIST_SKILL', + missingToolSearch: 'E2E_NOT_EXIST_TOOL', + missingToolSearchWithSuffix: 'E2E_NOT_EXIST_TOOL_12345', + customKnowledgeQuery: 'Dify Agent E2E 测试暗号', + envPlainKey: 'E2E_AGENT_FLAG', + envPlainValue: 'enabled', + inputModerationReply: 'E2E_INPUT_BLOCKED_REPLY', + outputModerationReply: 'E2E_OUTPUT_BLOCKED_REPLY', + previewSuccessQuery: '请回复测试成功', + backendApiUser: 'e2e-agent-access-point', +} as const + +export const agentBuilderExpectedTokens = { + agentReply: 'AGENT_E2E_PASS', + updatedAgentReply: 'E2E_AGENT_UPDATED', + knowledgeReply: 'AGENT_KNOWLEDGE_PASS', + jsonToolBefore: 'JSON_TOOL_E2E', + jsonToolAfter: 'E2E_AFTER', +} as const diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts new file mode 100644 index 00000000000000..197b1a0e4fcd5e --- /dev/null +++ b/e2e/support/preflight.ts @@ -0,0 +1,55 @@ +import type { DifyWorld } from '../features/support/world' +import { agentBuilderPreseededResources } from './agent-builder-resources' + +export type E2EResourcePrecondition + = | { + ok: true + value: string + } + | { + ok: false + reason: string + } + +export const readRequiredEnvResource = ( + envName: string, + description: string, +): E2EResourcePrecondition => { + const value = process.env[envName]?.trim() + if (value) + return { ok: true, value } + + return { + ok: false, + reason: `${description} requires ${envName}.`, + } +} + +export function skipBlockedPrecondition(world: DifyWorld, reason: string): 'skipped' { + const message = `Blocked precondition: ${reason}` + console.warn(`[e2e] ${message}`) + world.attach(message, 'text/plain') + return 'skipped' +} + +export function skipMissingEnvResource( + world: DifyWorld, + envName: string, + description: string, +): 'skipped' | string { + const resource = readRequiredEnvResource(envName, description) + if (resource.ok) + return resource.value + + return skipBlockedPrecondition(world, resource.reason) +} + +export const requiredAgentBuilderPreseededResources = Object.values(agentBuilderPreseededResources) + +export function skipMissingAgentBuilderPreseed( + world: DifyWorld, + resourceName: string, + envName: string, +): 'skipped' | string { + return skipMissingEnvResource(world, envName, `Preseeded Agent Builder resource "${resourceName}"`) +} From 12191c20c6461b5d1e4b302ec09aff1900062c48 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 11:43:19 +0800 Subject: [PATCH 007/185] test(e2e): add scenario cleanup registry --- e2e/features/support/hooks.ts | 1 + e2e/features/support/world.ts | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/e2e/features/support/hooks.ts b/e2e/features/support/hooks.ts index e67291a7ad2932..4fbcdfed9d5bb9 100644 --- a/e2e/features/support/hooks.ts +++ b/e2e/features/support/hooks.ts @@ -93,6 +93,7 @@ After(async function (this: DifyWorld, { pickle, result }) { for (const id of this.createdAgentIds) await deleteTestAgent(id).catch(() => {}) for (const id of this.createdAppIds) await deleteTestApp(id).catch(() => {}) + await this.runRegisteredCleanups() await this.closeSession() }) diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index e79a58281e5eaf..4d0f8427df6744 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -5,6 +5,8 @@ import { setWorldConstructor, World } from '@cucumber/cucumber' import { authStatePath, readAuthSessionMetadata } from '../../fixtures/auth' import { baseURL, defaultLocale } from '../../test-env' +export type ScenarioCleanup = () => Promise | void + export class DifyWorld extends World { context: BrowserContext | undefined page: Page | undefined @@ -20,6 +22,7 @@ export class DifyWorld extends World { lastAgentApiReferencePage: Page | undefined createdAppIds: string[] = [] createdAgentIds: string[] = [] + scenarioCleanups: ScenarioCleanup[] = [] capturedDownloads: Download[] = [] shareURL: string | undefined @@ -39,6 +42,7 @@ export class DifyWorld extends World { this.lastAgentApiReferencePage = undefined this.createdAppIds = [] this.createdAgentIds = [] + this.scenarioCleanups = [] this.capturedDownloads = [] this.shareURL = undefined } @@ -86,6 +90,26 @@ export class DifyWorld extends World { return this.session } + registerCleanup(cleanup: ScenarioCleanup) { + this.scenarioCleanups.push(cleanup) + } + + async runRegisteredCleanups() { + const errors: string[] = [] + + for (const cleanup of this.scenarioCleanups.toReversed()) { + try { + await cleanup() + } + catch (error) { + errors.push(error instanceof Error ? error.message : String(error)) + } + } + + if (errors.length > 0) + this.attach(`Cleanup errors:\n${errors.join('\n')}`, 'text/plain') + } + async closeSession() { await this.context?.close() this.context = undefined From baa5c678a2ce9bc3c3b2c810ce7ac2aed7790206 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 11:43:40 +0800 Subject: [PATCH 008/185] test(e2e): add agent configure autosave wait --- e2e/features/step-definitions/agent-v2/configure.steps.ts | 5 +++++ e2e/support/agent-configure.ts | 6 ++++++ 2 files changed, 11 insertions(+) create mode 100644 e2e/support/agent-configure.ts diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 80887dcd96968d..4b2f1f7d297d2e 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -13,6 +13,7 @@ import { updatedAgentPrompt, updatedAgentSoulConfig, } from '../../../support/agent' +import { waitForAgentConfigureAutosaved } from '../../../support/agent-configure' const getCurrentAgentId = (world: DifyWorld) => { const agentId = world.createdAgentIds.at(-1) @@ -147,6 +148,10 @@ Then('the Agent v2 Build draft should no longer be active', async function (this await expect(page.getByRole('button', { name: 'Discard' })).not.toBeVisible() }) +Then('the Agent v2 configuration should be saved automatically', async function (this: DifyWorld) { + await waitForAgentConfigureAutosaved(this.getPage()) +}) + Then('the Agent v2 draft should be published and up to date', async function (this: DifyWorld) { const page = this.getPage() const agentId = getCurrentAgentId(this) diff --git a/e2e/support/agent-configure.ts b/e2e/support/agent-configure.ts new file mode 100644 index 00000000000000..8c883378a2c031 --- /dev/null +++ b/e2e/support/agent-configure.ts @@ -0,0 +1,6 @@ +import type { Page } from '@playwright/test' +import { expect } from '@playwright/test' + +export async function waitForAgentConfigureAutosaved(page: Page) { + await expect(page.getByText(/^Saved(?:\s|$)/).first()).toBeVisible({ timeout: 30_000 }) +} From 0ac64fc2984dc9de6421dcfa0c99851f60100c1c Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 11:44:01 +0800 Subject: [PATCH 009/185] docs(e2e): document agent builder fixture boundaries --- e2e/AGENTS.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 017594bbe346c7..58dc4681186e5e 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -286,6 +286,16 @@ The `After` hook automatically captures on failure: Artifacts are saved to `cucumber-report/artifacts/` and attached to the HTML report. No extra code needed in step definitions. +### Seed resources and preflight checks + +Use `support/naming.ts` for generated test resource names. New app, Agent, dataset, file, or credential seeds should start with `E2E` so local and shared environments can identify disposable resources. + +Use `fixtures/test-materials/` for checked-in files that scenarios upload, preview, index, or retrieve. Keep these fixtures small and deterministic, and use `support/test-materials.ts` to resolve their absolute paths. + +Use `support/preflight.ts` for scenarios that require optional external resources such as a stable model provider, plugin/tool credential, or knowledge base seed. Prefer an explicit `Given` step that returns a skipped result with a clear blocked-precondition reason over hidden setup in hooks. + +Use `DifyWorld.registerCleanup(...)` when a scenario creates a resource that is not covered by `createdAppIds` or `createdAgentIds`. Cleanup callbacks run after the built-in App and Agent cleanup, even when the scenario fails. + ## Reusing existing steps Before writing a new step definition, inspect the existing step definition files first. Reuse a matching step when the wording and behavior already fit, and only add a new step when the scenario needs a genuinely new user action or assertion. Steps in `common/` are designed for broad reuse across all features. @@ -304,3 +314,7 @@ The E2E web environment enables Agent v2 through `NEXT_PUBLIC_ENABLE_AGENT_V2=tr Use `support/agent.ts` for Agent v2 API fixtures. It owns roster-shaped Agent IDs, configure/access route helpers, composer draft sync, build-draft helpers, publish, API access toggles, and Agent cleanup. Store created roster Agent IDs in `DifyWorld.createdAgentIds`; the shared `After` hook deletes them after each scenario. Keep Agent v2 step definitions under `features/step-definitions/agent-v2/`. Prefer API setup for prerequisite state, then use Playwright only for user-observable navigation, editing, and assertions. + +Use `a basic configured Agent v2 test agent has been created via API` when a scenario only needs a created Agent with a composer draft. Do not use that basic shell for runtime, model, tool, skill, knowledge, environment variable, moderation, or output-variable coverage until those resources have explicit seed helpers and readiness checks. + +Use `the Agent v2 configuration should be saved automatically` after UI edits that rely on Configure autosave. It waits for the user-visible publish bar saved state; do not replace it with network-idle waits or internal store checks. From 81c96a02f419315cfff584bf86d49b54670b61b8 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 11:44:58 +0800 Subject: [PATCH 010/185] style(e2e): format check output --- .../agent-v2/configure.steps.ts | 27 +++++++++---------- .../file_tree_fixture/public/index.html | 6 ++++- e2e/support/preflight.ts | 6 ++++- e2e/support/test-materials.ts | 5 ++-- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 4b2f1f7d297d2e..e9057a2ccddacf 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -58,21 +58,18 @@ When('I open the Agent v2 configure page', async function (this: DifyWorld) { await this.getPage().goto(getAgentConfigurePath(getCurrentAgentId(this))) }) -When( - 'I open the Agent v2 configure page from the Agent Roster', - async function (this: DifyWorld) { - const page = this.getPage() - const agentId = getCurrentAgentId(this) - const agentName = this.lastCreatedAgentName - if (!agentName) - throw new Error('No Agent v2 name found. Create an Agent v2 test agent first.') - - await page.goto('/roster') - await page.getByRole('link', { name: agentName }).click() - await expect(page).toHaveURL(new RegExp(`/roster/agent/${agentId}/configure(?:\\?.*)?$`)) - await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) - }, -) +When('I open the Agent v2 configure page from the Agent Roster', async function (this: DifyWorld) { + const page = this.getPage() + const agentId = getCurrentAgentId(this) + const agentName = this.lastCreatedAgentName + if (!agentName) + throw new Error('No Agent v2 name found. Create an Agent v2 test agent first.') + + await page.goto('/roster') + await page.getByRole('link', { name: agentName }).click() + await expect(page).toHaveURL(new RegExp(`/roster/agent/${agentId}/configure(?:\\?.*)?$`)) + await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) +}) When('I discard the Agent v2 Build draft', async function (this: DifyWorld) { await this.getPage().getByRole('button', { name: 'Discard' }).click() diff --git a/e2e/fixtures/test-materials/file_tree_fixture/public/index.html b/e2e/fixtures/test-materials/file_tree_fixture/public/index.html index 1e053518f4b564..babd6315f7ab62 100644 --- a/e2e/fixtures/test-materials/file_tree_fixture/public/index.html +++ b/e2e/fixtures/test-materials/file_tree_fixture/public/index.html @@ -1,2 +1,6 @@ -E2E_FILE_TREE_INDEX + + + E2E_FILE_TREE_INDEX + + diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index 197b1a0e4fcd5e..6d3b1db17bc468 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -51,5 +51,9 @@ export function skipMissingAgentBuilderPreseed( resourceName: string, envName: string, ): 'skipped' | string { - return skipMissingEnvResource(world, envName, `Preseeded Agent Builder resource "${resourceName}"`) + return skipMissingEnvResource( + world, + envName, + `Preseeded Agent Builder resource "${resourceName}"`, + ) } diff --git a/e2e/support/test-materials.ts b/e2e/support/test-materials.ts index ccefedd7fc7842..8049114024c10f 100644 --- a/e2e/support/test-materials.ts +++ b/e2e/support/test-materials.ts @@ -29,9 +29,8 @@ export const agentBuilderTestMaterials = { countTotalExtra1: 'count_total_extra_1_valid_file', } as const -export const getAgentBuilderTestMaterialPath = ( - material: keyof typeof agentBuilderTestMaterials, -) => getTestMaterialPath(agentBuilderTestMaterials[material]) +export const getAgentBuilderTestMaterialPath = (material: keyof typeof agentBuilderTestMaterials) => + getTestMaterialPath(agentBuilderTestMaterials[material]) export async function getGeneratedTextMaterialPath({ fileName, From 833990ba6a5e35f4e260be1d65aa3e328c2a66c4 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 11:58:12 +0800 Subject: [PATCH 011/185] test(e2e): expose generated agent builder fixtures --- e2e/support/test-materials.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/e2e/support/test-materials.ts b/e2e/support/test-materials.ts index 8049114024c10f..9a9ce36fbfa315 100644 --- a/e2e/support/test-materials.ts +++ b/e2e/support/test-materials.ts @@ -29,6 +29,11 @@ export const agentBuilderTestMaterials = { countTotalExtra1: 'count_total_extra_1_valid_file', } as const +export const agentBuilderGeneratedTestMaterials = { + slowUploadFile: 'agent-slow-upload-file.txt', + tooLargeFile: 'agent-too-large-file.txt', +} as const + export const getAgentBuilderTestMaterialPath = (material: keyof typeof agentBuilderTestMaterials) => getTestMaterialPath(agentBuilderTestMaterials[material]) From 25bd0f925c90735ec7cec869be97c11959640231 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 12:01:01 +0800 Subject: [PATCH 012/185] test(e2e): add typed cleanup for agent builder resources --- e2e/AGENTS.md | 4 ++-- e2e/features/support/hooks.ts | 6 +++++- e2e/features/support/world.ts | 8 ++++++++ e2e/support/agent.ts | 12 ++++++++++++ e2e/support/datasets.ts | 12 ++++++++++++ 5 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 e2e/support/datasets.ts diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 58dc4681186e5e..cbf7a969df66e0 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -294,7 +294,7 @@ Use `fixtures/test-materials/` for checked-in files that scenarios upload, previ Use `support/preflight.ts` for scenarios that require optional external resources such as a stable model provider, plugin/tool credential, or knowledge base seed. Prefer an explicit `Given` step that returns a skipped result with a clear blocked-precondition reason over hidden setup in hooks. -Use `DifyWorld.registerCleanup(...)` when a scenario creates a resource that is not covered by `createdAppIds` or `createdAgentIds`. Cleanup callbacks run after the built-in App and Agent cleanup, even when the scenario fails. +Use `DifyWorld.createdDatasetIds` for datasets created by a scenario and `DifyWorld.createdAgentDriveFiles` for Agent drive files committed during a scenario. The shared `After` hook deletes Agent drive files before deleting created Agents so file cleanup also works for scenarios that upload into a preseeded Agent. Use `DifyWorld.registerCleanup(...)` when a scenario creates any other resource type that is not covered by the typed cleanup fields. Cleanup callbacks run after the typed cleanup queues, even when the scenario fails. ## Reusing existing steps @@ -311,7 +311,7 @@ Agent v2 scenarios live under `features/agent-v2/` and use the `@agent-v2` capab The E2E web environment enables Agent v2 through `NEXT_PUBLIC_ENABLE_AGENT_V2=true` in `scripts/common.ts`, because `/roster` routes are guarded by that feature flag. -Use `support/agent.ts` for Agent v2 API fixtures. It owns roster-shaped Agent IDs, configure/access route helpers, composer draft sync, build-draft helpers, publish, API access toggles, and Agent cleanup. Store created roster Agent IDs in `DifyWorld.createdAgentIds`; the shared `After` hook deletes them after each scenario. +Use `support/agent.ts` for Agent v2 API fixtures. It owns roster-shaped Agent IDs, configure/access route helpers, composer draft sync, build-draft helpers, publish, API access toggles, Agent drive file cleanup, and Agent cleanup. Store created roster Agent IDs in `DifyWorld.createdAgentIds`; the shared `After` hook deletes them after each scenario. Keep Agent v2 step definitions under `features/step-definitions/agent-v2/`. Prefer API setup for prerequisite state, then use Playwright only for user-observable navigation, editing, and assertions. diff --git a/e2e/features/support/hooks.ts b/e2e/features/support/hooks.ts index 4fbcdfed9d5bb9..c406f871589204 100644 --- a/e2e/features/support/hooks.ts +++ b/e2e/features/support/hooks.ts @@ -7,8 +7,9 @@ import { fileURLToPath } from 'node:url' import { After, AfterAll, Before, BeforeAll, setDefaultTimeout, Status } from '@cucumber/cucumber' import { chromium } from '@playwright/test' import { AUTH_BOOTSTRAP_TIMEOUT_MS, ensureAuthenticatedState } from '../../fixtures/auth' -import { deleteTestAgent } from '../../support/agent' +import { deleteAgentDriveFile, deleteTestAgent } from '../../support/agent' import { deleteTestApp } from '../../support/api' +import { deleteTestDataset } from '../../support/datasets' import { baseURL, cucumberHeadless, cucumberSlowMo } from '../../test-env' const e2eRoot = fileURLToPath(new URL('../..', import.meta.url)) @@ -91,6 +92,9 @@ After(async function (this: DifyWorld, { pickle, result }) { `[e2e] end ${pickle.name} status=${status}${elapsedMs ? ` durationMs=${elapsedMs}` : ''}`, ) + for (const file of this.createdAgentDriveFiles.toReversed()) + await deleteAgentDriveFile(file.agentId, file.key).catch(() => {}) + for (const id of this.createdDatasetIds) await deleteTestDataset(id).catch(() => {}) for (const id of this.createdAgentIds) await deleteTestAgent(id).catch(() => {}) for (const id of this.createdAppIds) await deleteTestApp(id).catch(() => {}) await this.runRegisteredCleanups() diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index 4d0f8427df6744..9b9191a30c6b5a 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -6,6 +6,10 @@ import { authStatePath, readAuthSessionMetadata } from '../../fixtures/auth' import { baseURL, defaultLocale } from '../../test-env' export type ScenarioCleanup = () => Promise | void +export type CreatedAgentDriveFile = { + agentId: string + key: string +} export class DifyWorld extends World { context: BrowserContext | undefined @@ -22,6 +26,8 @@ export class DifyWorld extends World { lastAgentApiReferencePage: Page | undefined createdAppIds: string[] = [] createdAgentIds: string[] = [] + createdDatasetIds: string[] = [] + createdAgentDriveFiles: CreatedAgentDriveFile[] = [] scenarioCleanups: ScenarioCleanup[] = [] capturedDownloads: Download[] = [] shareURL: string | undefined @@ -42,6 +48,8 @@ export class DifyWorld extends World { this.lastAgentApiReferencePage = undefined this.createdAppIds = [] this.createdAgentIds = [] + this.createdDatasetIds = [] + this.createdAgentDriveFiles = [] this.scenarioCleanups = [] this.capturedDownloads = [] this.shareURL = undefined diff --git a/e2e/support/agent.ts b/e2e/support/agent.ts index 95baf67fdf06f5..4afa63940e59e8 100644 --- a/e2e/support/agent.ts +++ b/e2e/support/agent.ts @@ -281,3 +281,15 @@ export async function deleteAgentApiKey(agentId: string, apiKeyId: string): Prom await ctx.dispose() } } + +export async function deleteAgentDriveFile(agentId: string, key: string): Promise { + const ctx = await createApiContext() + try { + const searchParams = new URLSearchParams({ key }) + const response = await ctx.delete(`/console/api/agent/${agentId}/files?${searchParams}`) + await expectApiResponseOK(response, `Delete Agent v2 drive file ${key} for ${agentId}`) + } + finally { + await ctx.dispose() + } +} diff --git a/e2e/support/datasets.ts b/e2e/support/datasets.ts new file mode 100644 index 00000000000000..4c0851db26d5f3 --- /dev/null +++ b/e2e/support/datasets.ts @@ -0,0 +1,12 @@ +import { createApiContext, expectApiResponseOK } from './api' + +export async function deleteTestDataset(datasetId: string): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.delete(`/console/api/datasets/${datasetId}`) + await expectApiResponseOK(response, `Delete dataset ${datasetId}`) + } + finally { + await ctx.dispose() + } +} From 45525aa190147242f826db22feede15da70366e3 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 12:03:24 +0800 Subject: [PATCH 013/185] test(e2e): verify agent builder stable model preflight --- e2e/AGENTS.md | 2 + .../agent-v2/preflight.steps.ts | 11 +++ e2e/features/support/world.ts | 7 ++ e2e/support/preflight.ts | 91 +++++++++++++++++++ 4 files changed, 111 insertions(+) create mode 100644 e2e/features/step-definitions/agent-v2/preflight.steps.ts diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index cbf7a969df66e0..99ad353919c97b 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -294,6 +294,8 @@ Use `fixtures/test-materials/` for checked-in files that scenarios upload, previ Use `support/preflight.ts` for scenarios that require optional external resources such as a stable model provider, plugin/tool credential, or knowledge base seed. Prefer an explicit `Given` step that returns a skipped result with a clear blocked-precondition reason over hidden setup in hooks. +Use `the Agent Builder stable chat model is available` before scenarios that must run an Agent with a real model. The step requires `E2E_STABLE_MODEL_PROVIDER` and `E2E_STABLE_MODEL_NAME`, defaults `E2E_STABLE_MODEL_TYPE` to `llm`, and verifies the model through `/console/api/workspaces/current/models/model-types/{type}` before storing it on `DifyWorld.agentBuilderStableChatModel`. + Use `DifyWorld.createdDatasetIds` for datasets created by a scenario and `DifyWorld.createdAgentDriveFiles` for Agent drive files committed during a scenario. The shared `After` hook deletes Agent drive files before deleting created Agents so file cleanup also works for scenarios that upload into a preseeded Agent. Use `DifyWorld.registerCleanup(...)` when a scenario creates any other resource type that is not covered by the typed cleanup fields. Cleanup callbacks run after the typed cleanup queues, even when the scenario fails. ## Reusing existing steps diff --git a/e2e/features/step-definitions/agent-v2/preflight.steps.ts b/e2e/features/step-definitions/agent-v2/preflight.steps.ts new file mode 100644 index 00000000000000..a5e1cc2656a639 --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/preflight.steps.ts @@ -0,0 +1,11 @@ +import type { DifyWorld } from '../../support/world' +import { Given } from '@cucumber/cucumber' +import { skipMissingAgentBuilderStableChatModel } from '../../../support/preflight' + +Given('the Agent Builder stable chat model is available', async function (this: DifyWorld) { + const stableModel = await skipMissingAgentBuilderStableChatModel(this) + if (stableModel === 'skipped') + return stableModel + + this.agentBuilderStableChatModel = stableModel +}) diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index 9b9191a30c6b5a..df643fc14eebcc 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -10,6 +10,11 @@ export type CreatedAgentDriveFile = { agentId: string key: string } +export type AgentBuilderStableChatModel = { + name: string + provider: string + type: string +} export class DifyWorld extends World { context: BrowserContext | undefined @@ -28,6 +33,7 @@ export class DifyWorld extends World { createdAgentIds: string[] = [] createdDatasetIds: string[] = [] createdAgentDriveFiles: CreatedAgentDriveFile[] = [] + agentBuilderStableChatModel: AgentBuilderStableChatModel | undefined scenarioCleanups: ScenarioCleanup[] = [] capturedDownloads: Download[] = [] shareURL: string | undefined @@ -50,6 +56,7 @@ export class DifyWorld extends World { this.createdAgentIds = [] this.createdDatasetIds = [] this.createdAgentDriveFiles = [] + this.agentBuilderStableChatModel = undefined this.scenarioCleanups = [] this.capturedDownloads = [] this.shareURL = undefined diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index 6d3b1db17bc468..2872d9322bdec6 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -1,5 +1,11 @@ import type { DifyWorld } from '../features/support/world' import { agentBuilderPreseededResources } from './agent-builder-resources' +import { createApiContext, expectApiResponseOK } from './api' + +const stableChatModelProviderEnv = 'E2E_STABLE_MODEL_PROVIDER' +const stableChatModelNameEnv = 'E2E_STABLE_MODEL_NAME' +const stableChatModelTypeEnv = 'E2E_STABLE_MODEL_TYPE' +const defaultStableChatModelType = 'llm' export type E2EResourcePrecondition = | { @@ -57,3 +63,88 @@ export function skipMissingAgentBuilderPreseed( `Preseeded Agent Builder resource "${resourceName}"`, ) } + +type ModelTypeListResponse = { + data: Array<{ + provider: string + models: Array<{ + label?: { + en_US?: string + zh_Hans?: string + } + model: string + status?: string + }> + status?: string + }> +} + +type StableChatModelConfig + = | { + ok: true + provider: string + type: string + value: string + } + | { + ok: false + reason: string + } + +export function readAgentBuilderStableChatModelConfig(): StableChatModelConfig { + const provider = process.env[stableChatModelProviderEnv]?.trim() + const name = process.env[stableChatModelNameEnv]?.trim() + const type = process.env[stableChatModelTypeEnv]?.trim() || defaultStableChatModelType + + const missing: string[] = [] + if (!provider) + missing.push(stableChatModelProviderEnv) + if (!name) + missing.push(stableChatModelNameEnv) + + if (!provider || !name) { + return { + ok: false, + reason: `${agentBuilderPreseededResources.stableChatModel} requires ${missing.join(', ')}.`, + } + } + + return { ok: true, provider, type, value: name } +} + +export async function skipMissingAgentBuilderStableChatModel( + world: DifyWorld, +): Promise<'skipped' | NonNullable> { + const config = readAgentBuilderStableChatModelConfig() + if (!config.ok) + return skipBlockedPrecondition(world, config.reason) + + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/workspaces/current/models/model-types/${config.type}`) + await expectApiResponseOK(response, `Check ${agentBuilderPreseededResources.stableChatModel}`) + const body = (await response.json()) as ModelTypeListResponse + const provider = body.data.find(item => item.provider === config.provider) + const model = provider?.models.find(item => + item.model === config.value + || item.label?.en_US === config.value + || item.label?.zh_Hans === config.value, + ) + + if (!provider || !model) { + return skipBlockedPrecondition( + world, + `${agentBuilderPreseededResources.stableChatModel} was not found as ${config.provider}/${config.value} (${config.type}).`, + ) + } + + return { + name: model.model, + provider: provider.provider, + type: config.type, + } + } + finally { + await ctx.dispose() + } +} From 4432ce1c9090e0ff836e1f3b48dec8844c7602d6 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 12:04:42 +0800 Subject: [PATCH 014/185] test(e2e): add agent builder resource preflight steps --- e2e/AGENTS.md | 2 + .../agent-v2/preflight.steps.ts | 40 +++++++- e2e/features/support/world.ts | 7 ++ e2e/support/preflight.ts | 96 +++++++++++++++++++ 4 files changed, 144 insertions(+), 1 deletion(-) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 99ad353919c97b..5a9005f498c94b 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -296,6 +296,8 @@ Use `support/preflight.ts` for scenarios that require optional external resource Use `the Agent Builder stable chat model is available` before scenarios that must run an Agent with a real model. The step requires `E2E_STABLE_MODEL_PROVIDER` and `E2E_STABLE_MODEL_NAME`, defaults `E2E_STABLE_MODEL_TYPE` to `llm`, and verifies the model through `/console/api/workspaces/current/models/model-types/{type}` before storing it on `DifyWorld.agentBuilderStableChatModel`. +Use `the Agent Builder preseeded Agent "{name}" is available`, `the Agent Builder preseeded workflow "{name}" is available`, and `the Agent Builder preseeded dataset "{name}" is available` when a scenario depends on a fixed environment resource. These steps verify the resource through Console APIs, store the result in `DifyWorld.agentBuilderPreseededResources`, and return `skipped` with a blocked-precondition attachment when the resource is missing. + Use `DifyWorld.createdDatasetIds` for datasets created by a scenario and `DifyWorld.createdAgentDriveFiles` for Agent drive files committed during a scenario. The shared `After` hook deletes Agent drive files before deleting created Agents so file cleanup also works for scenarios that upload into a preseeded Agent. Use `DifyWorld.registerCleanup(...)` when a scenario creates any other resource type that is not covered by the typed cleanup fields. Cleanup callbacks run after the typed cleanup queues, even when the scenario fails. ## Reusing existing steps diff --git a/e2e/features/step-definitions/agent-v2/preflight.steps.ts b/e2e/features/step-definitions/agent-v2/preflight.steps.ts index a5e1cc2656a639..5ac2f958cfb790 100644 --- a/e2e/features/step-definitions/agent-v2/preflight.steps.ts +++ b/e2e/features/step-definitions/agent-v2/preflight.steps.ts @@ -1,6 +1,11 @@ import type { DifyWorld } from '../../support/world' import { Given } from '@cucumber/cucumber' -import { skipMissingAgentBuilderStableChatModel } from '../../../support/preflight' +import { + skipMissingAgentBuilderStableChatModel, + skipMissingPreseededAgent, + skipMissingPreseededDataset, + skipMissingPreseededWorkflow, +} from '../../../support/preflight' Given('the Agent Builder stable chat model is available', async function (this: DifyWorld) { const stableModel = await skipMissingAgentBuilderStableChatModel(this) @@ -9,3 +14,36 @@ Given('the Agent Builder stable chat model is available', async function (this: this.agentBuilderStableChatModel = stableModel }) + +Given( + 'the Agent Builder preseeded Agent {string} is available', + async function (this: DifyWorld, resourceName: string) { + const resource = await skipMissingPreseededAgent(this, resourceName) + if (resource === 'skipped') + return resource + + this.agentBuilderPreseededResources[resourceName] = resource + }, +) + +Given( + 'the Agent Builder preseeded workflow {string} is available', + async function (this: DifyWorld, resourceName: string) { + const resource = await skipMissingPreseededWorkflow(this, resourceName) + if (resource === 'skipped') + return resource + + this.agentBuilderPreseededResources[resourceName] = resource + }, +) + +Given( + 'the Agent Builder preseeded dataset {string} is available', + async function (this: DifyWorld, resourceName: string) { + const resource = await skipMissingPreseededDataset(this, resourceName) + if (resource === 'skipped') + return resource + + this.agentBuilderPreseededResources[resourceName] = resource + }, +) diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index df643fc14eebcc..fb30cbc6f0638d 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -15,6 +15,11 @@ export type AgentBuilderStableChatModel = { provider: string type: string } +export type AgentBuilderPreseededResource = { + id: string + kind: 'agent' | 'dataset' | 'workflow' + name: string +} export class DifyWorld extends World { context: BrowserContext | undefined @@ -34,6 +39,7 @@ export class DifyWorld extends World { createdDatasetIds: string[] = [] createdAgentDriveFiles: CreatedAgentDriveFile[] = [] agentBuilderStableChatModel: AgentBuilderStableChatModel | undefined + agentBuilderPreseededResources: Record = {} scenarioCleanups: ScenarioCleanup[] = [] capturedDownloads: Download[] = [] shareURL: string | undefined @@ -57,6 +63,7 @@ export class DifyWorld extends World { this.createdDatasetIds = [] this.createdAgentDriveFiles = [] this.agentBuilderStableChatModel = undefined + this.agentBuilderPreseededResources = {} this.scenarioCleanups = [] this.capturedDownloads = [] this.shareURL = undefined diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index 2872d9322bdec6..9373c36e236f10 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -79,6 +79,102 @@ type ModelTypeListResponse = { }> } +type NamedResource = { + id: string + name: string +} + +type NamedResourceListResponse = { + data: NamedResource[] +} + +const findConsoleResourceByName = async ({ + action, + path, + resourceName, +}: { + action: string + path: string + resourceName: string +}) => { + const ctx = await createApiContext() + try { + const response = await ctx.get(path) + await expectApiResponseOK(response, action) + const body = (await response.json()) as NamedResourceListResponse + + return body.data.find(item => item.name === resourceName) + } + finally { + await ctx.dispose() + } +} + +const buildQuery = (params: Record) => new URLSearchParams(params).toString() + +export async function skipMissingPreseededAgent( + world: DifyWorld, + resourceName: string, +): Promise<'skipped' | NonNullable> { + const query = buildQuery({ limit: '20', name: resourceName, page: '1' }) + const resource = await findConsoleResourceByName({ + action: `Check preseeded Agent ${resourceName}`, + path: `/console/api/agent?${query}`, + resourceName, + }) + + if (!resource) + return skipBlockedPrecondition(world, `Preseeded Agent "${resourceName}" was not found.`) + + return { + id: resource.id, + kind: 'agent', + name: resource.name, + } +} + +export async function skipMissingPreseededWorkflow( + world: DifyWorld, + resourceName: string, +): Promise<'skipped' | NonNullable> { + const query = buildQuery({ limit: '20', mode: 'workflow', name: resourceName, page: '1' }) + const resource = await findConsoleResourceByName({ + action: `Check preseeded workflow ${resourceName}`, + path: `/console/api/apps?${query}`, + resourceName, + }) + + if (!resource) + return skipBlockedPrecondition(world, `Preseeded workflow "${resourceName}" was not found.`) + + return { + id: resource.id, + kind: 'workflow', + name: resource.name, + } +} + +export async function skipMissingPreseededDataset( + world: DifyWorld, + resourceName: string, +): Promise<'skipped' | NonNullable> { + const query = buildQuery({ keyword: resourceName, limit: '20', page: '1' }) + const resource = await findConsoleResourceByName({ + action: `Check preseeded dataset ${resourceName}`, + path: `/console/api/datasets?${query}`, + resourceName, + }) + + if (!resource) + return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`) + + return { + id: resource.id, + kind: 'dataset', + name: resource.name, + } +} + type StableChatModelConfig = | { ok: true From 6b201b3ee3a0736fe2175042979dcaff5d304fb3 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 12:05:31 +0800 Subject: [PATCH 015/185] style(e2e): format agent builder support files --- e2e/support/preflight.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index 9373c36e236f10..d51129dddfab6c 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -217,14 +217,17 @@ export async function skipMissingAgentBuilderStableChatModel( const ctx = await createApiContext() try { - const response = await ctx.get(`/console/api/workspaces/current/models/model-types/${config.type}`) + const response = await ctx.get( + `/console/api/workspaces/current/models/model-types/${config.type}`, + ) await expectApiResponseOK(response, `Check ${agentBuilderPreseededResources.stableChatModel}`) const body = (await response.json()) as ModelTypeListResponse const provider = body.data.find(item => item.provider === config.provider) - const model = provider?.models.find(item => - item.model === config.value - || item.label?.en_US === config.value - || item.label?.zh_Hans === config.value, + const model = provider?.models.find( + item => + item.model === config.value + || item.label?.en_US === config.value + || item.label?.zh_Hans === config.value, ) if (!provider || !model) { From aa0e4393a326674531d5e03487bd87f0899b109b Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 12:13:54 +0800 Subject: [PATCH 016/185] test(e2e): add agent builder tool preflight --- e2e/AGENTS.md | 2 +- .../agent-v2/preflight.steps.ts | 12 ++++ e2e/features/support/world.ts | 2 +- e2e/support/preflight.ts | 68 +++++++++++++++++++ 4 files changed, 82 insertions(+), 2 deletions(-) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 5a9005f498c94b..ccecdb45dead0a 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -296,7 +296,7 @@ Use `support/preflight.ts` for scenarios that require optional external resource Use `the Agent Builder stable chat model is available` before scenarios that must run an Agent with a real model. The step requires `E2E_STABLE_MODEL_PROVIDER` and `E2E_STABLE_MODEL_NAME`, defaults `E2E_STABLE_MODEL_TYPE` to `llm`, and verifies the model through `/console/api/workspaces/current/models/model-types/{type}` before storing it on `DifyWorld.agentBuilderStableChatModel`. -Use `the Agent Builder preseeded Agent "{name}" is available`, `the Agent Builder preseeded workflow "{name}" is available`, and `the Agent Builder preseeded dataset "{name}" is available` when a scenario depends on a fixed environment resource. These steps verify the resource through Console APIs, store the result in `DifyWorld.agentBuilderPreseededResources`, and return `skipped` with a blocked-precondition attachment when the resource is missing. +Use `the Agent Builder preseeded Agent "{name}" is available`, `the Agent Builder preseeded workflow "{name}" is available`, `the Agent Builder preseeded dataset "{name}" is available`, and `the Agent Builder preseeded tool "{provider} / {tool}" is available` when a scenario depends on a fixed environment resource. These steps verify the resource through Console APIs, store the result in `DifyWorld.agentBuilderPreseededResources`, and return `skipped` with a blocked-precondition attachment when the resource is missing. The tool step checks built-in tool availability only; credential-validity scenarios still need explicit credential-state setup or assertions. Use `DifyWorld.createdDatasetIds` for datasets created by a scenario and `DifyWorld.createdAgentDriveFiles` for Agent drive files committed during a scenario. The shared `After` hook deletes Agent drive files before deleting created Agents so file cleanup also works for scenarios that upload into a preseeded Agent. Use `DifyWorld.registerCleanup(...)` when a scenario creates any other resource type that is not covered by the typed cleanup fields. Cleanup callbacks run after the typed cleanup queues, even when the scenario fails. diff --git a/e2e/features/step-definitions/agent-v2/preflight.steps.ts b/e2e/features/step-definitions/agent-v2/preflight.steps.ts index 5ac2f958cfb790..c579922f22aa74 100644 --- a/e2e/features/step-definitions/agent-v2/preflight.steps.ts +++ b/e2e/features/step-definitions/agent-v2/preflight.steps.ts @@ -4,6 +4,7 @@ import { skipMissingAgentBuilderStableChatModel, skipMissingPreseededAgent, skipMissingPreseededDataset, + skipMissingPreseededTool, skipMissingPreseededWorkflow, } from '../../../support/preflight' @@ -47,3 +48,14 @@ Given( this.agentBuilderPreseededResources[resourceName] = resource }, ) + +Given( + 'the Agent Builder preseeded tool {string} is available', + async function (this: DifyWorld, resourceName: string) { + const resource = await skipMissingPreseededTool(this, resourceName) + if (resource === 'skipped') + return resource + + this.agentBuilderPreseededResources[resourceName] = resource + }, +) diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index fb30cbc6f0638d..f5998f346334fc 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -17,7 +17,7 @@ export type AgentBuilderStableChatModel = { } export type AgentBuilderPreseededResource = { id: string - kind: 'agent' | 'dataset' | 'workflow' + kind: 'agent' | 'dataset' | 'tool' | 'workflow' name: string } diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index d51129dddfab6c..dae0930434da2d 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -88,6 +88,20 @@ type NamedResourceListResponse = { data: NamedResource[] } +type LocalizedLabel = { + en_US?: string + zh_Hans?: string +} + +type BuiltinToolProvider = { + label?: LocalizedLabel + name: string + tools: Array<{ + label?: LocalizedLabel + name: string + }> +} + const findConsoleResourceByName = async ({ action, path, @@ -112,6 +126,26 @@ const findConsoleResourceByName = async ({ const buildQuery = (params: Record) => new URLSearchParams(params).toString() +const matchesNameOrLabel = (value: string, name: string, label?: LocalizedLabel) => + value === name || value === label?.en_US || value === label?.zh_Hans + +const splitToolDisplayName = (resourceName: string) => { + const [providerName, toolName] = resourceName.split('/').map(item => item.trim()) + + if (!providerName || !toolName) { + return { + ok: false as const, + reason: `Preseeded tool "${resourceName}" must use "Provider / Tool" format.`, + } + } + + return { + ok: true as const, + providerName, + toolName, + } +} + export async function skipMissingPreseededAgent( world: DifyWorld, resourceName: string, @@ -175,6 +209,40 @@ export async function skipMissingPreseededDataset( } } +export async function skipMissingPreseededTool( + world: DifyWorld, + resourceName: string, +): Promise<'skipped' | NonNullable> { + const parsed = splitToolDisplayName(resourceName) + if (!parsed.ok) + return skipBlockedPrecondition(world, parsed.reason) + + const ctx = await createApiContext() + try { + const response = await ctx.get('/console/api/workspaces/current/tools/builtin') + await expectApiResponseOK(response, `Check preseeded tool ${resourceName}`) + const providers = (await response.json()) as BuiltinToolProvider[] + const provider = providers.find(item => + matchesNameOrLabel(parsed.providerName, item.name, item.label), + ) + const tool = provider?.tools.find(item => + matchesNameOrLabel(parsed.toolName, item.name, item.label), + ) + + if (!provider || !tool) + return skipBlockedPrecondition(world, `Preseeded tool "${resourceName}" was not found.`) + + return { + id: `${provider.name}/${tool.name}`, + kind: 'tool', + name: resourceName, + } + } + finally { + await ctx.dispose() + } +} + type StableChatModelConfig = | { ok: true From ea5e00c6184797912b6a5786904c5068dc55f43b Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 12:17:39 +0800 Subject: [PATCH 017/185] test(e2e): add agent builder broken model preflight --- e2e/AGENTS.md | 2 + .../agent-v2/preflight.steps.ts | 9 +++ e2e/features/support/world.ts | 2 + e2e/support/preflight.ts | 58 ++++++++++++++++--- 4 files changed, 64 insertions(+), 7 deletions(-) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index ccecdb45dead0a..f7373d1ce34cd2 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -296,6 +296,8 @@ Use `support/preflight.ts` for scenarios that require optional external resource Use `the Agent Builder stable chat model is available` before scenarios that must run an Agent with a real model. The step requires `E2E_STABLE_MODEL_PROVIDER` and `E2E_STABLE_MODEL_NAME`, defaults `E2E_STABLE_MODEL_TYPE` to `llm`, and verifies the model through `/console/api/workspaces/current/models/model-types/{type}` before storing it on `DifyWorld.agentBuilderStableChatModel`. +Use `the Agent Builder broken chat model is available` before model-recovery scenarios that intentionally start from an invalid model. The step requires `E2E_BROKEN_MODEL_PROVIDER`, defaults `E2E_BROKEN_MODEL_NAME` to `e2e-broken-model`, defaults `E2E_BROKEN_MODEL_TYPE` to `llm`, and only verifies that the model entry exists. The scenario must still assert the user-visible failure and recovery behavior. + Use `the Agent Builder preseeded Agent "{name}" is available`, `the Agent Builder preseeded workflow "{name}" is available`, `the Agent Builder preseeded dataset "{name}" is available`, and `the Agent Builder preseeded tool "{provider} / {tool}" is available` when a scenario depends on a fixed environment resource. These steps verify the resource through Console APIs, store the result in `DifyWorld.agentBuilderPreseededResources`, and return `skipped` with a blocked-precondition attachment when the resource is missing. The tool step checks built-in tool availability only; credential-validity scenarios still need explicit credential-state setup or assertions. Use `DifyWorld.createdDatasetIds` for datasets created by a scenario and `DifyWorld.createdAgentDriveFiles` for Agent drive files committed during a scenario. The shared `After` hook deletes Agent drive files before deleting created Agents so file cleanup also works for scenarios that upload into a preseeded Agent. Use `DifyWorld.registerCleanup(...)` when a scenario creates any other resource type that is not covered by the typed cleanup fields. Cleanup callbacks run after the typed cleanup queues, even when the scenario fails. diff --git a/e2e/features/step-definitions/agent-v2/preflight.steps.ts b/e2e/features/step-definitions/agent-v2/preflight.steps.ts index c579922f22aa74..38c73a672941d9 100644 --- a/e2e/features/step-definitions/agent-v2/preflight.steps.ts +++ b/e2e/features/step-definitions/agent-v2/preflight.steps.ts @@ -1,6 +1,7 @@ import type { DifyWorld } from '../../support/world' import { Given } from '@cucumber/cucumber' import { + skipMissingAgentBuilderBrokenChatModel, skipMissingAgentBuilderStableChatModel, skipMissingPreseededAgent, skipMissingPreseededDataset, @@ -16,6 +17,14 @@ Given('the Agent Builder stable chat model is available', async function (this: this.agentBuilderStableChatModel = stableModel }) +Given('the Agent Builder broken chat model is available', async function (this: DifyWorld) { + const brokenModel = await skipMissingAgentBuilderBrokenChatModel(this) + if (brokenModel === 'skipped') + return brokenModel + + this.agentBuilderBrokenChatModel = brokenModel +}) + Given( 'the Agent Builder preseeded Agent {string} is available', async function (this: DifyWorld, resourceName: string) { diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index f5998f346334fc..c48f16ce6bea19 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -38,6 +38,7 @@ export class DifyWorld extends World { createdAgentIds: string[] = [] createdDatasetIds: string[] = [] createdAgentDriveFiles: CreatedAgentDriveFile[] = [] + agentBuilderBrokenChatModel: AgentBuilderStableChatModel | undefined agentBuilderStableChatModel: AgentBuilderStableChatModel | undefined agentBuilderPreseededResources: Record = {} scenarioCleanups: ScenarioCleanup[] = [] @@ -62,6 +63,7 @@ export class DifyWorld extends World { this.createdAgentIds = [] this.createdDatasetIds = [] this.createdAgentDriveFiles = [] + this.agentBuilderBrokenChatModel = undefined this.agentBuilderStableChatModel = undefined this.agentBuilderPreseededResources = {} this.scenarioCleanups = [] diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index dae0930434da2d..75aaa343472152 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -5,7 +5,11 @@ import { createApiContext, expectApiResponseOK } from './api' const stableChatModelProviderEnv = 'E2E_STABLE_MODEL_PROVIDER' const stableChatModelNameEnv = 'E2E_STABLE_MODEL_NAME' const stableChatModelTypeEnv = 'E2E_STABLE_MODEL_TYPE' +const brokenChatModelProviderEnv = 'E2E_BROKEN_MODEL_PROVIDER' +const brokenChatModelNameEnv = 'E2E_BROKEN_MODEL_NAME' +const brokenChatModelTypeEnv = 'E2E_BROKEN_MODEL_TYPE' const defaultStableChatModelType = 'llm' +const defaultBrokenChatModelName = agentBuilderPreseededResources.brokenModel export type E2EResourcePrecondition = | { @@ -243,10 +247,11 @@ export async function skipMissingPreseededTool( } } -type StableChatModelConfig +type ModelPreflightConfig = | { ok: true provider: string + resourceName: string type: string value: string } @@ -255,7 +260,7 @@ type StableChatModelConfig reason: string } -export function readAgentBuilderStableChatModelConfig(): StableChatModelConfig { +export function readAgentBuilderStableChatModelConfig(): ModelPreflightConfig { const provider = process.env[stableChatModelProviderEnv]?.trim() const name = process.env[stableChatModelNameEnv]?.trim() const type = process.env[stableChatModelTypeEnv]?.trim() || defaultStableChatModelType @@ -273,13 +278,40 @@ export function readAgentBuilderStableChatModelConfig(): StableChatModelConfig { } } - return { ok: true, provider, type, value: name } + return { + ok: true, + provider, + resourceName: agentBuilderPreseededResources.stableChatModel, + type, + value: name, + } } -export async function skipMissingAgentBuilderStableChatModel( +export function readAgentBuilderBrokenChatModelConfig(): ModelPreflightConfig { + const provider = process.env[brokenChatModelProviderEnv]?.trim() + const name = process.env[brokenChatModelNameEnv]?.trim() || defaultBrokenChatModelName + const type = process.env[brokenChatModelTypeEnv]?.trim() || defaultStableChatModelType + + if (!provider) { + return { + ok: false, + reason: `${agentBuilderPreseededResources.brokenModelProvider} requires ${brokenChatModelProviderEnv}.`, + } + } + + return { + ok: true, + provider, + resourceName: agentBuilderPreseededResources.brokenModelProvider, + type, + value: name, + } +} + +async function skipMissingAgentBuilderModel( world: DifyWorld, + config: ModelPreflightConfig, ): Promise<'skipped' | NonNullable> { - const config = readAgentBuilderStableChatModelConfig() if (!config.ok) return skipBlockedPrecondition(world, config.reason) @@ -288,7 +320,7 @@ export async function skipMissingAgentBuilderStableChatModel( const response = await ctx.get( `/console/api/workspaces/current/models/model-types/${config.type}`, ) - await expectApiResponseOK(response, `Check ${agentBuilderPreseededResources.stableChatModel}`) + await expectApiResponseOK(response, `Check ${config.resourceName}`) const body = (await response.json()) as ModelTypeListResponse const provider = body.data.find(item => item.provider === config.provider) const model = provider?.models.find( @@ -301,7 +333,7 @@ export async function skipMissingAgentBuilderStableChatModel( if (!provider || !model) { return skipBlockedPrecondition( world, - `${agentBuilderPreseededResources.stableChatModel} was not found as ${config.provider}/${config.value} (${config.type}).`, + `${config.resourceName} was not found as ${config.provider}/${config.value} (${config.type}).`, ) } @@ -315,3 +347,15 @@ export async function skipMissingAgentBuilderStableChatModel( await ctx.dispose() } } + +export async function skipMissingAgentBuilderStableChatModel( + world: DifyWorld, +): Promise<'skipped' | NonNullable> { + return skipMissingAgentBuilderModel(world, readAgentBuilderStableChatModelConfig()) +} + +export async function skipMissingAgentBuilderBrokenChatModel( + world: DifyWorld, +): Promise<'skipped' | NonNullable> { + return skipMissingAgentBuilderModel(world, readAgentBuilderBrokenChatModelConfig()) +} From 79750c50e7a9c09a5f28b567a765403f9c4af549 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 12:18:53 +0800 Subject: [PATCH 018/185] test(e2e): add agent attached resource preflight --- e2e/AGENTS.md | 2 + .../agent-v2/preflight.steps.ts | 24 +++++ e2e/features/support/world.ts | 2 +- e2e/support/preflight.ts | 94 +++++++++++++++++++ 4 files changed, 121 insertions(+), 1 deletion(-) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index f7373d1ce34cd2..0686961ef62af3 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -300,6 +300,8 @@ Use `the Agent Builder broken chat model is available` before model-recovery sce Use `the Agent Builder preseeded Agent "{name}" is available`, `the Agent Builder preseeded workflow "{name}" is available`, `the Agent Builder preseeded dataset "{name}" is available`, and `the Agent Builder preseeded tool "{provider} / {tool}" is available` when a scenario depends on a fixed environment resource. These steps verify the resource through Console APIs, store the result in `DifyWorld.agentBuilderPreseededResources`, and return `skipped` with a blocked-precondition attachment when the resource is missing. The tool step checks built-in tool availability only; credential-validity scenarios still need explicit credential-state setup or assertions. +Use `the Agent Builder preseeded Agent "{agent}" includes drive skill "{skill}"` to verify that a fixed Agent has a drive-backed Skill attached. Use `the Agent Builder preseeded Agent "{agent}" has Backend service API access with an API key` to verify that a fixed Agent has Backend service API enabled and at least one key. The API key step does not validate a human-readable key name because the Console API key response does not expose one. + Use `DifyWorld.createdDatasetIds` for datasets created by a scenario and `DifyWorld.createdAgentDriveFiles` for Agent drive files committed during a scenario. The shared `After` hook deletes Agent drive files before deleting created Agents so file cleanup also works for scenarios that upload into a preseeded Agent. Use `DifyWorld.registerCleanup(...)` when a scenario creates any other resource type that is not covered by the typed cleanup fields. Cleanup callbacks run after the typed cleanup queues, even when the scenario fails. ## Reusing existing steps diff --git a/e2e/features/step-definitions/agent-v2/preflight.steps.ts b/e2e/features/step-definitions/agent-v2/preflight.steps.ts index 38c73a672941d9..2984499b66ce8e 100644 --- a/e2e/features/step-definitions/agent-v2/preflight.steps.ts +++ b/e2e/features/step-definitions/agent-v2/preflight.steps.ts @@ -4,6 +4,8 @@ import { skipMissingAgentBuilderBrokenChatModel, skipMissingAgentBuilderStableChatModel, skipMissingPreseededAgent, + skipMissingPreseededAgentBackendApiKey, + skipMissingPreseededAgentDriveSkill, skipMissingPreseededDataset, skipMissingPreseededTool, skipMissingPreseededWorkflow, @@ -68,3 +70,25 @@ Given( this.agentBuilderPreseededResources[resourceName] = resource }, ) + +Given( + 'the Agent Builder preseeded Agent {string} includes drive skill {string}', + async function (this: DifyWorld, agentName: string, skillName: string) { + const resource = await skipMissingPreseededAgentDriveSkill(this, agentName, skillName) + if (resource === 'skipped') + return resource + + this.agentBuilderPreseededResources[`${agentName} / ${skillName}`] = resource + }, +) + +Given( + 'the Agent Builder preseeded Agent {string} has Backend service API access with an API key', + async function (this: DifyWorld, agentName: string) { + const resource = await skipMissingPreseededAgentBackendApiKey(this, agentName) + if (resource === 'skipped') + return resource + + this.agentBuilderPreseededResources[`${agentName} / Backend service API key`] = resource + }, +) diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index c48f16ce6bea19..bb7a4dbfcaf175 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -17,7 +17,7 @@ export type AgentBuilderStableChatModel = { } export type AgentBuilderPreseededResource = { id: string - kind: 'agent' | 'dataset' | 'tool' | 'workflow' + kind: 'agent' | 'api-key' | 'dataset' | 'skill' | 'tool' | 'workflow' name: string } diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index 75aaa343472152..3e8b270e418b0b 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -106,6 +106,24 @@ type BuiltinToolProvider = { }> } +type AgentDriveSkillListResponse = { + items: Array<{ + name: string + path: string + }> +} + +type AgentApiAccessResponse = { + api_key_count: number + enabled: boolean +} + +type AgentApiKeyListResponse = { + data: Array<{ + id: string + }> +} + const findConsoleResourceByName = async ({ action, path, @@ -247,6 +265,82 @@ export async function skipMissingPreseededTool( } } +export async function skipMissingPreseededAgentDriveSkill( + world: DifyWorld, + agentName: string, + skillName: string, +): Promise<'skipped' | NonNullable> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agent.id}/drive/skills`) + await expectApiResponseOK(response, `Check preseeded Agent skill ${skillName}`) + const body = (await response.json()) as AgentDriveSkillListResponse + const skill = body.items.find(item => item.name === skillName) + + if (!skill) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" does not include drive skill "${skillName}".`, + ) + } + + return { + id: skill.path, + kind: 'skill', + name: skill.name, + } + } + finally { + await ctx.dispose() + } +} + +export async function skipMissingPreseededAgentBackendApiKey( + world: DifyWorld, + agentName: string, +): Promise<'skipped' | NonNullable> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const ctx = await createApiContext() + try { + const accessResponse = await ctx.get(`/console/api/agent/${agent.id}/api-access`) + await expectApiResponseOK(accessResponse, `Check preseeded Agent API access ${agentName}`) + const access = (await accessResponse.json()) as AgentApiAccessResponse + if (!access.enabled || access.api_key_count < 1) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" does not have Backend service API enabled with an API key.`, + ) + } + + const keyResponse = await ctx.get(`/console/api/agent/${agent.id}/api-keys`) + await expectApiResponseOK(keyResponse, `Check preseeded Agent API key ${agentName}`) + const keys = (await keyResponse.json()) as AgentApiKeyListResponse + const key = keys.data.at(0) + if (!key) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" Backend service API key list is empty.`, + ) + } + + return { + id: key.id, + kind: 'api-key', + name: `${agentName} Backend service API key`, + } + } + finally { + await ctx.dispose() + } +} + type ModelPreflightConfig = | { ok: true From 33dfaf8c868645fc7cdac3dfa1a7f465d70b0796 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 12:26:38 +0800 Subject: [PATCH 019/185] test(e2e): add agent builder tool cleanup contract --- e2e/AGENTS.md | 4 ++-- e2e/features/support/hooks.ts | 5 ++++- e2e/features/support/world.ts | 6 ++++++ e2e/support/tools.ts | 23 +++++++++++++++++++++++ 4 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 e2e/support/tools.ts diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 0686961ef62af3..87657e657fc716 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -294,7 +294,7 @@ Use `fixtures/test-materials/` for checked-in files that scenarios upload, previ Use `support/preflight.ts` for scenarios that require optional external resources such as a stable model provider, plugin/tool credential, or knowledge base seed. Prefer an explicit `Given` step that returns a skipped result with a clear blocked-precondition reason over hidden setup in hooks. -Use `the Agent Builder stable chat model is available` before scenarios that must run an Agent with a real model. The step requires `E2E_STABLE_MODEL_PROVIDER` and `E2E_STABLE_MODEL_NAME`, defaults `E2E_STABLE_MODEL_TYPE` to `llm`, and verifies the model through `/console/api/workspaces/current/models/model-types/{type}` before storing it on `DifyWorld.agentBuilderStableChatModel`. +Use `the Agent Builder stable chat model is available` before scenarios that must run an Agent with a real model. The step requires `E2E_STABLE_MODEL_PROVIDER` and `E2E_STABLE_MODEL_NAME`, defaults `E2E_STABLE_MODEL_TYPE` to `llm`, and verifies the model through `/console/api/workspaces/current/models/model-types/{type}` before storing it on `DifyWorld.agentBuilderStableChatModel`. Any Agent Builder scenario that needs a usable model should explicitly apply this stored model during its own setup instead of hard-coding provider or model names in feature files or hooks. Use `the Agent Builder broken chat model is available` before model-recovery scenarios that intentionally start from an invalid model. The step requires `E2E_BROKEN_MODEL_PROVIDER`, defaults `E2E_BROKEN_MODEL_NAME` to `e2e-broken-model`, defaults `E2E_BROKEN_MODEL_TYPE` to `llm`, and only verifies that the model entry exists. The scenario must still assert the user-visible failure and recovery behavior. @@ -302,7 +302,7 @@ Use `the Agent Builder preseeded Agent "{name}" is available`, `the Agent Builde Use `the Agent Builder preseeded Agent "{agent}" includes drive skill "{skill}"` to verify that a fixed Agent has a drive-backed Skill attached. Use `the Agent Builder preseeded Agent "{agent}" has Backend service API access with an API key` to verify that a fixed Agent has Backend service API enabled and at least one key. The API key step does not validate a human-readable key name because the Console API key response does not expose one. -Use `DifyWorld.createdDatasetIds` for datasets created by a scenario and `DifyWorld.createdAgentDriveFiles` for Agent drive files committed during a scenario. The shared `After` hook deletes Agent drive files before deleting created Agents so file cleanup also works for scenarios that upload into a preseeded Agent. Use `DifyWorld.registerCleanup(...)` when a scenario creates any other resource type that is not covered by the typed cleanup fields. Cleanup callbacks run after the typed cleanup queues, even when the scenario fails. +Use `DifyWorld.createdDatasetIds` for datasets created by a scenario, `DifyWorld.createdAgentDriveFiles` for Agent drive files committed during a scenario, and `DifyWorld.createdBuiltinToolCredentials` for built-in tool credentials created during a scenario. The shared `After` hook deletes Agent drive files first so file cleanup also works for scenarios that upload into a preseeded Agent, then deletes created Agents and Apps before deleting dependent datasets and tool credentials. Use `DifyWorld.registerCleanup(...)` when a scenario creates any other resource type that is not covered by the typed cleanup fields. Cleanup callbacks run after the typed cleanup queues, even when the scenario fails. ## Reusing existing steps diff --git a/e2e/features/support/hooks.ts b/e2e/features/support/hooks.ts index c406f871589204..69848b91bbe38d 100644 --- a/e2e/features/support/hooks.ts +++ b/e2e/features/support/hooks.ts @@ -10,6 +10,7 @@ import { AUTH_BOOTSTRAP_TIMEOUT_MS, ensureAuthenticatedState } from '../../fixtu import { deleteAgentDriveFile, deleteTestAgent } from '../../support/agent' import { deleteTestApp } from '../../support/api' import { deleteTestDataset } from '../../support/datasets' +import { deleteBuiltinToolCredential } from '../../support/tools' import { baseURL, cucumberHeadless, cucumberSlowMo } from '../../test-env' const e2eRoot = fileURLToPath(new URL('../..', import.meta.url)) @@ -94,9 +95,11 @@ After(async function (this: DifyWorld, { pickle, result }) { for (const file of this.createdAgentDriveFiles.toReversed()) await deleteAgentDriveFile(file.agentId, file.key).catch(() => {}) - for (const id of this.createdDatasetIds) await deleteTestDataset(id).catch(() => {}) for (const id of this.createdAgentIds) await deleteTestAgent(id).catch(() => {}) for (const id of this.createdAppIds) await deleteTestApp(id).catch(() => {}) + for (const id of this.createdDatasetIds) await deleteTestDataset(id).catch(() => {}) + for (const credential of this.createdBuiltinToolCredentials.toReversed()) + await deleteBuiltinToolCredential(credential.provider, credential.credentialId).catch(() => {}) await this.runRegisteredCleanups() await this.closeSession() diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index bb7a4dbfcaf175..a749f0197920e1 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -10,6 +10,10 @@ export type CreatedAgentDriveFile = { agentId: string key: string } +export type CreatedBuiltinToolCredential = { + credentialId: string + provider: string +} export type AgentBuilderStableChatModel = { name: string provider: string @@ -38,6 +42,7 @@ export class DifyWorld extends World { createdAgentIds: string[] = [] createdDatasetIds: string[] = [] createdAgentDriveFiles: CreatedAgentDriveFile[] = [] + createdBuiltinToolCredentials: CreatedBuiltinToolCredential[] = [] agentBuilderBrokenChatModel: AgentBuilderStableChatModel | undefined agentBuilderStableChatModel: AgentBuilderStableChatModel | undefined agentBuilderPreseededResources: Record = {} @@ -63,6 +68,7 @@ export class DifyWorld extends World { this.createdAgentIds = [] this.createdDatasetIds = [] this.createdAgentDriveFiles = [] + this.createdBuiltinToolCredentials = [] this.agentBuilderBrokenChatModel = undefined this.agentBuilderStableChatModel = undefined this.agentBuilderPreseededResources = {} diff --git a/e2e/support/tools.ts b/e2e/support/tools.ts new file mode 100644 index 00000000000000..c4bee2278bb3ad --- /dev/null +++ b/e2e/support/tools.ts @@ -0,0 +1,23 @@ +import { createApiContext, expectApiResponseOK } from './api' + +export async function deleteBuiltinToolCredential( + provider: string, + credentialId: string, +): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.post( + `/console/api/workspaces/current/tool-provider/builtin/${provider}/delete`, + { + data: { credential_id: credentialId }, + }, + ) + await expectApiResponseOK( + response, + `Delete built-in tool credential ${credentialId} for ${provider}`, + ) + } + finally { + await ctx.dispose() + } +} From ccacfdb771b428aea4e5c760776a66ad7f9d390e Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 12:33:08 +0800 Subject: [PATCH 020/185] test(e2e): align package check with eslint --- e2e/AGENTS.md | 2 +- e2e/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 87657e657fc716..51c151fe9772d6 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -31,7 +31,7 @@ pnpm -C e2e check `pnpm install` is resolved through the repository workspace and uses the shared root lockfile plus `pnpm-workspace.yaml`. -Use `pnpm -C e2e check` as the default local verification step after editing E2E TypeScript, Cucumber support code, or feature glue. It runs formatting, linting, and type checks for this package. +Use `pnpm -C e2e check` as the default local verification step after editing E2E TypeScript, Cucumber support code, or feature glue. It runs ESLint autofix, linting, and type checks for this package. Common commands: diff --git a/e2e/package.json b/e2e/package.json index 77d7db80f0ae63..800b6ca33ab6ea 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -3,7 +3,7 @@ "type": "module", "private": true, "scripts": { - "check": "vp check --fix", + "check": "eslint --fix . && tsgo --noEmit --pretty false", "e2e": "tsx ./scripts/run-cucumber.ts", "e2e:full": "tsx ./scripts/run-cucumber.ts --full", "e2e:full:headed": "tsx ./scripts/run-cucumber.ts --full --headed", From a7592649196e9e6948342ccf0e268d04e3146673 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 12:35:03 +0800 Subject: [PATCH 021/185] test(e2e): add stable model agent fixture --- e2e/AGENTS.md | 2 + .../agent-v2/configure.steps.ts | 19 ++++++++++ e2e/support/agent.ts | 37 +++++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 51c151fe9772d6..8c9d9eb5a79ae2 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -325,4 +325,6 @@ Keep Agent v2 step definitions under `features/step-definitions/agent-v2/`. Pref Use `a basic configured Agent v2 test agent has been created via API` when a scenario only needs a created Agent with a composer draft. Do not use that basic shell for runtime, model, tool, skill, knowledge, environment variable, moderation, or output-variable coverage until those resources have explicit seed helpers and readiness checks. +Use `a runnable Agent v2 test agent has been created via API` after `the Agent Builder stable chat model is available` when a scenario needs a real model-backed Agent. The step writes the preflight model into the Agent Soul model config through `support/agent.ts`; do not duplicate provider/model payload construction in individual steps. + Use `the Agent v2 configuration should be saved automatically` after UI edits that rely on Configure autosave. It waits for the user-visible publish bar saved state; do not replace it with network-idle waits or internal store checks. diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index e9057a2ccddacf..a7fd68d76b6c30 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -2,6 +2,7 @@ import type { DifyWorld } from '../../support/world' import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { + createAgentSoulConfigWithModel, createConfiguredTestAgent, createTestAgent, getAgentConfigurePath, @@ -40,6 +41,24 @@ Given( }, ) +Given( + 'a runnable Agent v2 test agent has been created via API', + async function (this: DifyWorld) { + if (!this.agentBuilderStableChatModel) + throw new Error('Create a runnable Agent v2 test agent after stable model preflight.') + + const agent = await createConfiguredTestAgent({ + agentSoul: createAgentSoulConfigWithModel( + normalAgentSoulConfig, + this.agentBuilderStableChatModel, + ), + }) + this.createdAgentIds.push(agent.id) + this.lastCreatedAgentName = agent.name + this.lastCreatedAgentRole = agent.role + }, +) + Given('a minimal Agent v2 composer draft has been synced', async function (this: DifyWorld) { const agentId = getCurrentAgentId(this) diff --git a/e2e/support/agent.ts b/e2e/support/agent.ts index 4afa63940e59e8..e59fa44bbb79c0 100644 --- a/e2e/support/agent.ts +++ b/e2e/support/agent.ts @@ -18,6 +18,10 @@ export type AgentSeed = { } export type AgentSoulConfig = Record +export type AgentModelSelection = { + name: string + provider: string +} export type AgentComposerResponse = { agent_soul?: AgentSoulConfig @@ -74,6 +78,39 @@ export const updatedAgentSoulConfig: AgentSoulConfig = { export const getAgentConfigurePath = (agentId: string) => `/roster/agent/${agentId}/configure` export const getAgentAccessPath = (agentId: string) => `/roster/agent/${agentId}/access` +const getAgentModelPluginId = (provider: string) => { + const [organization, pluginName] = provider.split('/').filter(Boolean) + + if (organization && pluginName) + return `${organization}/${pluginName}` + + return provider ? `langgenius/${provider}` : '' +} + +const getExistingModelConfig = (agentSoul: AgentSoulConfig) => { + const model = agentSoul.model + + if (model && typeof model === 'object' && !Array.isArray(model)) + return model as Record + + return {} +} + +export function createAgentSoulConfigWithModel( + agentSoul: AgentSoulConfig, + model: AgentModelSelection, +): AgentSoulConfig { + return { + ...agentSoul, + model: { + ...getExistingModelConfig(agentSoul), + plugin_id: getAgentModelPluginId(model.provider), + model_provider: model.provider, + model: model.name, + }, + } +} + export async function createTestAgent({ description = 'Created by Dify E2E.', name = createE2EResourceName('Agent'), From 8ee203c5f55ea9959dd5a2199943620f11eca880 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 12:36:34 +0800 Subject: [PATCH 022/185] test(e2e): require active stable agent model --- e2e/AGENTS.md | 4 ++-- e2e/support/agent.ts | 4 ++++ e2e/support/preflight.ts | 21 +++++++++++++++++++-- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 8c9d9eb5a79ae2..352ca8afbe50f1 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -294,7 +294,7 @@ Use `fixtures/test-materials/` for checked-in files that scenarios upload, previ Use `support/preflight.ts` for scenarios that require optional external resources such as a stable model provider, plugin/tool credential, or knowledge base seed. Prefer an explicit `Given` step that returns a skipped result with a clear blocked-precondition reason over hidden setup in hooks. -Use `the Agent Builder stable chat model is available` before scenarios that must run an Agent with a real model. The step requires `E2E_STABLE_MODEL_PROVIDER` and `E2E_STABLE_MODEL_NAME`, defaults `E2E_STABLE_MODEL_TYPE` to `llm`, and verifies the model through `/console/api/workspaces/current/models/model-types/{type}` before storing it on `DifyWorld.agentBuilderStableChatModel`. Any Agent Builder scenario that needs a usable model should explicitly apply this stored model during its own setup instead of hard-coding provider or model names in feature files or hooks. +Use `the Agent Builder stable chat model is available` before scenarios that must run an Agent with a real model. The step requires `E2E_STABLE_MODEL_PROVIDER` and `E2E_STABLE_MODEL_NAME`, defaults `E2E_STABLE_MODEL_TYPE` to `llm`, and verifies the model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}` before storing it on `DifyWorld.agentBuilderStableChatModel`. Any Agent Builder scenario that needs a usable model should explicitly apply this stored model during its own setup instead of hard-coding provider or model names in feature files or hooks. Use `the Agent Builder broken chat model is available` before model-recovery scenarios that intentionally start from an invalid model. The step requires `E2E_BROKEN_MODEL_PROVIDER`, defaults `E2E_BROKEN_MODEL_NAME` to `e2e-broken-model`, defaults `E2E_BROKEN_MODEL_TYPE` to `llm`, and only verifies that the model entry exists. The scenario must still assert the user-visible failure and recovery behavior. @@ -325,6 +325,6 @@ Keep Agent v2 step definitions under `features/step-definitions/agent-v2/`. Pref Use `a basic configured Agent v2 test agent has been created via API` when a scenario only needs a created Agent with a composer draft. Do not use that basic shell for runtime, model, tool, skill, knowledge, environment variable, moderation, or output-variable coverage until those resources have explicit seed helpers and readiness checks. -Use `a runnable Agent v2 test agent has been created via API` after `the Agent Builder stable chat model is available` when a scenario needs a real model-backed Agent. The step writes the preflight model into the Agent Soul model config through `support/agent.ts`; do not duplicate provider/model payload construction in individual steps. +Use `a runnable Agent v2 test agent has been created via API` after `the Agent Builder stable chat model is available` when a scenario needs a real model-backed Agent. The step writes the preflight model into the Agent Soul model config through `support/agent.ts` with deterministic E2E model settings; do not duplicate provider/model payload construction in individual steps. Use `the Agent v2 configuration should be saved automatically` after UI edits that rely on Configure autosave. It waits for the user-visible publish bar saved state; do not replace it with network-idle waits or internal store checks. diff --git a/e2e/support/agent.ts b/e2e/support/agent.ts index e59fa44bbb79c0..8ec31c18c523cb 100644 --- a/e2e/support/agent.ts +++ b/e2e/support/agent.ts @@ -107,6 +107,10 @@ export function createAgentSoulConfigWithModel( plugin_id: getAgentModelPluginId(model.provider), model_provider: model.provider, model: model.name, + model_settings: { + temperature: 0, + max_tokens: 512, + }, }, } } diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index 3e8b270e418b0b..806e5c2d36c32e 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -8,6 +8,7 @@ const stableChatModelTypeEnv = 'E2E_STABLE_MODEL_TYPE' const brokenChatModelProviderEnv = 'E2E_BROKEN_MODEL_PROVIDER' const brokenChatModelNameEnv = 'E2E_BROKEN_MODEL_NAME' const brokenChatModelTypeEnv = 'E2E_BROKEN_MODEL_TYPE' +const activeModelStatus = 'active' const defaultStableChatModelType = 'llm' const defaultBrokenChatModelName = agentBuilderPreseededResources.brokenModel @@ -405,6 +406,11 @@ export function readAgentBuilderBrokenChatModelConfig(): ModelPreflightConfig { async function skipMissingAgentBuilderModel( world: DifyWorld, config: ModelPreflightConfig, + { + requireActive, + }: { + requireActive: boolean + }, ): Promise<'skipped' | NonNullable> { if (!config.ok) return skipBlockedPrecondition(world, config.reason) @@ -431,6 +437,13 @@ async function skipMissingAgentBuilderModel( ) } + if (requireActive && model.status !== activeModelStatus) { + return skipBlockedPrecondition( + world, + `${config.resourceName} is ${model.status ?? 'missing status'} instead of ${activeModelStatus}.`, + ) + } + return { name: model.model, provider: provider.provider, @@ -445,11 +458,15 @@ async function skipMissingAgentBuilderModel( export async function skipMissingAgentBuilderStableChatModel( world: DifyWorld, ): Promise<'skipped' | NonNullable> { - return skipMissingAgentBuilderModel(world, readAgentBuilderStableChatModelConfig()) + return skipMissingAgentBuilderModel(world, readAgentBuilderStableChatModelConfig(), { + requireActive: true, + }) } export async function skipMissingAgentBuilderBrokenChatModel( world: DifyWorld, ): Promise<'skipped' | NonNullable> { - return skipMissingAgentBuilderModel(world, readAgentBuilderBrokenChatModelConfig()) + return skipMissingAgentBuilderModel(world, readAgentBuilderBrokenChatModelConfig(), { + requireActive: false, + }) } From 29d14f40043f42be142f15306df53b6c2c757a10 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 12:42:03 +0800 Subject: [PATCH 023/185] test(e2e): add agent builder preflight feature --- e2e/AGENTS.md | 2 + e2e/features/agent-v2/preflight.feature | 65 +++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 e2e/features/agent-v2/preflight.feature diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 352ca8afbe50f1..7d50c8ffd3f13e 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -302,6 +302,8 @@ Use `the Agent Builder preseeded Agent "{name}" is available`, `the Agent Builde Use `the Agent Builder preseeded Agent "{agent}" includes drive skill "{skill}"` to verify that a fixed Agent has a drive-backed Skill attached. Use `the Agent Builder preseeded Agent "{agent}" has Backend service API access with an API key` to verify that a fixed Agent has Backend service API enabled and at least one key. The API key step does not validate a human-readable key name because the Console API key response does not expose one. +Run `pnpm -C e2e e2e -- --tags @agent-v2-preflight` against a seeded environment to verify Agent Builder preseeded resource readiness before running dependent scenarios. Keep each resource as a separate preflight scenario so a missing resource marks only its dependent precondition as blocked instead of hiding the rest of the readiness report. + Use `DifyWorld.createdDatasetIds` for datasets created by a scenario, `DifyWorld.createdAgentDriveFiles` for Agent drive files committed during a scenario, and `DifyWorld.createdBuiltinToolCredentials` for built-in tool credentials created during a scenario. The shared `After` hook deletes Agent drive files first so file cleanup also works for scenarios that upload into a preseeded Agent, then deletes created Agents and Apps before deleting dependent datasets and tool credentials. Use `DifyWorld.registerCleanup(...)` when a scenario creates any other resource type that is not covered by the typed cleanup fields. Cleanup callbacks run after the typed cleanup queues, even when the scenario fails. ## Reusing existing steps diff --git a/e2e/features/agent-v2/preflight.feature b/e2e/features/agent-v2/preflight.feature new file mode 100644 index 00000000000000..a1d86214e17022 --- /dev/null +++ b/e2e/features/agent-v2/preflight.feature @@ -0,0 +1,65 @@ +@agent-v2 @authenticated @infra @agent-v2-preflight +Feature: Agent Builder preseeded environment + Scenario: Stable chat model is available + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + + Scenario: Broken chat model is available for recovery scenarios + Given I am signed in as the default E2E admin + And the Agent Builder broken chat model is available + + Scenario: JSON Replace tool is available + Given I am signed in as the default E2E admin + And the Agent Builder preseeded tool "JSON Process / JSON Replace" is available + + Scenario: Tavily Search tool is available + Given I am signed in as the default E2E admin + And the Agent Builder preseeded tool "Tavily / Tavily Search" is available + + Scenario: Agent knowledge base is available + Given I am signed in as the default E2E admin + And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is available + + Scenario: Indexing knowledge base is available + Given I am signed in as the default E2E admin + And the Agent Builder preseeded dataset "E2E Agent Knowledge Base Indexing" is available + + Scenario: Full config Agent is available + Given I am signed in as the default E2E admin + And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" is available + + Scenario: Full config Agent includes the Summary Skill + Given I am signed in as the default E2E admin + And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" includes drive skill "E2E Summary Skill" + + Scenario: Tool states Agent is available + Given I am signed in as the default E2E admin + And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" is available + + Scenario: File tree Agent is available + Given I am signed in as the default E2E admin + And the Agent Builder preseeded Agent "E2E Agent With File Tree" is available + + Scenario: Dual retrieval Agent is available + Given I am signed in as the default E2E admin + And the Agent Builder preseeded Agent "E2E Agent With Dual Retrieval" is available + + Scenario: Published Web app Agent is available + Given I am signed in as the default E2E admin + And the Agent Builder preseeded Agent "E2E Agent Published Web App" is available + + Scenario: Backend API-enabled Agent is available + Given I am signed in as the default E2E admin + And the Agent Builder preseeded Agent "E2E Agent Backend API Enabled" is available + + Scenario: Backend API-enabled Agent exposes API access with a key + Given I am signed in as the default E2E admin + And the Agent Builder preseeded Agent "E2E Agent Backend API Enabled" has Backend service API access with an API key + + Scenario: Workflow reference Agent is available + Given I am signed in as the default E2E admin + And the Agent Builder preseeded Agent "E2E Agent With Workflow Reference" is available + + Scenario: Reference workflow is available + Given I am signed in as the default E2E admin + And the Agent Builder preseeded workflow "E2E Agent Reference Workflow" is available From 3cd774de8d426551dfcfb88d14d088582bb46a2e Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 12:50:20 +0800 Subject: [PATCH 024/185] test(e2e): verify agent builder dataset readiness --- e2e/AGENTS.md | 2 + e2e/features/agent-v2/preflight.feature | 4 +- .../agent-v2/preflight.steps.ts | 24 +++ e2e/support/preflight.ts | 149 ++++++++++++++++-- 4 files changed, 163 insertions(+), 16 deletions(-) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 7d50c8ffd3f13e..5fb2f9289cc2b3 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -300,6 +300,8 @@ Use `the Agent Builder broken chat model is available` before model-recovery sce Use `the Agent Builder preseeded Agent "{name}" is available`, `the Agent Builder preseeded workflow "{name}" is available`, `the Agent Builder preseeded dataset "{name}" is available`, and `the Agent Builder preseeded tool "{provider} / {tool}" is available` when a scenario depends on a fixed environment resource. These steps verify the resource through Console APIs, store the result in `DifyWorld.agentBuilderPreseededResources`, and return `skipped` with a blocked-precondition attachment when the resource is missing. The tool step checks built-in tool availability only; credential-validity scenarios still need explicit credential-state setup or assertions. +Use `the Agent Builder preseeded dataset "{name}" is indexed and ready` for knowledge retrieval scenarios that require a completed knowledge base. It verifies that the dataset exists, has documents, all listed documents are available, and every document indexing status is `completed`. Use `the Agent Builder preseeded dataset "{name}" is indexing` for failure-recovery scenarios that require an indexing or queued knowledge base; it verifies at least one document is in `waiting`, `parsing`, `cleaning`, `splitting`, or `indexing`. Fixed-content assertions such as `AGENT_KNOWLEDGE_PASS` belong in the dependent runtime scenario, where the user-visible Agent reply can prove retrieval actually hit the expected content. + Use `the Agent Builder preseeded Agent "{agent}" includes drive skill "{skill}"` to verify that a fixed Agent has a drive-backed Skill attached. Use `the Agent Builder preseeded Agent "{agent}" has Backend service API access with an API key` to verify that a fixed Agent has Backend service API enabled and at least one key. The API key step does not validate a human-readable key name because the Console API key response does not expose one. Run `pnpm -C e2e e2e -- --tags @agent-v2-preflight` against a seeded environment to verify Agent Builder preseeded resource readiness before running dependent scenarios. Keep each resource as a separate preflight scenario so a missing resource marks only its dependent precondition as blocked instead of hiding the rest of the readiness report. diff --git a/e2e/features/agent-v2/preflight.feature b/e2e/features/agent-v2/preflight.feature index a1d86214e17022..918bf234102207 100644 --- a/e2e/features/agent-v2/preflight.feature +++ b/e2e/features/agent-v2/preflight.feature @@ -18,11 +18,11 @@ Feature: Agent Builder preseeded environment Scenario: Agent knowledge base is available Given I am signed in as the default E2E admin - And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is available + And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready Scenario: Indexing knowledge base is available Given I am signed in as the default E2E admin - And the Agent Builder preseeded dataset "E2E Agent Knowledge Base Indexing" is available + And the Agent Builder preseeded dataset "E2E Agent Knowledge Base Indexing" is indexing Scenario: Full config Agent is available Given I am signed in as the default E2E admin diff --git a/e2e/features/step-definitions/agent-v2/preflight.steps.ts b/e2e/features/step-definitions/agent-v2/preflight.steps.ts index 2984499b66ce8e..452fbe5320ea7d 100644 --- a/e2e/features/step-definitions/agent-v2/preflight.steps.ts +++ b/e2e/features/step-definitions/agent-v2/preflight.steps.ts @@ -3,12 +3,14 @@ import { Given } from '@cucumber/cucumber' import { skipMissingAgentBuilderBrokenChatModel, skipMissingAgentBuilderStableChatModel, + skipMissingIndexingPreseededDataset, skipMissingPreseededAgent, skipMissingPreseededAgentBackendApiKey, skipMissingPreseededAgentDriveSkill, skipMissingPreseededDataset, skipMissingPreseededTool, skipMissingPreseededWorkflow, + skipMissingReadyPreseededDataset, } from '../../../support/preflight' Given('the Agent Builder stable chat model is available', async function (this: DifyWorld) { @@ -60,6 +62,28 @@ Given( }, ) +Given( + 'the Agent Builder preseeded dataset {string} is indexed and ready', + async function (this: DifyWorld, resourceName: string) { + const resource = await skipMissingReadyPreseededDataset(this, resourceName) + if (resource === 'skipped') + return resource + + this.agentBuilderPreseededResources[resourceName] = resource + }, +) + +Given( + 'the Agent Builder preseeded dataset {string} is indexing', + async function (this: DifyWorld, resourceName: string) { + const resource = await skipMissingIndexingPreseededDataset(this, resourceName) + if (resource === 'skipped') + return resource + + this.agentBuilderPreseededResources[resourceName] = resource + }, +) + Given( 'the Agent Builder preseeded tool {string} is available', async function (this: DifyWorld, resourceName: string) { diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index 806e5c2d36c32e..2460d3d5ec44fa 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -89,10 +89,33 @@ type NamedResource = { name: string } -type NamedResourceListResponse = { - data: NamedResource[] +type DatasetResource = NamedResource & { + document_count: number + total_available_documents: number } +type NamedResourceListResponse = { + data: T[] +} + +type DocumentIndexingStatus = 'cleaning' | 'completed' | 'indexing' | 'parsing' | 'splitting' | 'waiting' + +type DatasetIndexingStatusResponse = { + data: Array<{ + id: string + indexing_status?: string + }> +} + +const completedDocumentIndexingStatus: DocumentIndexingStatus = 'completed' +const activeDocumentIndexingStatuses = new Set([ + 'cleaning', + 'indexing', + 'parsing', + 'splitting', + 'waiting', +]) + type LocalizedLabel = { en_US?: string zh_Hans?: string @@ -125,7 +148,7 @@ type AgentApiKeyListResponse = { }> } -const findConsoleResourceByName = async ({ +const findConsoleResourceByName = async ({ action, path, resourceName, @@ -138,7 +161,7 @@ const findConsoleResourceByName = async ({ try { const response = await ctx.get(path) await expectApiResponseOK(response, action) - const body = (await response.json()) as NamedResourceListResponse + const body = (await response.json()) as NamedResourceListResponse return body.data.find(item => item.name === resourceName) } @@ -152,6 +175,41 @@ const buildQuery = (params: Record) => new URLSearchParams(param const matchesNameOrLabel = (value: string, name: string, label?: LocalizedLabel) => value === name || value === label?.en_US || value === label?.zh_Hans +const getPreseededDataset = async (resourceName: string) => { + const query = buildQuery({ keyword: resourceName, limit: '20', page: '1' }) + + return findConsoleResourceByName({ + action: `Check preseeded dataset ${resourceName}`, + path: `/console/api/datasets?${query}`, + resourceName, + }) +} + +const getDatasetIndexingStatuses = async ( + datasetId: string, + resourceName: string, +) => { + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/datasets/${datasetId}/indexing-status`) + await expectApiResponseOK(response, `Check preseeded dataset indexing status ${resourceName}`) + const body = (await response.json()) as DatasetIndexingStatusResponse + + return body.data + } + finally { + await ctx.dispose() + } +} + +const toDatasetResource = ( + resource: NamedResource, +): NonNullable => ({ + id: resource.id, + kind: 'dataset', + name: resource.name, +}) + const splitToolDisplayName = (resourceName: string) => { const [providerName, toolName] = resourceName.split('/').map(item => item.trim()) @@ -215,21 +273,84 @@ export async function skipMissingPreseededDataset( world: DifyWorld, resourceName: string, ): Promise<'skipped' | NonNullable> { - const query = buildQuery({ keyword: resourceName, limit: '20', page: '1' }) - const resource = await findConsoleResourceByName({ - action: `Check preseeded dataset ${resourceName}`, - path: `/console/api/datasets?${query}`, - resourceName, - }) + const resource = await getPreseededDataset(resourceName) if (!resource) return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`) - return { - id: resource.id, - kind: 'dataset', - name: resource.name, + return toDatasetResource(resource) +} + +export async function skipMissingReadyPreseededDataset( + world: DifyWorld, + resourceName: string, +): Promise<'skipped' | NonNullable> { + const resource = await getPreseededDataset(resourceName) + + if (!resource) + return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`) + + if (resource.document_count < 1) { + return skipBlockedPrecondition( + world, + `Preseeded dataset "${resourceName}" has no documents.`, + ) + } + + if (resource.total_available_documents !== resource.document_count) { + return skipBlockedPrecondition( + world, + `Preseeded dataset "${resourceName}" has ${resource.total_available_documents}/${resource.document_count} available documents.`, + ) + } + + const statuses = await getDatasetIndexingStatuses(resource.id, resourceName) + if (statuses.length < 1) { + return skipBlockedPrecondition( + world, + `Preseeded dataset "${resourceName}" has no document indexing status.`, + ) + } + + const incompleteStatus = statuses.find( + item => item.indexing_status !== completedDocumentIndexingStatus, + ) + if (incompleteStatus) { + return skipBlockedPrecondition( + world, + `Preseeded dataset "${resourceName}" includes document ${incompleteStatus.id} with indexing status "${incompleteStatus.indexing_status ?? 'missing'}".`, + ) + } + + return toDatasetResource(resource) +} + +export async function skipMissingIndexingPreseededDataset( + world: DifyWorld, + resourceName: string, +): Promise<'skipped' | NonNullable> { + const resource = await getPreseededDataset(resourceName) + + if (!resource) + return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`) + + const statuses = await getDatasetIndexingStatuses(resource.id, resourceName) + const indexingStatus = statuses.find(item => + activeDocumentIndexingStatuses.has(item.indexing_status ?? ''), + ) + + if (!indexingStatus) { + const actualStatuses = statuses + .map(item => item.indexing_status ?? 'missing') + .join(', ') || 'none' + + return skipBlockedPrecondition( + world, + `Preseeded dataset "${resourceName}" is not indexing or queued; document statuses: ${actualStatuses}.`, + ) } + + return toDatasetResource(resource) } export async function skipMissingPreseededTool( From cdfda3938a06ca2f5dda7a738a67666eb2fc36e4 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 12:55:20 +0800 Subject: [PATCH 025/185] test(e2e): verify agent builder web app readiness --- e2e/AGENTS.md | 2 + e2e/features/agent-v2/preflight.feature | 4 +- .../agent-v2/preflight.steps.ts | 12 ++++ e2e/support/preflight.ts | 56 +++++++++++++++++++ 4 files changed, 72 insertions(+), 2 deletions(-) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 5fb2f9289cc2b3..461a037c711b50 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -304,6 +304,8 @@ Use `the Agent Builder preseeded dataset "{name}" is indexed and ready` for know Use `the Agent Builder preseeded Agent "{agent}" includes drive skill "{skill}"` to verify that a fixed Agent has a drive-backed Skill attached. Use `the Agent Builder preseeded Agent "{agent}" has Backend service API access with an API key` to verify that a fixed Agent has Backend service API enabled and at least one key. The API key step does not validate a human-readable key name because the Console API key response does not expose one. +Use `the Agent Builder preseeded Agent "{agent}" has published Web app access` to verify that a fixed Agent is published, Web app access is enabled, and the Agent detail response includes the site token and base URL needed to open the Web app. Keep Web app launch and runtime assertions in dependent user-visible scenarios. + Run `pnpm -C e2e e2e -- --tags @agent-v2-preflight` against a seeded environment to verify Agent Builder preseeded resource readiness before running dependent scenarios. Keep each resource as a separate preflight scenario so a missing resource marks only its dependent precondition as blocked instead of hiding the rest of the readiness report. Use `DifyWorld.createdDatasetIds` for datasets created by a scenario, `DifyWorld.createdAgentDriveFiles` for Agent drive files committed during a scenario, and `DifyWorld.createdBuiltinToolCredentials` for built-in tool credentials created during a scenario. The shared `After` hook deletes Agent drive files first so file cleanup also works for scenarios that upload into a preseeded Agent, then deletes created Agents and Apps before deleting dependent datasets and tool credentials. Use `DifyWorld.registerCleanup(...)` when a scenario creates any other resource type that is not covered by the typed cleanup fields. Cleanup callbacks run after the typed cleanup queues, even when the scenario fails. diff --git a/e2e/features/agent-v2/preflight.feature b/e2e/features/agent-v2/preflight.feature index 918bf234102207..d367f60c79845a 100644 --- a/e2e/features/agent-v2/preflight.feature +++ b/e2e/features/agent-v2/preflight.feature @@ -44,9 +44,9 @@ Feature: Agent Builder preseeded environment Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E Agent With Dual Retrieval" is available - Scenario: Published Web app Agent is available + Scenario: Published Web app Agent exposes Web app access Given I am signed in as the default E2E admin - And the Agent Builder preseeded Agent "E2E Agent Published Web App" is available + And the Agent Builder preseeded Agent "E2E Agent Published Web App" has published Web app access Scenario: Backend API-enabled Agent is available Given I am signed in as the default E2E admin diff --git a/e2e/features/step-definitions/agent-v2/preflight.steps.ts b/e2e/features/step-definitions/agent-v2/preflight.steps.ts index 452fbe5320ea7d..11fed1cbf407cf 100644 --- a/e2e/features/step-definitions/agent-v2/preflight.steps.ts +++ b/e2e/features/step-definitions/agent-v2/preflight.steps.ts @@ -7,6 +7,7 @@ import { skipMissingPreseededAgent, skipMissingPreseededAgentBackendApiKey, skipMissingPreseededAgentDriveSkill, + skipMissingPreseededAgentPublishedWebApp, skipMissingPreseededDataset, skipMissingPreseededTool, skipMissingPreseededWorkflow, @@ -116,3 +117,14 @@ Given( this.agentBuilderPreseededResources[`${agentName} / Backend service API key`] = resource }, ) + +Given( + 'the Agent Builder preseeded Agent {string} has published Web app access', + async function (this: DifyWorld, agentName: string) { + const resource = await skipMissingPreseededAgentPublishedWebApp(this, agentName) + if (resource === 'skipped') + return resource + + this.agentBuilderPreseededResources[`${agentName} / Web app`] = resource + }, +) diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index 2460d3d5ec44fa..b6a622fa425bb5 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -148,6 +148,16 @@ type AgentApiKeyListResponse = { }> } +type PreseededAgentDetailResponse = { + active_config_is_published?: boolean + enable_site?: boolean + site?: { + access_token?: string | null + app_base_url?: string | null + code?: string | null + } | null +} + const findConsoleResourceByName = async ({ action, path, @@ -463,6 +473,52 @@ export async function skipMissingPreseededAgentBackendApiKey( } } +export async function skipMissingPreseededAgentPublishedWebApp( + world: DifyWorld, + agentName: string, +): Promise<'skipped' | NonNullable> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agent.id}`) + await expectApiResponseOK(response, `Check preseeded Agent published Web app ${agentName}`) + const detail = (await response.json()) as PreseededAgentDetailResponse + if (detail.active_config_is_published !== true) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" is not published.`, + ) + } + + if (detail.enable_site !== true) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" Web app is not enabled.`, + ) + } + + const siteToken = detail.site?.access_token ?? detail.site?.code + if (!siteToken || !detail.site?.app_base_url) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" Web app URL is not available.`, + ) + } + + return { + id: agent.id, + kind: 'agent', + name: agent.name, + } + } + finally { + await ctx.dispose() + } +} + type ModelPreflightConfig = | { ok: true From 68e5219e75abdc02bbe2abbd54b1e2c361ee4cb9 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 13:03:14 +0800 Subject: [PATCH 026/185] test(e2e): verify agent builder workflow references --- e2e/AGENTS.md | 2 + e2e/features/agent-v2/preflight.feature | 4 ++ .../agent-v2/preflight.steps.ts | 16 ++++++ e2e/support/preflight.ts | 55 +++++++++++++++++++ 4 files changed, 77 insertions(+) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 461a037c711b50..e0c63b81e237a7 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -306,6 +306,8 @@ Use `the Agent Builder preseeded Agent "{agent}" includes drive skill "{skill}"` Use `the Agent Builder preseeded Agent "{agent}" has published Web app access` to verify that a fixed Agent is published, Web app access is enabled, and the Agent detail response includes the site token and base URL needed to open the Web app. Keep Web app launch and runtime assertions in dependent user-visible scenarios. +Use `the Agent Builder preseeded Agent "{agent}" is referenced by workflow "{workflow}"` to verify Workflow access prerequisites. It checks both fixed resources exist, then uses `/console/api/agent/{agent_id}/referencing-workflows`, the same Console API used by the Access Point Workflow references table, to verify the workflow references the Agent through at least one published Agent node. + Run `pnpm -C e2e e2e -- --tags @agent-v2-preflight` against a seeded environment to verify Agent Builder preseeded resource readiness before running dependent scenarios. Keep each resource as a separate preflight scenario so a missing resource marks only its dependent precondition as blocked instead of hiding the rest of the readiness report. Use `DifyWorld.createdDatasetIds` for datasets created by a scenario, `DifyWorld.createdAgentDriveFiles` for Agent drive files committed during a scenario, and `DifyWorld.createdBuiltinToolCredentials` for built-in tool credentials created during a scenario. The shared `After` hook deletes Agent drive files first so file cleanup also works for scenarios that upload into a preseeded Agent, then deletes created Agents and Apps before deleting dependent datasets and tool credentials. Use `DifyWorld.registerCleanup(...)` when a scenario creates any other resource type that is not covered by the typed cleanup fields. Cleanup callbacks run after the typed cleanup queues, even when the scenario fails. diff --git a/e2e/features/agent-v2/preflight.feature b/e2e/features/agent-v2/preflight.feature index d367f60c79845a..8a026a7ce911ea 100644 --- a/e2e/features/agent-v2/preflight.feature +++ b/e2e/features/agent-v2/preflight.feature @@ -63,3 +63,7 @@ Feature: Agent Builder preseeded environment Scenario: Reference workflow is available Given I am signed in as the default E2E admin And the Agent Builder preseeded workflow "E2E Agent Reference Workflow" is available + + Scenario: Workflow reference Agent is used by the reference workflow + Given I am signed in as the default E2E admin + And the Agent Builder preseeded Agent "E2E Agent With Workflow Reference" is referenced by workflow "E2E Agent Reference Workflow" diff --git a/e2e/features/step-definitions/agent-v2/preflight.steps.ts b/e2e/features/step-definitions/agent-v2/preflight.steps.ts index 11fed1cbf407cf..c895ebcffb56c4 100644 --- a/e2e/features/step-definitions/agent-v2/preflight.steps.ts +++ b/e2e/features/step-definitions/agent-v2/preflight.steps.ts @@ -8,6 +8,7 @@ import { skipMissingPreseededAgentBackendApiKey, skipMissingPreseededAgentDriveSkill, skipMissingPreseededAgentPublishedWebApp, + skipMissingPreseededAgentWorkflowReference, skipMissingPreseededDataset, skipMissingPreseededTool, skipMissingPreseededWorkflow, @@ -128,3 +129,18 @@ Given( this.agentBuilderPreseededResources[`${agentName} / Web app`] = resource }, ) + +Given( + 'the Agent Builder preseeded Agent {string} is referenced by workflow {string}', + async function (this: DifyWorld, agentName: string, workflowName: string) { + const resource = await skipMissingPreseededAgentWorkflowReference( + this, + agentName, + workflowName, + ) + if (resource === 'skipped') + return resource + + this.agentBuilderPreseededResources[`${agentName} / ${workflowName}`] = resource + }, +) diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index b6a622fa425bb5..c6b5c1d3040ce8 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -148,6 +148,14 @@ type AgentApiKeyListResponse = { }> } +type AgentReferencingWorkflowsResponse = { + data: Array<{ + app_id: string + app_name: string + node_ids?: string[] + }> +} + type PreseededAgentDetailResponse = { active_config_is_published?: boolean enable_site?: boolean @@ -519,6 +527,53 @@ export async function skipMissingPreseededAgentPublishedWebApp( } } +export async function skipMissingPreseededAgentWorkflowReference( + world: DifyWorld, + agentName: string, + workflowName: string, +): Promise<'skipped' | NonNullable> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const workflow = await skipMissingPreseededWorkflow(world, workflowName) + if (workflow === 'skipped') + return workflow + + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agent.id}/referencing-workflows`) + await expectApiResponseOK(response, `Check preseeded Agent workflow reference ${agentName}`) + const references = (await response.json()) as AgentReferencingWorkflowsResponse + const reference = references.data.find(item => + item.app_id === workflow.id || item.app_name === workflow.name, + ) + + if (!reference) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" is not referenced by workflow "${workflowName}".`, + ) + } + + if (!reference.node_ids || reference.node_ids.length < 1) { + return skipBlockedPrecondition( + world, + `Preseeded workflow "${workflowName}" does not expose Agent reference nodes for "${agentName}".`, + ) + } + + return { + id: workflow.id, + kind: 'workflow', + name: workflow.name, + } + } + finally { + await ctx.dispose() + } +} + type ModelPreflightConfig = | { ok: true From 4d72a8eec1750df8ac50502bce312d94a2f9a092 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 13:09:33 +0800 Subject: [PATCH 027/185] test(e2e): verify agent builder file tree readiness --- e2e/AGENTS.md | 4 +- e2e/features/agent-v2/preflight.feature | 4 +- .../agent-v2/preflight.steps.ts | 12 +++++ e2e/support/preflight.ts | 44 +++++++++++++++++++ e2e/support/test-materials.ts | 8 ++++ 5 files changed, 69 insertions(+), 3 deletions(-) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index e0c63b81e237a7..f4a49281c45b28 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -294,7 +294,7 @@ Use `fixtures/test-materials/` for checked-in files that scenarios upload, previ Use `support/preflight.ts` for scenarios that require optional external resources such as a stable model provider, plugin/tool credential, or knowledge base seed. Prefer an explicit `Given` step that returns a skipped result with a clear blocked-precondition reason over hidden setup in hooks. -Use `the Agent Builder stable chat model is available` before scenarios that must run an Agent with a real model. The step requires `E2E_STABLE_MODEL_PROVIDER` and `E2E_STABLE_MODEL_NAME`, defaults `E2E_STABLE_MODEL_TYPE` to `llm`, and verifies the model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}` before storing it on `DifyWorld.agentBuilderStableChatModel`. Any Agent Builder scenario that needs a usable model should explicitly apply this stored model during its own setup instead of hard-coding provider or model names in feature files or hooks. +Use `the Agent Builder stable chat model is available` before scenarios that must run an Agent with a real model. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` define the single usable model for the E2E environment. The step defaults the type to `llm`, verifies the model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilderStableChatModel`. Any Agent Builder scenario that needs a usable model should explicitly apply this stored model during its own setup instead of hard-coding provider or model names in feature files or hooks. Use `the Agent Builder broken chat model is available` before model-recovery scenarios that intentionally start from an invalid model. The step requires `E2E_BROKEN_MODEL_PROVIDER`, defaults `E2E_BROKEN_MODEL_NAME` to `e2e-broken-model`, defaults `E2E_BROKEN_MODEL_TYPE` to `llm`, and only verifies that the model entry exists. The scenario must still assert the user-visible failure and recovery behavior. @@ -304,6 +304,8 @@ Use `the Agent Builder preseeded dataset "{name}" is indexed and ready` for know Use `the Agent Builder preseeded Agent "{agent}" includes drive skill "{skill}"` to verify that a fixed Agent has a drive-backed Skill attached. Use `the Agent Builder preseeded Agent "{agent}" has Backend service API access with an API key` to verify that a fixed Agent has Backend service API enabled and at least one key. The API key step does not validate a human-readable key name because the Console API key response does not expose one. +Use `the Agent Builder preseeded Agent "{agent}" includes the file tree fixture files` for file-tree display prerequisites. It verifies the Agent drive contains every file from `agentBuilderFileTreeFixtureFiles` through `/console/api/agent/{agent_id}/drive/files?prefix=files/`. This proves the committed drive file set is ready; keep visual tree expansion, nesting, and preview assertions in dependent user-visible scenarios. + Use `the Agent Builder preseeded Agent "{agent}" has published Web app access` to verify that a fixed Agent is published, Web app access is enabled, and the Agent detail response includes the site token and base URL needed to open the Web app. Keep Web app launch and runtime assertions in dependent user-visible scenarios. Use `the Agent Builder preseeded Agent "{agent}" is referenced by workflow "{workflow}"` to verify Workflow access prerequisites. It checks both fixed resources exist, then uses `/console/api/agent/{agent_id}/referencing-workflows`, the same Console API used by the Access Point Workflow references table, to verify the workflow references the Agent through at least one published Agent node. diff --git a/e2e/features/agent-v2/preflight.feature b/e2e/features/agent-v2/preflight.feature index 8a026a7ce911ea..e6fc5fdc541e29 100644 --- a/e2e/features/agent-v2/preflight.feature +++ b/e2e/features/agent-v2/preflight.feature @@ -36,9 +36,9 @@ Feature: Agent Builder preseeded environment Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" is available - Scenario: File tree Agent is available + Scenario: File tree Agent includes fixture files Given I am signed in as the default E2E admin - And the Agent Builder preseeded Agent "E2E Agent With File Tree" is available + And the Agent Builder preseeded Agent "E2E Agent With File Tree" includes the file tree fixture files Scenario: Dual retrieval Agent is available Given I am signed in as the default E2E admin diff --git a/e2e/features/step-definitions/agent-v2/preflight.steps.ts b/e2e/features/step-definitions/agent-v2/preflight.steps.ts index c895ebcffb56c4..b59dead0a7d5cf 100644 --- a/e2e/features/step-definitions/agent-v2/preflight.steps.ts +++ b/e2e/features/step-definitions/agent-v2/preflight.steps.ts @@ -7,6 +7,7 @@ import { skipMissingPreseededAgent, skipMissingPreseededAgentBackendApiKey, skipMissingPreseededAgentDriveSkill, + skipMissingPreseededAgentFileTreeFixture, skipMissingPreseededAgentPublishedWebApp, skipMissingPreseededAgentWorkflowReference, skipMissingPreseededDataset, @@ -108,6 +109,17 @@ Given( }, ) +Given( + 'the Agent Builder preseeded Agent {string} includes the file tree fixture files', + async function (this: DifyWorld, agentName: string) { + const resource = await skipMissingPreseededAgentFileTreeFixture(this, agentName) + if (resource === 'skipped') + return resource + + this.agentBuilderPreseededResources[`${agentName} / file tree fixture`] = resource + }, +) + Given( 'the Agent Builder preseeded Agent {string} has Backend service API access with an API key', async function (this: DifyWorld, agentName: string) { diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index c6b5c1d3040ce8..2e1984bb69fbc5 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -1,6 +1,7 @@ import type { DifyWorld } from '../features/support/world' import { agentBuilderPreseededResources } from './agent-builder-resources' import { createApiContext, expectApiResponseOK } from './api' +import { agentBuilderFileTreeFixtureFiles } from './test-materials' const stableChatModelProviderEnv = 'E2E_STABLE_MODEL_PROVIDER' const stableChatModelNameEnv = 'E2E_STABLE_MODEL_NAME' @@ -137,6 +138,12 @@ type AgentDriveSkillListResponse = { }> } +type AgentDriveFileListResponse = { + items?: Array<{ + key: string + }> +} + type AgentApiAccessResponse = { api_key_count: number enabled: boolean @@ -439,6 +446,43 @@ export async function skipMissingPreseededAgentDriveSkill( } } +export async function skipMissingPreseededAgentFileTreeFixture( + world: DifyWorld, + agentName: string, +): Promise<'skipped' | NonNullable> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const ctx = await createApiContext() + try { + const query = buildQuery({ prefix: 'files/' }) + const response = await ctx.get(`/console/api/agent/${agent.id}/drive/files?${query}`) + await expectApiResponseOK(response, `Check preseeded Agent file tree ${agentName}`) + const body = (await response.json()) as AgentDriveFileListResponse + const keys = (body.items ?? []).map(item => item.key) + const missingFiles = agentBuilderFileTreeFixtureFiles.filter(filePath => + !keys.some(key => key === `files/${filePath}` || key.endsWith(`/${filePath}`)), + ) + + if (missingFiles.length > 0) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" is missing file tree fixture files: ${missingFiles.join(', ')}.`, + ) + } + + return { + id: agent.id, + kind: 'agent', + name: agent.name, + } + } + finally { + await ctx.dispose() + } +} + export async function skipMissingPreseededAgentBackendApiKey( world: DifyWorld, agentName: string, diff --git a/e2e/support/test-materials.ts b/e2e/support/test-materials.ts index 9a9ce36fbfa315..76524447a8e405 100644 --- a/e2e/support/test-materials.ts +++ b/e2e/support/test-materials.ts @@ -34,6 +34,14 @@ export const agentBuilderGeneratedTestMaterials = { tooLargeFile: 'agent-too-large-file.txt', } as const +export const agentBuilderFileTreeFixtureFiles = [ + 'assets/sample.csv', + 'docs/中文说明.md', + 'public/index.html', + 'src/main.txt', + 'web-game/README.md', +] as const + export const getAgentBuilderTestMaterialPath = (material: keyof typeof agentBuilderTestMaterials) => getTestMaterialPath(agentBuilderTestMaterials[material]) From 5744ed0782ad622607963528e83562ac70750959 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 13:16:08 +0800 Subject: [PATCH 028/185] chore(e2e): restore package check script --- e2e/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/package.json b/e2e/package.json index 800b6ca33ab6ea..77d7db80f0ae63 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -3,7 +3,7 @@ "type": "module", "private": true, "scripts": { - "check": "eslint --fix . && tsgo --noEmit --pretty false", + "check": "vp check --fix", "e2e": "tsx ./scripts/run-cucumber.ts", "e2e:full": "tsx ./scripts/run-cucumber.ts --full", "e2e:full:headed": "tsx ./scripts/run-cucumber.ts --full --headed", From 690ecf50b32c8f908d4234b72a43f34c56374d99 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 13:17:48 +0800 Subject: [PATCH 029/185] style(e2e): format package with vp check --- .../agent-v2/access-point.steps.ts | 12 +- .../agent-v2/configure.steps.ts | 37 ++-- .../agent-v2/preflight.steps.ts | 45 ++--- .../apps/duplicate-app.steps.ts | 3 +- .../apps/switch-app-mode.steps.ts | 3 +- e2e/features/support/hooks.ts | 10 +- e2e/features/support/world.ts | 12 +- e2e/fixtures/auth.ts | 9 +- e2e/package.json | 2 +- e2e/scripts/common.ts | 25 ++- e2e/scripts/run-cucumber.ts | 17 +- e2e/scripts/setup.ts | 16 +- e2e/support/agent.ts | 53 ++---- e2e/support/api.ts | 27 ++- e2e/support/datasets.ts | 3 +- e2e/support/preflight.ts | 164 ++++++++---------- e2e/support/process.ts | 18 +- e2e/support/tools.ts | 3 +- e2e/support/web-server.ts | 9 +- 19 files changed, 180 insertions(+), 288 deletions(-) diff --git a/e2e/features/step-definitions/agent-v2/access-point.steps.ts b/e2e/features/step-definitions/agent-v2/access-point.steps.ts index 405f8f06fca8ba..b034ee0d580d67 100644 --- a/e2e/features/step-definitions/agent-v2/access-point.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point.steps.ts @@ -5,8 +5,7 @@ import { getAgentAccessPath, setAgentApiAccess } from '../../../support/agent' const getCurrentAgentId = (world: DifyWorld) => { const agentId = world.createdAgentIds.at(-1) - if (!agentId) - throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') + if (!agentId) throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') return agentId } @@ -115,8 +114,7 @@ Then('I should see the newly generated Agent v2 API key once', async function (t await expect(generatedKeyDialog.getByLabel('Copy')).toBeVisible() this.lastGeneratedAgentApiKey = (await generatedKey.textContent())?.trim() - if (!this.lastGeneratedAgentApiKey) - throw new Error('Generated Agent v2 API key was empty.') + if (!this.lastGeneratedAgentApiKey) throw new Error('Generated Agent v2 API key was empty.') }) When('I close the newly generated Agent v2 API key', async function (this: DifyWorld) { @@ -131,8 +129,7 @@ Then( 'the Agent v2 API key list should not expose the full generated secret', async function (this: DifyWorld) { const fullSecret = this.lastGeneratedAgentApiKey - if (!fullSecret) - throw new Error('No generated Agent v2 API key found.') + if (!fullSecret) throw new Error('No generated Agent v2 API key found.') const apiKeyDialog = this.getPage().getByRole('dialog', { name: /API Secret key/i }) @@ -166,8 +163,7 @@ When('I open the Agent v2 API Reference', async function (this: DifyWorld) { Then('the Agent v2 API Reference should open in a new tab', async function (this: DifyWorld) { const apiReferencePage = this.lastAgentApiReferencePage - if (!apiReferencePage) - throw new Error('No Agent v2 API Reference page was opened.') + if (!apiReferencePage) throw new Error('No Agent v2 API Reference page was opened.') await expect(apiReferencePage).toHaveURL(/developing-with-apis/) await apiReferencePage.close() diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index a7fd68d76b6c30..ab59200dd4fa30 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -18,8 +18,7 @@ import { waitForAgentConfigureAutosaved } from '../../../support/agent-configure const getCurrentAgentId = (world: DifyWorld) => { const agentId = world.createdAgentIds.at(-1) - if (!agentId) - throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') + if (!agentId) throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') return agentId } @@ -41,23 +40,20 @@ Given( }, ) -Given( - 'a runnable Agent v2 test agent has been created via API', - async function (this: DifyWorld) { - if (!this.agentBuilderStableChatModel) - throw new Error('Create a runnable Agent v2 test agent after stable model preflight.') - - const agent = await createConfiguredTestAgent({ - agentSoul: createAgentSoulConfigWithModel( - normalAgentSoulConfig, - this.agentBuilderStableChatModel, - ), - }) - this.createdAgentIds.push(agent.id) - this.lastCreatedAgentName = agent.name - this.lastCreatedAgentRole = agent.role - }, -) +Given('a runnable Agent v2 test agent has been created via API', async function (this: DifyWorld) { + if (!this.agentBuilderStableChatModel) + throw new Error('Create a runnable Agent v2 test agent after stable model preflight.') + + const agent = await createConfiguredTestAgent({ + agentSoul: createAgentSoulConfigWithModel( + normalAgentSoulConfig, + this.agentBuilderStableChatModel, + ), + }) + this.createdAgentIds.push(agent.id) + this.lastCreatedAgentName = agent.name + this.lastCreatedAgentRole = agent.role +}) Given('a minimal Agent v2 composer draft has been synced', async function (this: DifyWorld) { const agentId = getCurrentAgentId(this) @@ -81,8 +77,7 @@ When('I open the Agent v2 configure page from the Agent Roster', async function const page = this.getPage() const agentId = getCurrentAgentId(this) const agentName = this.lastCreatedAgentName - if (!agentName) - throw new Error('No Agent v2 name found. Create an Agent v2 test agent first.') + if (!agentName) throw new Error('No Agent v2 name found. Create an Agent v2 test agent first.') await page.goto('/roster') await page.getByRole('link', { name: agentName }).click() diff --git a/e2e/features/step-definitions/agent-v2/preflight.steps.ts b/e2e/features/step-definitions/agent-v2/preflight.steps.ts index b59dead0a7d5cf..58364b0fc655ed 100644 --- a/e2e/features/step-definitions/agent-v2/preflight.steps.ts +++ b/e2e/features/step-definitions/agent-v2/preflight.steps.ts @@ -18,16 +18,14 @@ import { Given('the Agent Builder stable chat model is available', async function (this: DifyWorld) { const stableModel = await skipMissingAgentBuilderStableChatModel(this) - if (stableModel === 'skipped') - return stableModel + if (stableModel === 'skipped') return stableModel this.agentBuilderStableChatModel = stableModel }) Given('the Agent Builder broken chat model is available', async function (this: DifyWorld) { const brokenModel = await skipMissingAgentBuilderBrokenChatModel(this) - if (brokenModel === 'skipped') - return brokenModel + if (brokenModel === 'skipped') return brokenModel this.agentBuilderBrokenChatModel = brokenModel }) @@ -36,8 +34,7 @@ Given( 'the Agent Builder preseeded Agent {string} is available', async function (this: DifyWorld, resourceName: string) { const resource = await skipMissingPreseededAgent(this, resourceName) - if (resource === 'skipped') - return resource + if (resource === 'skipped') return resource this.agentBuilderPreseededResources[resourceName] = resource }, @@ -47,8 +44,7 @@ Given( 'the Agent Builder preseeded workflow {string} is available', async function (this: DifyWorld, resourceName: string) { const resource = await skipMissingPreseededWorkflow(this, resourceName) - if (resource === 'skipped') - return resource + if (resource === 'skipped') return resource this.agentBuilderPreseededResources[resourceName] = resource }, @@ -58,8 +54,7 @@ Given( 'the Agent Builder preseeded dataset {string} is available', async function (this: DifyWorld, resourceName: string) { const resource = await skipMissingPreseededDataset(this, resourceName) - if (resource === 'skipped') - return resource + if (resource === 'skipped') return resource this.agentBuilderPreseededResources[resourceName] = resource }, @@ -69,8 +64,7 @@ Given( 'the Agent Builder preseeded dataset {string} is indexed and ready', async function (this: DifyWorld, resourceName: string) { const resource = await skipMissingReadyPreseededDataset(this, resourceName) - if (resource === 'skipped') - return resource + if (resource === 'skipped') return resource this.agentBuilderPreseededResources[resourceName] = resource }, @@ -80,8 +74,7 @@ Given( 'the Agent Builder preseeded dataset {string} is indexing', async function (this: DifyWorld, resourceName: string) { const resource = await skipMissingIndexingPreseededDataset(this, resourceName) - if (resource === 'skipped') - return resource + if (resource === 'skipped') return resource this.agentBuilderPreseededResources[resourceName] = resource }, @@ -91,8 +84,7 @@ Given( 'the Agent Builder preseeded tool {string} is available', async function (this: DifyWorld, resourceName: string) { const resource = await skipMissingPreseededTool(this, resourceName) - if (resource === 'skipped') - return resource + if (resource === 'skipped') return resource this.agentBuilderPreseededResources[resourceName] = resource }, @@ -102,8 +94,7 @@ Given( 'the Agent Builder preseeded Agent {string} includes drive skill {string}', async function (this: DifyWorld, agentName: string, skillName: string) { const resource = await skipMissingPreseededAgentDriveSkill(this, agentName, skillName) - if (resource === 'skipped') - return resource + if (resource === 'skipped') return resource this.agentBuilderPreseededResources[`${agentName} / ${skillName}`] = resource }, @@ -113,8 +104,7 @@ Given( 'the Agent Builder preseeded Agent {string} includes the file tree fixture files', async function (this: DifyWorld, agentName: string) { const resource = await skipMissingPreseededAgentFileTreeFixture(this, agentName) - if (resource === 'skipped') - return resource + if (resource === 'skipped') return resource this.agentBuilderPreseededResources[`${agentName} / file tree fixture`] = resource }, @@ -124,8 +114,7 @@ Given( 'the Agent Builder preseeded Agent {string} has Backend service API access with an API key', async function (this: DifyWorld, agentName: string) { const resource = await skipMissingPreseededAgentBackendApiKey(this, agentName) - if (resource === 'skipped') - return resource + if (resource === 'skipped') return resource this.agentBuilderPreseededResources[`${agentName} / Backend service API key`] = resource }, @@ -135,8 +124,7 @@ Given( 'the Agent Builder preseeded Agent {string} has published Web app access', async function (this: DifyWorld, agentName: string) { const resource = await skipMissingPreseededAgentPublishedWebApp(this, agentName) - if (resource === 'skipped') - return resource + if (resource === 'skipped') return resource this.agentBuilderPreseededResources[`${agentName} / Web app`] = resource }, @@ -145,13 +133,8 @@ Given( Given( 'the Agent Builder preseeded Agent {string} is referenced by workflow {string}', async function (this: DifyWorld, agentName: string, workflowName: string) { - const resource = await skipMissingPreseededAgentWorkflowReference( - this, - agentName, - workflowName, - ) - if (resource === 'skipped') - return resource + const resource = await skipMissingPreseededAgentWorkflowReference(this, agentName, workflowName) + if (resource === 'skipped') return resource this.agentBuilderPreseededResources[`${agentName} / ${workflowName}`] = resource }, diff --git a/e2e/features/step-definitions/apps/duplicate-app.steps.ts b/e2e/features/step-definitions/apps/duplicate-app.steps.ts index aa8c06f5df8f02..bc5fd2ddee099b 100644 --- a/e2e/features/step-definitions/apps/duplicate-app.steps.ts +++ b/e2e/features/step-definitions/apps/duplicate-app.steps.ts @@ -13,8 +13,7 @@ Given('there is an existing E2E app available for testing', async function (this When('I open the options menu for the last created E2E app', async function (this: DifyWorld) { const appName = this.lastCreatedAppName - if (!appName) - throw new Error('No app name stored. Run "I enter a unique E2E app name" first.') + if (!appName) throw new Error('No app name stored. Run "I enter a unique E2E app name" first.') const page = this.getPage() const appLink = page.getByRole('link', { name: appName, exact: true }) diff --git a/e2e/features/step-definitions/apps/switch-app-mode.steps.ts b/e2e/features/step-definitions/apps/switch-app-mode.steps.ts index cfaf21a66bde54..20f513f471bf9c 100644 --- a/e2e/features/step-definitions/apps/switch-app-mode.steps.ts +++ b/e2e/features/step-definitions/apps/switch-app-mode.steps.ts @@ -24,6 +24,5 @@ Then('I should land on the switched app', async function (this: DifyWorld) { // Capture the new app's ID so the After hook can clean it up const match = page.url().match(/\/app\/([^/]+)\/workflow/) - if (match?.[1]) - this.createdAppIds.push(match[1]) + if (match?.[1]) this.createdAppIds.push(match[1]) }) diff --git a/e2e/features/support/hooks.ts b/e2e/features/support/hooks.ts index 69848b91bbe38d..117b3b7e8ca29a 100644 --- a/e2e/features/support/hooks.ts +++ b/e2e/features/support/hooks.ts @@ -50,18 +50,16 @@ BeforeAll({ timeout: AUTH_BOOTSTRAP_TIMEOUT_MS }, async () => { }) Before(async function (this: DifyWorld, { pickle }) { - if (!browser) - throw new Error('Shared Playwright browser is not available.') + if (!browser) throw new Error('Shared Playwright browser is not available.') - const isUnauthenticatedScenario = pickle.tags.some(tag => tag.name === '@unauthenticated') + const isUnauthenticatedScenario = pickle.tags.some((tag) => tag.name === '@unauthenticated') - if (isUnauthenticatedScenario) - await this.startUnauthenticatedSession(browser) + if (isUnauthenticatedScenario) await this.startUnauthenticatedSession(browser) else await this.startAuthenticatedSession(browser) this.scenarioStartedAt = Date.now() - const tags = pickle.tags.map(tag => tag.name).join(' ') + const tags = pickle.tags.map((tag) => tag.name).join(' ') console.warn(`[e2e] start ${pickle.name}${tags ? ` ${tags}` : ''}`) }) diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index a749f0197920e1..a5c089048d9d15 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -89,8 +89,7 @@ export class DifyWorld extends World { this.page.setDefaultTimeout(30_000) this.page.on('console', (message: ConsoleMessage) => { - if (message.type() === 'error') - this.consoleErrors.push(message.text()) + if (message.type() === 'error') this.consoleErrors.push(message.text()) }) this.page.on('pageerror', (error) => { this.pageErrors.push(error.message) @@ -109,8 +108,7 @@ export class DifyWorld extends World { } getPage() { - if (!this.page) - throw new Error('Playwright page has not been initialized for this scenario.') + if (!this.page) throw new Error('Playwright page has not been initialized for this scenario.') return this.page } @@ -130,14 +128,12 @@ export class DifyWorld extends World { for (const cleanup of this.scenarioCleanups.toReversed()) { try { await cleanup() - } - catch (error) { + } catch (error) { errors.push(error instanceof Error ? error.message : String(error)) } } - if (errors.length > 0) - this.attach(`Cleanup errors:\n${errors.join('\n')}`, 'text/plain') + if (errors.length > 0) this.attach(`Cleanup errors:\n${errors.join('\n')}`, 'text/plain') } async closeSession() { diff --git a/e2e/fixtures/auth.ts b/e2e/fixtures/auth.ts index 9039f97483de55..0356c1d1ceba04 100644 --- a/e2e/fixtures/auth.ts +++ b/e2e/fixtures/auth.ts @@ -58,8 +58,7 @@ const getRemainingTimeout = (deadline: number) => Math.max(deadline - Date.now() const encodeField = (value: string) => Buffer.from(value, 'utf8').toString('base64') const assertAPIResponse = async (response: APIResponse, action: string) => { - if (response.ok()) - return + if (response.ok()) return const body = await response.text().catch(() => '') throw new Error( @@ -90,8 +89,7 @@ const postConsoleAPI = async ( const validateInitPasswordIfNeeded = async (context: BrowserContext, deadline: number) => { const initStatus = await getConsoleAPI(context, '/console/api/init', deadline) - if (initStatus.status === 'finished') - return false + if (initStatus.status === 'finished') return false console.warn('[e2e] auth bootstrap: validating init password') await postConsoleAPI(context, '/console/api/init', deadline, { password: initPassword }) @@ -167,8 +165,7 @@ export const ensureAuthenticatedState = async (browser: Browser, configuredBaseU } await writeFile(authMetadataPath, `${JSON.stringify(metadata, null, 2)}\n`, 'utf8') - } - finally { + } finally { await context.close() } } diff --git a/e2e/package.json b/e2e/package.json index 77d7db80f0ae63..7c83203c27cbb4 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -1,7 +1,7 @@ { "name": "dify-e2e", - "type": "module", "private": true, + "type": "module", "scripts": { "check": "vp check --fix", "e2e": "tsx ./scripts/run-cucumber.ts", diff --git a/e2e/scripts/common.ts b/e2e/scripts/common.ts index 20b9ab7fbd830a..21f8f0fe5675e1 100644 --- a/e2e/scripts/common.ts +++ b/e2e/scripts/common.ts @@ -51,8 +51,7 @@ const formatCommand = (command: string, args: string[]) => [command, ...args].jo export const isMainModule = (metaUrl: string) => { const entrypoint = process.argv[1] - if (!entrypoint) - return false + if (!entrypoint) return false return pathToFileURL(entrypoint).href === metaUrl } @@ -111,8 +110,7 @@ export const runCommandOrThrow = async (options: RunCommandOptions) => { const forwardSignalsToChild = (childProcess: ChildProcess) => { const handleSignal = (signal: NodeJS.Signals) => { - if (childProcess.exitCode === null) - childProcess.kill(signal) + if (childProcess.exitCode === null) childProcess.kill(signal) } const onSigint = () => handleSignal('SIGINT') @@ -157,8 +155,7 @@ export const runForegroundProcess = async ({ export const ensureFileExists = async (filePath: string, exampleFilePath: string) => { try { await access(filePath) - } - catch { + } catch { await copyFile(exampleFilePath, filePath) } } @@ -168,10 +165,9 @@ export const ensureLineInFile = async (filePath: string, line: string) => { const lines = fileContent.split(/\r?\n/) const assignmentPrefix = line.includes('=') ? `${line.slice(0, line.indexOf('='))}=` : null - if (lines.includes(line)) - return + if (lines.includes(line)) return - if (assignmentPrefix && lines.some(existingLine => existingLine.startsWith(assignmentPrefix))) + if (assignmentPrefix && lines.some((existingLine) => existingLine.startsWith(assignmentPrefix))) return const normalizedContent = fileContent.endsWith('\n') ? fileContent : `${fileContent}\n` @@ -194,16 +190,16 @@ export const readSimpleDotenv = async (filePath: string) => { const fileContent = await readFile(filePath, 'utf8') const entries = fileContent .split(/\r?\n/) - .map(line => line.trim()) - .filter(line => line && !line.startsWith('#')) + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith('#')) .map<[string, string]>((line) => { const separatorIndex = line.indexOf('=') const key = separatorIndex === -1 ? line : line.slice(0, separatorIndex).trim() const rawValue = separatorIndex === -1 ? '' : line.slice(separatorIndex + 1).trim() if ( - (rawValue.startsWith('"') && rawValue.endsWith('"')) - || (rawValue.startsWith('\'') && rawValue.endsWith('\'')) + (rawValue.startsWith('"') && rawValue.endsWith('"')) || + (rawValue.startsWith("'") && rawValue.endsWith("'")) ) { return [key, rawValue.slice(1, -1)] } @@ -228,8 +224,7 @@ export const waitForCondition = async ({ const deadline = Date.now() + timeoutMs while (Date.now() < deadline) { - if (await check()) - return + if (await check()) return await sleep(intervalMs) } diff --git a/e2e/scripts/run-cucumber.ts b/e2e/scripts/run-cucumber.ts index 3c8e895e90f569..fb105e60faa7ad 100644 --- a/e2e/scripts/run-cucumber.ts +++ b/e2e/scripts/run-cucumber.ts @@ -40,18 +40,16 @@ const parseArgs = (argv: string[]): RunOptions => { } const hasCustomTags = (forwardArgs: string[]) => - forwardArgs.some(arg => arg === '--tags' || arg.startsWith('--tags=')) + forwardArgs.some((arg) => arg === '--tags' || arg.startsWith('--tags=')) const main = async () => { const { forwardArgs, full, headed } = parseArgs(process.argv.slice(2)) const startMiddlewareForRun = full const resetStateForRun = full - if (resetStateForRun) - await resetState() + if (resetStateForRun) await resetState() - if (startMiddlewareForRun) - await startMiddleware() + if (startMiddlewareForRun) await startMiddleware() const cucumberReportDir = path.join(e2eDir, 'cucumber-report') const logDir = path.join(e2eDir, '.logs') @@ -86,8 +84,7 @@ const main = async () => { if (startMiddlewareForRun) { try { await stopMiddleware() - } - catch { + } catch { // Cleanup should continue even if middleware shutdown fails. } } @@ -109,8 +106,7 @@ const main = async () => { try { try { await waitForUrl(`${apiURL}/health`, 180_000, 1_000) - } - catch { + } catch { throw new Error(`API did not become ready at ${apiURL}/health.`) } @@ -146,8 +142,7 @@ const main = async () => { }) process.exitCode = result.exitCode - } - finally { + } finally { process.off('SIGINT', onTerminate) process.off('SIGTERM', onTerminate) await cleanup() diff --git a/e2e/scripts/setup.ts b/e2e/scripts/setup.ts index 3f77a3f72a7e54..bb37b616e1f72a 100644 --- a/e2e/scripts/setup.ts +++ b/e2e/scripts/setup.ts @@ -79,8 +79,7 @@ const getContainerHealth = async (containerId: string) => { stdio: 'pipe', }) - if (result.exitCode !== 0) - return '' + if (result.exitCode !== 0) return '' return result.stdout.trim() } @@ -106,8 +105,7 @@ const waitForDependency = async ({ try { await wait() - } - catch (error) { + } catch (error) { await printComposeLogs(services) throw error } @@ -136,7 +134,7 @@ export const ensureWebBuild = async () => { .then(() => true) .catch(() => false), readFile(webBuildEnvStampPath, 'utf8') - .then(value => value.trim()) + .then((value) => value.trim()) .catch(() => ''), ]) @@ -144,8 +142,7 @@ export const ensureWebBuild = async () => { console.log('Reusing existing web build artifact.') return } - } - catch { + } catch { // Fall through to rebuild when the existing build cannot be verified. } @@ -240,8 +237,7 @@ export const resetState = async () => { console.log('Stopping middleware services...') try { await stopMiddleware() - } - catch { + } catch { // Reset should continue even if middleware is already stopped. } @@ -255,7 +251,7 @@ export const resetState = async () => { console.log('Removing E2E local state...') await Promise.all( - e2eStatePaths.map(targetPath => rm(targetPath, { force: true, recursive: true })), + e2eStatePaths.map((targetPath) => rm(targetPath, { force: true, recursive: true })), ) console.log('E2E state reset complete.') diff --git a/e2e/support/agent.ts b/e2e/support/agent.ts index 8ec31c18c523cb..f6ca42bc8f62ac 100644 --- a/e2e/support/agent.ts +++ b/e2e/support/agent.ts @@ -57,11 +57,11 @@ export const defaultAgentSoulConfig: AgentSoulConfig = { }, } -export const normalAgentPrompt - = 'You are a Dify Agent E2E test assistant. Reply briefly to every user message, and always include AGENT_E2E_PASS in your response.' +export const normalAgentPrompt = + 'You are a Dify Agent E2E test assistant. Reply briefly to every user message, and always include AGENT_E2E_PASS in your response.' -export const updatedAgentPrompt - = 'You are a Dify Agent E2E test assistant. Every response must start with E2E_AGENT_UPDATED.' +export const updatedAgentPrompt = + 'You are a Dify Agent E2E test assistant. Every response must start with E2E_AGENT_UPDATED.' export const normalAgentSoulConfig: AgentSoulConfig = { prompt: { @@ -81,8 +81,7 @@ export const getAgentAccessPath = (agentId: string) => `/roster/agent/${agentId} const getAgentModelPluginId = (provider: string) => { const [organization, pluginName] = provider.split('/').filter(Boolean) - if (organization && pluginName) - return `${organization}/${pluginName}` + if (organization && pluginName) return `${organization}/${pluginName}` return provider ? `langgenius/${provider}` : '' } @@ -134,8 +133,7 @@ export async function createTestAgent({ }) await expectApiResponseOK(response, 'Create Agent v2 test agent') return (await response.json()) as AgentSeed - } - finally { + } finally { await ctx.dispose() } } @@ -158,8 +156,7 @@ export async function getTestAgent(agentId: string): Promise { const response = await ctx.get(`/console/api/agent/${agentId}`) await expectApiResponseOK(response, `Get Agent v2 test agent ${agentId}`) return (await response.json()) as AgentSeed - } - finally { + } finally { await ctx.dispose() } } @@ -169,8 +166,7 @@ export async function deleteTestAgent(agentId: string): Promise { try { const response = await ctx.delete(`/console/api/agent/${agentId}`) await expectApiResponseOK(response, `Delete Agent v2 test agent ${agentId}`) - } - finally { + } finally { await ctx.dispose() } } @@ -190,8 +186,7 @@ export async function saveAgentComposerDraft( }) await expectApiResponseOK(response, `Save Agent v2 composer draft for ${agentId}`) return (await response.json()) as AgentComposerResponse - } - finally { + } finally { await ctx.dispose() } } @@ -204,8 +199,7 @@ export async function checkoutAgentBuildDraft(agentId: string): Promise { try { const response = await ctx.delete(`/console/api/agent/${agentId}/build-draft`) await expectApiResponseOK(response, `Discard Agent v2 build draft for ${agentId}`) - } - finally { + } finally { await ctx.dispose() } } @@ -249,8 +241,7 @@ export async function publishAgent(agentId: string, versionNote = 'E2E publish') data: { version_note: versionNote }, }) await expectApiResponseOK(response, `Publish Agent v2 test agent ${agentId}`) - } - finally { + } finally { await ctx.dispose() } } @@ -258,8 +249,7 @@ export async function publishAgent(agentId: string, versionNote = 'E2E publish') export async function enableAgentSiteAndGetURL(agentId: string): Promise { const agent = await getTestAgent(agentId) const appId = agent.app_id ?? agent.backing_app_id - if (!appId) - throw new Error(`Agent v2 ${agentId} does not expose a backing app ID.`) + if (!appId) throw new Error(`Agent v2 ${agentId} does not expose a backing app ID.`) const appDetail = await setAppSiteEnabled(appId, true) const token = agent.site?.access_token ?? agent.site?.code ?? appDetail.site.access_token @@ -274,8 +264,7 @@ export async function getAgentApiAccess(agentId: string): Promise { const response = await ctx.post(`/console/api/agent/${agentId}/api-keys`) await expectApiResponseOK(response, `Create Agent v2 API key for ${agentId}`) return (await response.json()) as AgentApiKey - } - finally { + } finally { await ctx.dispose() } } @@ -317,8 +304,7 @@ export async function deleteAgentApiKey(agentId: string, apiKeyId: string): Prom try { const response = await ctx.delete(`/console/api/agent/${agentId}/api-keys/${apiKeyId}`) await expectApiResponseOK(response, `Delete Agent v2 API key ${apiKeyId} for ${agentId}`) - } - finally { + } finally { await ctx.dispose() } } @@ -329,8 +315,7 @@ export async function deleteAgentDriveFile(agentId: string, key: string): Promis const searchParams = new URLSearchParams({ key }) const response = await ctx.delete(`/console/api/agent/${agentId}/files?${searchParams}`) await expectApiResponseOK(response, `Delete Agent v2 drive file ${key} for ${agentId}`) - } - finally { + } finally { await ctx.dispose() } } diff --git a/e2e/support/api.ts b/e2e/support/api.ts index f03de5f349b5fb..dd0b88b2a31653 100644 --- a/e2e/support/api.ts +++ b/e2e/support/api.ts @@ -6,12 +6,12 @@ import { apiURL } from '../test-env' import { createE2EResourceName } from './naming' type StorageState = { - cookies: Array<{ name: string, value: string }> + cookies: Array<{ name: string; value: string }> } export async function createApiContext() { const state = JSON.parse(await readFile(authStatePath, 'utf8')) as StorageState - const csrfToken = state.cookies.find(c => c.name.endsWith('csrf_token'))?.value ?? '' + const csrfToken = state.cookies.find((c) => c.name.endsWith('csrf_token'))?.value ?? '' return request.newContext({ baseURL: apiURL, @@ -21,8 +21,7 @@ export async function createApiContext() { } export async function expectApiResponseOK(response: APIResponse, action: string): Promise { - if (response.ok()) - return + if (response.ok()) return const body = await response.text().catch(() => '') throw new Error(`${action} failed with ${response.status()} ${response.statusText()}: ${body}`) @@ -50,8 +49,7 @@ export async function createTestApp( }) const body = (await response.json()) as AppSeed return body - } - finally { + } finally { await ctx.dispose() } } @@ -78,8 +76,7 @@ export async function syncMinimalWorkflowDraft(appId: string): Promise { conversation_variables: [], }, }) - } - finally { + } finally { await ctx.dispose() } } @@ -88,8 +85,7 @@ export async function deleteTestApp(id: string): Promise { const ctx = await createApiContext() try { await ctx.delete(`/console/api/apps/${id}`) - } - finally { + } finally { await ctx.dispose() } } @@ -136,8 +132,7 @@ export async function syncRunnableWorkflowDraft(appId: string): Promise { conversation_variables: [], }, }) - } - finally { + } finally { await ctx.dispose() } } @@ -148,15 +143,14 @@ export async function publishWorkflowApp(appId: string): Promise { await ctx.post(`/console/api/apps/${appId}/workflows/publish`, { data: { marked_name: '', marked_comment: '' }, }) - } - finally { + } finally { await ctx.dispose() } } export type AppDetailWithSite = { mode?: string - site: { access_token: string, app_base_url: string, enable_site: boolean } + site: { access_token: string; app_base_url: string; enable_site: boolean } } export function getAppSiteURL({ mode, site }: AppDetailWithSite): string { @@ -182,8 +176,7 @@ export async function setAppSiteEnabled( const detailResponse = await ctx.get(`/console/api/apps/${appId}`) await expectApiResponseOK(detailResponse, `Get app site detail for ${appId}`) return (await detailResponse.json()) as AppDetailWithSite - } - finally { + } finally { await ctx.dispose() } } diff --git a/e2e/support/datasets.ts b/e2e/support/datasets.ts index 4c0851db26d5f3..196b335af326cc 100644 --- a/e2e/support/datasets.ts +++ b/e2e/support/datasets.ts @@ -5,8 +5,7 @@ export async function deleteTestDataset(datasetId: string): Promise { try { const response = await ctx.delete(`/console/api/datasets/${datasetId}`) await expectApiResponseOK(response, `Delete dataset ${datasetId}`) - } - finally { + } finally { await ctx.dispose() } } diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index 2e1984bb69fbc5..2b358d5afb21a5 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -13,23 +13,22 @@ const activeModelStatus = 'active' const defaultStableChatModelType = 'llm' const defaultBrokenChatModelName = agentBuilderPreseededResources.brokenModel -export type E2EResourcePrecondition - = | { - ok: true - value: string - } +export type E2EResourcePrecondition = | { - ok: false - reason: string - } + ok: true + value: string + } + | { + ok: false + reason: string + } export const readRequiredEnvResource = ( envName: string, description: string, ): E2EResourcePrecondition => { const value = process.env[envName]?.trim() - if (value) - return { ok: true, value } + if (value) return { ok: true, value } return { ok: false, @@ -50,8 +49,7 @@ export function skipMissingEnvResource( description: string, ): 'skipped' | string { const resource = readRequiredEnvResource(envName, description) - if (resource.ok) - return resource.value + if (resource.ok) return resource.value return skipBlockedPrecondition(world, resource.reason) } @@ -99,7 +97,13 @@ type NamedResourceListResponse = { data: T[] } -type DocumentIndexingStatus = 'cleaning' | 'completed' | 'indexing' | 'parsing' | 'splitting' | 'waiting' +type DocumentIndexingStatus = + | 'cleaning' + | 'completed' + | 'indexing' + | 'parsing' + | 'splitting' + | 'waiting' type DatasetIndexingStatusResponse = { data: Array<{ @@ -188,9 +192,8 @@ const findConsoleResourceByName = async - return body.data.find(item => item.name === resourceName) - } - finally { + return body.data.find((item) => item.name === resourceName) + } finally { await ctx.dispose() } } @@ -210,10 +213,7 @@ const getPreseededDataset = async (resourceName: string) => { }) } -const getDatasetIndexingStatuses = async ( - datasetId: string, - resourceName: string, -) => { +const getDatasetIndexingStatuses = async (datasetId: string, resourceName: string) => { const ctx = await createApiContext() try { const response = await ctx.get(`/console/api/datasets/${datasetId}/indexing-status`) @@ -221,8 +221,7 @@ const getDatasetIndexingStatuses = async ( const body = (await response.json()) as DatasetIndexingStatusResponse return body.data - } - finally { + } finally { await ctx.dispose() } } @@ -236,7 +235,7 @@ const toDatasetResource = ( }) const splitToolDisplayName = (resourceName: string) => { - const [providerName, toolName] = resourceName.split('/').map(item => item.trim()) + const [providerName, toolName] = resourceName.split('/').map((item) => item.trim()) if (!providerName || !toolName) { return { @@ -316,10 +315,7 @@ export async function skipMissingReadyPreseededDataset( return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`) if (resource.document_count < 1) { - return skipBlockedPrecondition( - world, - `Preseeded dataset "${resourceName}" has no documents.`, - ) + return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" has no documents.`) } if (resource.total_available_documents !== resource.document_count) { @@ -338,7 +334,7 @@ export async function skipMissingReadyPreseededDataset( } const incompleteStatus = statuses.find( - item => item.indexing_status !== completedDocumentIndexingStatus, + (item) => item.indexing_status !== completedDocumentIndexingStatus, ) if (incompleteStatus) { return skipBlockedPrecondition( @@ -360,14 +356,13 @@ export async function skipMissingIndexingPreseededDataset( return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`) const statuses = await getDatasetIndexingStatuses(resource.id, resourceName) - const indexingStatus = statuses.find(item => + const indexingStatus = statuses.find((item) => activeDocumentIndexingStatuses.has(item.indexing_status ?? ''), ) if (!indexingStatus) { - const actualStatuses = statuses - .map(item => item.indexing_status ?? 'missing') - .join(', ') || 'none' + const actualStatuses = + statuses.map((item) => item.indexing_status ?? 'missing').join(', ') || 'none' return skipBlockedPrecondition( world, @@ -383,18 +378,17 @@ export async function skipMissingPreseededTool( resourceName: string, ): Promise<'skipped' | NonNullable> { const parsed = splitToolDisplayName(resourceName) - if (!parsed.ok) - return skipBlockedPrecondition(world, parsed.reason) + if (!parsed.ok) return skipBlockedPrecondition(world, parsed.reason) const ctx = await createApiContext() try { const response = await ctx.get('/console/api/workspaces/current/tools/builtin') await expectApiResponseOK(response, `Check preseeded tool ${resourceName}`) const providers = (await response.json()) as BuiltinToolProvider[] - const provider = providers.find(item => + const provider = providers.find((item) => matchesNameOrLabel(parsed.providerName, item.name, item.label), ) - const tool = provider?.tools.find(item => + const tool = provider?.tools.find((item) => matchesNameOrLabel(parsed.toolName, item.name, item.label), ) @@ -406,8 +400,7 @@ export async function skipMissingPreseededTool( kind: 'tool', name: resourceName, } - } - finally { + } finally { await ctx.dispose() } } @@ -418,15 +411,14 @@ export async function skipMissingPreseededAgentDriveSkill( skillName: string, ): Promise<'skipped' | NonNullable> { const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent + if (agent === 'skipped') return agent const ctx = await createApiContext() try { const response = await ctx.get(`/console/api/agent/${agent.id}/drive/skills`) await expectApiResponseOK(response, `Check preseeded Agent skill ${skillName}`) const body = (await response.json()) as AgentDriveSkillListResponse - const skill = body.items.find(item => item.name === skillName) + const skill = body.items.find((item) => item.name === skillName) if (!skill) { return skipBlockedPrecondition( @@ -440,8 +432,7 @@ export async function skipMissingPreseededAgentDriveSkill( kind: 'skill', name: skill.name, } - } - finally { + } finally { await ctx.dispose() } } @@ -451,8 +442,7 @@ export async function skipMissingPreseededAgentFileTreeFixture( agentName: string, ): Promise<'skipped' | NonNullable> { const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent + if (agent === 'skipped') return agent const ctx = await createApiContext() try { @@ -460,9 +450,10 @@ export async function skipMissingPreseededAgentFileTreeFixture( const response = await ctx.get(`/console/api/agent/${agent.id}/drive/files?${query}`) await expectApiResponseOK(response, `Check preseeded Agent file tree ${agentName}`) const body = (await response.json()) as AgentDriveFileListResponse - const keys = (body.items ?? []).map(item => item.key) - const missingFiles = agentBuilderFileTreeFixtureFiles.filter(filePath => - !keys.some(key => key === `files/${filePath}` || key.endsWith(`/${filePath}`)), + const keys = (body.items ?? []).map((item) => item.key) + const missingFiles = agentBuilderFileTreeFixtureFiles.filter( + (filePath) => + !keys.some((key) => key === `files/${filePath}` || key.endsWith(`/${filePath}`)), ) if (missingFiles.length > 0) { @@ -477,8 +468,7 @@ export async function skipMissingPreseededAgentFileTreeFixture( kind: 'agent', name: agent.name, } - } - finally { + } finally { await ctx.dispose() } } @@ -488,8 +478,7 @@ export async function skipMissingPreseededAgentBackendApiKey( agentName: string, ): Promise<'skipped' | NonNullable> { const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent + if (agent === 'skipped') return agent const ctx = await createApiContext() try { @@ -519,8 +508,7 @@ export async function skipMissingPreseededAgentBackendApiKey( kind: 'api-key', name: `${agentName} Backend service API key`, } - } - finally { + } finally { await ctx.dispose() } } @@ -530,8 +518,7 @@ export async function skipMissingPreseededAgentPublishedWebApp( agentName: string, ): Promise<'skipped' | NonNullable> { const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent + if (agent === 'skipped') return agent const ctx = await createApiContext() try { @@ -539,10 +526,7 @@ export async function skipMissingPreseededAgentPublishedWebApp( await expectApiResponseOK(response, `Check preseeded Agent published Web app ${agentName}`) const detail = (await response.json()) as PreseededAgentDetailResponse if (detail.active_config_is_published !== true) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" is not published.`, - ) + return skipBlockedPrecondition(world, `Preseeded Agent "${agentName}" is not published.`) } if (detail.enable_site !== true) { @@ -565,8 +549,7 @@ export async function skipMissingPreseededAgentPublishedWebApp( kind: 'agent', name: agent.name, } - } - finally { + } finally { await ctx.dispose() } } @@ -577,20 +560,18 @@ export async function skipMissingPreseededAgentWorkflowReference( workflowName: string, ): Promise<'skipped' | NonNullable> { const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent + if (agent === 'skipped') return agent const workflow = await skipMissingPreseededWorkflow(world, workflowName) - if (workflow === 'skipped') - return workflow + if (workflow === 'skipped') return workflow const ctx = await createApiContext() try { const response = await ctx.get(`/console/api/agent/${agent.id}/referencing-workflows`) await expectApiResponseOK(response, `Check preseeded Agent workflow reference ${agentName}`) const references = (await response.json()) as AgentReferencingWorkflowsResponse - const reference = references.data.find(item => - item.app_id === workflow.id || item.app_name === workflow.name, + const reference = references.data.find( + (item) => item.app_id === workflow.id || item.app_name === workflow.name, ) if (!reference) { @@ -612,24 +593,23 @@ export async function skipMissingPreseededAgentWorkflowReference( kind: 'workflow', name: workflow.name, } - } - finally { + } finally { await ctx.dispose() } } -type ModelPreflightConfig - = | { - ok: true - provider: string - resourceName: string - type: string - value: string - } +type ModelPreflightConfig = | { - ok: false - reason: string - } + ok: true + provider: string + resourceName: string + type: string + value: string + } + | { + ok: false + reason: string + } export function readAgentBuilderStableChatModelConfig(): ModelPreflightConfig { const provider = process.env[stableChatModelProviderEnv]?.trim() @@ -637,10 +617,8 @@ export function readAgentBuilderStableChatModelConfig(): ModelPreflightConfig { const type = process.env[stableChatModelTypeEnv]?.trim() || defaultStableChatModelType const missing: string[] = [] - if (!provider) - missing.push(stableChatModelProviderEnv) - if (!name) - missing.push(stableChatModelNameEnv) + if (!provider) missing.push(stableChatModelProviderEnv) + if (!name) missing.push(stableChatModelNameEnv) if (!provider || !name) { return { @@ -688,8 +666,7 @@ async function skipMissingAgentBuilderModel( requireActive: boolean }, ): Promise<'skipped' | NonNullable> { - if (!config.ok) - return skipBlockedPrecondition(world, config.reason) + if (!config.ok) return skipBlockedPrecondition(world, config.reason) const ctx = await createApiContext() try { @@ -698,12 +675,12 @@ async function skipMissingAgentBuilderModel( ) await expectApiResponseOK(response, `Check ${config.resourceName}`) const body = (await response.json()) as ModelTypeListResponse - const provider = body.data.find(item => item.provider === config.provider) + const provider = body.data.find((item) => item.provider === config.provider) const model = provider?.models.find( - item => - item.model === config.value - || item.label?.en_US === config.value - || item.label?.zh_Hans === config.value, + (item) => + item.model === config.value || + item.label?.en_US === config.value || + item.label?.zh_Hans === config.value, ) if (!provider || !model) { @@ -725,8 +702,7 @@ async function skipMissingAgentBuilderModel( provider: provider.provider, type: config.type, } - } - finally { + } finally { await ctx.dispose() } } diff --git a/e2e/support/process.ts b/e2e/support/process.ts index b54108e6855390..84994bc615b640 100644 --- a/e2e/support/process.ts +++ b/e2e/support/process.ts @@ -64,14 +64,11 @@ export const waitForUrl = async ( const response = await fetch(url, { signal: controller.signal, }) - if (response.ok) - return - } - finally { + if (response.ok) return + } finally { clearTimeout(timeout) } - } - catch { + } catch { // Keep polling until timeout. } @@ -144,8 +141,7 @@ const waitForProcessExit = (childProcess: ChildProcess, timeoutMs: number) => const signalManagedProcess = (childProcess: ChildProcess, signal: NodeJS.Signals) => { const { pid } = childProcess - if (!pid) - return + if (!pid) return try { if (process.platform !== 'win32') { @@ -154,15 +150,13 @@ const signalManagedProcess = (childProcess: ChildProcess, signal: NodeJS.Signals } childProcess.kill(signal) - } - catch { + } catch { // Best-effort shutdown. Cleanup continues even when the process is already gone. } } export const stopManagedProcess = async (managedProcess?: ManagedProcess) => { - if (!managedProcess) - return + if (!managedProcess) return const { childProcess, logStream } = managedProcess diff --git a/e2e/support/tools.ts b/e2e/support/tools.ts index c4bee2278bb3ad..11ea82d4dcd8e0 100644 --- a/e2e/support/tools.ts +++ b/e2e/support/tools.ts @@ -16,8 +16,7 @@ export async function deleteBuiltinToolCredential( response, `Delete built-in tool credential ${credentialId} for ${provider}`, ) - } - finally { + } finally { await ctx.dispose() } } diff --git a/e2e/support/web-server.ts b/e2e/support/web-server.ts index 819f7effe38b9f..ad5d5d916a1daa 100644 --- a/e2e/support/web-server.ts +++ b/e2e/support/web-server.ts @@ -34,8 +34,7 @@ export const startWebServer = async ({ }: WebServerStartOptions) => { const { host, port } = getUrlHostAndPort(baseURL) - if (reuseExistingServer && (await isPortReachable(host, port))) - return + if (reuseExistingServer && (await isPortReachable(host, port))) return activeProcess = await startLoggedProcess({ command, @@ -50,8 +49,7 @@ export const startWebServer = async ({ startupError = error }) activeProcess.childProcess.once('exit', (code, signal) => { - if (startupError) - return + if (startupError) return startupError = new Error( `Web server exited before readiness (code: ${code ?? 'unknown'}, signal: ${signal ?? 'none'}).`, @@ -69,8 +67,7 @@ export const startWebServer = async ({ try { await waitForUrl(baseURL, 1_000, 250, 1_000) return - } - catch { + } catch { // Continue polling until timeout or child exit. } } From 4401b1b8cda4a5cfc3d5fe1d3cb19dff8740e6c1 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 13:22:43 +0800 Subject: [PATCH 030/185] test(e2e): verify agent builder full config readiness --- e2e/AGENTS.md | 2 + e2e/features/agent-v2/preflight.feature | 4 + .../agent-v2/preflight.steps.ts | 11 ++ e2e/support/preflight.ts | 157 +++++++++++++++++- 4 files changed, 172 insertions(+), 2 deletions(-) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index f4a49281c45b28..7da2a7cfd11600 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -304,6 +304,8 @@ Use `the Agent Builder preseeded dataset "{name}" is indexed and ready` for know Use `the Agent Builder preseeded Agent "{agent}" includes drive skill "{skill}"` to verify that a fixed Agent has a drive-backed Skill attached. Use `the Agent Builder preseeded Agent "{agent}" has Backend service API access with an API key` to verify that a fixed Agent has Backend service API enabled and at least one key. The API key step does not validate a human-readable key name because the Console API key response does not expose one. +Use `the Agent Builder preseeded Agent "{agent}" includes the core fixture configuration` for the fixed Full Config Agent prerequisite. It composes the stable model, Summary Skill, JSON Replace tool, and indexed knowledge-base preflights, then reads `/console/api/agent/{agent_id}/composer` to verify the Agent Soul contains the selected model, prompt success token, required file fixtures, JSON Replace tool entry, and knowledge dataset reference. Do not use this step for Agent node output variables; those live in workflow node-job `declared_outputs`, not the roster Agent App composer response. + Use `the Agent Builder preseeded Agent "{agent}" includes the file tree fixture files` for file-tree display prerequisites. It verifies the Agent drive contains every file from `agentBuilderFileTreeFixtureFiles` through `/console/api/agent/{agent_id}/drive/files?prefix=files/`. This proves the committed drive file set is ready; keep visual tree expansion, nesting, and preview assertions in dependent user-visible scenarios. Use `the Agent Builder preseeded Agent "{agent}" has published Web app access` to verify that a fixed Agent is published, Web app access is enabled, and the Agent detail response includes the site token and base URL needed to open the Web app. Keep Web app launch and runtime assertions in dependent user-visible scenarios. diff --git a/e2e/features/agent-v2/preflight.feature b/e2e/features/agent-v2/preflight.feature index e6fc5fdc541e29..86d3c167119678 100644 --- a/e2e/features/agent-v2/preflight.feature +++ b/e2e/features/agent-v2/preflight.feature @@ -32,6 +32,10 @@ Feature: Agent Builder preseeded environment Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" includes drive skill "E2E Summary Skill" + Scenario: Full config Agent includes core fixture configuration + Given I am signed in as the default E2E admin + And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" includes the core fixture configuration + Scenario: Tool states Agent is available Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" is available diff --git a/e2e/features/step-definitions/agent-v2/preflight.steps.ts b/e2e/features/step-definitions/agent-v2/preflight.steps.ts index 58364b0fc655ed..4ac96764a3c0f8 100644 --- a/e2e/features/step-definitions/agent-v2/preflight.steps.ts +++ b/e2e/features/step-definitions/agent-v2/preflight.steps.ts @@ -11,6 +11,7 @@ import { skipMissingPreseededAgentPublishedWebApp, skipMissingPreseededAgentWorkflowReference, skipMissingPreseededDataset, + skipMissingPreseededFullConfigAgentCoreConfiguration, skipMissingPreseededTool, skipMissingPreseededWorkflow, skipMissingReadyPreseededDataset, @@ -100,6 +101,16 @@ Given( }, ) +Given( + 'the Agent Builder preseeded Agent {string} includes the core fixture configuration', + async function (this: DifyWorld, agentName: string) { + const resource = await skipMissingPreseededFullConfigAgentCoreConfiguration(this, agentName) + if (resource === 'skipped') return resource + + this.agentBuilderPreseededResources[`${agentName} / core fixture configuration`] = resource + }, +) + Given( 'the Agent Builder preseeded Agent {string} includes the file tree fixture files', async function (this: DifyWorld, agentName: string) { diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index 2b358d5afb21a5..43604b37ada6ae 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -1,7 +1,10 @@ import type { DifyWorld } from '../features/support/world' -import { agentBuilderPreseededResources } from './agent-builder-resources' +import { + agentBuilderExpectedTokens, + agentBuilderPreseededResources, +} from './agent-builder-resources' import { createApiContext, expectApiResponseOK } from './api' -import { agentBuilderFileTreeFixtureFiles } from './test-materials' +import { agentBuilderFileTreeFixtureFiles, agentBuilderTestMaterials } from './test-materials' const stableChatModelProviderEnv = 'E2E_STABLE_MODEL_PROVIDER' const stableChatModelNameEnv = 'E2E_STABLE_MODEL_NAME' @@ -148,6 +151,10 @@ type AgentDriveFileListResponse = { }> } +type AgentComposerResponse = { + agent_soul?: Record +} + type AgentApiAccessResponse = { api_key_count: number enabled: boolean @@ -203,6 +210,69 @@ const buildQuery = (params: Record) => new URLSearchParams(param const matchesNameOrLabel = (value: string, name: string, label?: LocalizedLabel) => value === name || value === label?.en_US || value === label?.zh_Hans +const isRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value) + +const asRecord = (value: unknown): Record => (isRecord(value) ? value : {}) + +const asArray = (value: unknown): unknown[] => (Array.isArray(value) ? value : []) + +const asString = (value: unknown) => (typeof value === 'string' ? value : '') + +const hasNamedOrKeyedEntry = (items: unknown[], expectedName: string) => + items.some((item) => { + const record = asRecord(item) + const values = [record.name, record.drive_key, record.reference, record.file_id, record.id].map( + asString, + ) + + return values.some((value) => value === expectedName || value.endsWith(`/${expectedName}`)) + }) + +const hasToolEntry = ( + items: unknown[], + { + providerDisplayName, + providerName, + toolDisplayName, + toolName, + }: { + providerDisplayName: string + providerName: string + toolDisplayName: string + toolName: string + }, +) => + items.some((item) => { + const record = asRecord(item) + const providerValues = [record.provider_id, record.provider, record.plugin_id, record.name].map( + asString, + ) + const toolValues = [record.tool_name, record.name].map(asString) + + return ( + providerValues.some((value) => value === providerName || value === providerDisplayName) && + toolValues.some((value) => value === toolName || value === toolDisplayName) + ) + }) + +const hasKnowledgeDataset = ( + soul: Record, + dataset: NonNullable, +) => { + const knowledge = asRecord(soul.knowledge) + const sets = asArray(knowledge.sets) + + return sets.some((set) => { + const datasets = asArray(asRecord(set).datasets) + + return datasets.some((item) => { + const record = asRecord(item) + return record.id === dataset.id || record.name === dataset.name + }) + }) +} + const getPreseededDataset = async (resourceName: string) => { const query = buildQuery({ keyword: resourceName, limit: '20', page: '1' }) @@ -437,6 +507,89 @@ export async function skipMissingPreseededAgentDriveSkill( } } +export async function skipMissingPreseededFullConfigAgentCoreConfiguration( + world: DifyWorld, + agentName: string, +): Promise<'skipped' | NonNullable> { + const stableModel = await skipMissingAgentBuilderStableChatModel(world) + if (stableModel === 'skipped') return stableModel + + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') return agent + + const summarySkill = await skipMissingPreseededAgentDriveSkill( + world, + agentName, + agentBuilderPreseededResources.summarySkill, + ) + if (summarySkill === 'skipped') return summarySkill + + const jsonTool = await skipMissingPreseededTool( + world, + agentBuilderPreseededResources.jsonReplaceTool, + ) + if (jsonTool === 'skipped') return jsonTool + + const knowledgeBase = await skipMissingReadyPreseededDataset( + world, + agentBuilderPreseededResources.agentKnowledgeBase, + ) + if (knowledgeBase === 'skipped') return knowledgeBase + + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) + await expectApiResponseOK(response, `Check preseeded Agent core configuration ${agentName}`) + const body = (await response.json()) as AgentComposerResponse + const soul = body.agent_soul ?? {} + const missing: string[] = [] + + const model = asRecord(soul.model) + if (model.model_provider !== stableModel.provider || model.model !== stableModel.name) + missing.push(`${agentBuilderPreseededResources.stableChatModel} model config`) + + const prompt = asString(asRecord(soul.prompt).system_prompt) + if (!prompt.includes(agentBuilderExpectedTokens.agentReply)) + missing.push(`Prompt token ${agentBuilderExpectedTokens.agentReply}`) + + const files = asArray(asRecord(soul.files).files) + for (const fileName of [ + agentBuilderTestMaterials.smallFile, + agentBuilderTestMaterials.specialFilename, + ]) { + if (!hasNamedOrKeyedEntry(files, fileName)) missing.push(`file ${fileName}`) + } + + const [providerName = '', toolName = ''] = jsonTool.id.split('/') + const parsedTool = splitToolDisplayName(agentBuilderPreseededResources.jsonReplaceTool) + if ( + parsedTool.ok && + !hasToolEntry(asArray(asRecord(soul.tools).dify_tools), { + providerDisplayName: parsedTool.providerName, + providerName, + toolDisplayName: parsedTool.toolName, + toolName, + }) + ) { + missing.push(agentBuilderPreseededResources.jsonReplaceTool) + } + + if (!hasKnowledgeDataset(soul, knowledgeBase)) + missing.push(agentBuilderPreseededResources.agentKnowledgeBase) + + if (missing.length > 0) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" is missing core fixture configuration: ${missing.join(', ')}.`, + ) + } + + return agent + } finally { + await ctx.dispose() + } +} + export async function skipMissingPreseededAgentFileTreeFixture( world: DifyWorld, agentName: string, From 9e9288f28af49d10095e2ba86bd09e344a9aa049 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 13:31:14 +0800 Subject: [PATCH 031/185] test(e2e): verify agent builder tool state readiness --- e2e/AGENTS.md | 2 + e2e/features/agent-v2/preflight.feature | 4 + .../agent-v2/preflight.steps.ts | 12 ++ e2e/support/preflight.ts | 104 +++++++++++++++++- 4 files changed, 120 insertions(+), 2 deletions(-) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 7da2a7cfd11600..17ec2cb770ef6a 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -306,6 +306,8 @@ Use `the Agent Builder preseeded Agent "{agent}" includes drive skill "{skill}"` Use `the Agent Builder preseeded Agent "{agent}" includes the core fixture configuration` for the fixed Full Config Agent prerequisite. It composes the stable model, Summary Skill, JSON Replace tool, and indexed knowledge-base preflights, then reads `/console/api/agent/{agent_id}/composer` to verify the Agent Soul contains the selected model, prompt success token, required file fixtures, JSON Replace tool entry, and knowledge dataset reference. Do not use this step for Agent node output variables; those live in workflow node-job `declared_outputs`, not the roster Agent App composer response. +Use `the Agent Builder preseeded Agent "{agent}" includes the tool state fixture configuration` for the fixed Tool States Agent prerequisite. It composes the Summary Skill, JSON Replace tool, and Tavily Search tool preflights, then reads `/console/api/agent/{agent_id}/composer` to verify the Agent Soul includes JSON Replace, Tavily Search, and a Tavily credential reference. This proves the seed is configured to exercise tool status UI; keep actual invalid-credential errors in dependent user-visible configuration or runtime scenarios. + Use `the Agent Builder preseeded Agent "{agent}" includes the file tree fixture files` for file-tree display prerequisites. It verifies the Agent drive contains every file from `agentBuilderFileTreeFixtureFiles` through `/console/api/agent/{agent_id}/drive/files?prefix=files/`. This proves the committed drive file set is ready; keep visual tree expansion, nesting, and preview assertions in dependent user-visible scenarios. Use `the Agent Builder preseeded Agent "{agent}" has published Web app access` to verify that a fixed Agent is published, Web app access is enabled, and the Agent detail response includes the site token and base URL needed to open the Web app. Keep Web app launch and runtime assertions in dependent user-visible scenarios. diff --git a/e2e/features/agent-v2/preflight.feature b/e2e/features/agent-v2/preflight.feature index 86d3c167119678..28b91675ce16be 100644 --- a/e2e/features/agent-v2/preflight.feature +++ b/e2e/features/agent-v2/preflight.feature @@ -40,6 +40,10 @@ Feature: Agent Builder preseeded environment Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" is available + Scenario: Tool states Agent includes tool state fixture configuration + Given I am signed in as the default E2E admin + And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" includes the tool state fixture configuration + Scenario: File tree Agent includes fixture files Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E Agent With File Tree" includes the file tree fixture files diff --git a/e2e/features/step-definitions/agent-v2/preflight.steps.ts b/e2e/features/step-definitions/agent-v2/preflight.steps.ts index 4ac96764a3c0f8..fa2108ec0d3cb0 100644 --- a/e2e/features/step-definitions/agent-v2/preflight.steps.ts +++ b/e2e/features/step-definitions/agent-v2/preflight.steps.ts @@ -13,6 +13,7 @@ import { skipMissingPreseededDataset, skipMissingPreseededFullConfigAgentCoreConfiguration, skipMissingPreseededTool, + skipMissingPreseededToolStatesAgentConfiguration, skipMissingPreseededWorkflow, skipMissingReadyPreseededDataset, } from '../../../support/preflight' @@ -111,6 +112,17 @@ Given( }, ) +Given( + 'the Agent Builder preseeded Agent {string} includes the tool state fixture configuration', + async function (this: DifyWorld, agentName: string) { + const resource = await skipMissingPreseededToolStatesAgentConfiguration(this, agentName) + if (resource === 'skipped') return resource + + this.agentBuilderPreseededResources[`${agentName} / tool state fixture configuration`] = + resource + }, +) + Given( 'the Agent Builder preseeded Agent {string} includes the file tree fixture files', async function (this: DifyWorld, agentName: string) { diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index 43604b37ada6ae..9caf3de9030556 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -229,7 +229,7 @@ const hasNamedOrKeyedEntry = (items: unknown[], expectedName: string) => return values.some((value) => value === expectedName || value.endsWith(`/${expectedName}`)) }) -const hasToolEntry = ( +const findToolEntry = ( items: unknown[], { providerDisplayName, @@ -243,7 +243,7 @@ const hasToolEntry = ( toolName: string }, ) => - items.some((item) => { + items.find((item) => { const record = asRecord(item) const providerValues = [record.provider_id, record.provider, record.plugin_id, record.name].map( asString, @@ -256,6 +256,27 @@ const hasToolEntry = ( ) }) +const hasToolEntry = ( + items: unknown[], + tool: { + providerDisplayName: string + providerName: string + toolDisplayName: string + toolName: string + }, +) => Boolean(findToolEntry(items, tool)) + +const hasToolCredentialReference = (item: unknown) => { + const record = asRecord(item) + const credentialRef = asRecord(record.credential_ref) + const credentialType = asString(record.credential_type) + + return ( + (credentialType === 'api-key' || credentialType === 'oauth2') && + (Boolean(asString(credentialRef.id)) || Boolean(asString(record.credential_id))) + ) +} + const hasKnowledgeDataset = ( soul: Record, dataset: NonNullable, @@ -590,6 +611,85 @@ export async function skipMissingPreseededFullConfigAgentCoreConfiguration( } } +export async function skipMissingPreseededToolStatesAgentConfiguration( + world: DifyWorld, + agentName: string, +): Promise<'skipped' | NonNullable> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') return agent + + const summarySkill = await skipMissingPreseededAgentDriveSkill( + world, + agentName, + agentBuilderPreseededResources.summarySkill, + ) + if (summarySkill === 'skipped') return summarySkill + + const jsonTool = await skipMissingPreseededTool( + world, + agentBuilderPreseededResources.jsonReplaceTool, + ) + if (jsonTool === 'skipped') return jsonTool + + const tavilyTool = await skipMissingPreseededTool( + world, + agentBuilderPreseededResources.tavilySearchTool, + ) + if (tavilyTool === 'skipped') return tavilyTool + + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) + await expectApiResponseOK(response, `Check preseeded Agent tool states ${agentName}`) + const body = (await response.json()) as AgentComposerResponse + const soul = body.agent_soul ?? {} + const toolItems = asArray(asRecord(soul.tools).dify_tools) + const missing: string[] = [] + + const [jsonProviderName = '', jsonToolName = ''] = jsonTool.id.split('/') + const parsedJsonTool = splitToolDisplayName(agentBuilderPreseededResources.jsonReplaceTool) + if ( + parsedJsonTool.ok && + !findToolEntry(toolItems, { + providerDisplayName: parsedJsonTool.providerName, + providerName: jsonProviderName, + toolDisplayName: parsedJsonTool.toolName, + toolName: jsonToolName, + }) + ) { + missing.push(agentBuilderPreseededResources.jsonReplaceTool) + } + + const [tavilyProviderName = '', tavilyToolName = ''] = tavilyTool.id.split('/') + const parsedTavilyTool = splitToolDisplayName(agentBuilderPreseededResources.tavilySearchTool) + const tavilyEntry = parsedTavilyTool.ok + ? findToolEntry(toolItems, { + providerDisplayName: parsedTavilyTool.providerName, + providerName: tavilyProviderName, + toolDisplayName: parsedTavilyTool.toolName, + toolName: tavilyToolName, + }) + : undefined + + if (!tavilyEntry) { + missing.push(agentBuilderPreseededResources.tavilySearchTool) + } else if (!hasToolCredentialReference(tavilyEntry)) { + missing.push(`${agentBuilderPreseededResources.tavilySearchTool} credential reference`) + } + + if (missing.length > 0) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" is missing tool state fixture configuration: ${missing.join(', ')}.`, + ) + } + + return agent + } finally { + await ctx.dispose() + } +} + export async function skipMissingPreseededAgentFileTreeFixture( world: DifyWorld, agentName: string, From 79505bf713b7a2ab522812b25d0cbe4625718a9b Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 13:36:11 +0800 Subject: [PATCH 032/185] test(e2e): verify agent builder dual retrieval readiness --- e2e/AGENTS.md | 2 + e2e/features/agent-v2/preflight.feature | 4 + .../agent-v2/preflight.steps.ts | 12 +++ e2e/support/preflight.ts | 77 +++++++++++++++++++ 4 files changed, 95 insertions(+) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 17ec2cb770ef6a..dbb7360caf9d8b 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -308,6 +308,8 @@ Use `the Agent Builder preseeded Agent "{agent}" includes the core fixture confi Use `the Agent Builder preseeded Agent "{agent}" includes the tool state fixture configuration` for the fixed Tool States Agent prerequisite. It composes the Summary Skill, JSON Replace tool, and Tavily Search tool preflights, then reads `/console/api/agent/{agent_id}/composer` to verify the Agent Soul includes JSON Replace, Tavily Search, and a Tavily credential reference. This proves the seed is configured to exercise tool status UI; keep actual invalid-credential errors in dependent user-visible configuration or runtime scenarios. +Use `the Agent Builder preseeded Agent "{agent}" includes the dual retrieval fixture configuration` for the fixed Dual Retrieval Agent prerequisite. It composes the indexed knowledge-base preflight, then reads `/console/api/agent/{agent_id}/composer` to verify `agent_soul.knowledge.sets` includes both an Agent-decide generated query set and a custom user-query set using the fixed custom query. This proves the seed is configured to exercise the Knowledge Retrieval display; keep retrieval hit results, labels, expand/collapse behavior, and runtime assertions in dependent user-visible scenarios. + Use `the Agent Builder preseeded Agent "{agent}" includes the file tree fixture files` for file-tree display prerequisites. It verifies the Agent drive contains every file from `agentBuilderFileTreeFixtureFiles` through `/console/api/agent/{agent_id}/drive/files?prefix=files/`. This proves the committed drive file set is ready; keep visual tree expansion, nesting, and preview assertions in dependent user-visible scenarios. Use `the Agent Builder preseeded Agent "{agent}" has published Web app access` to verify that a fixed Agent is published, Web app access is enabled, and the Agent detail response includes the site token and base URL needed to open the Web app. Keep Web app launch and runtime assertions in dependent user-visible scenarios. diff --git a/e2e/features/agent-v2/preflight.feature b/e2e/features/agent-v2/preflight.feature index 28b91675ce16be..f693d39bc59985 100644 --- a/e2e/features/agent-v2/preflight.feature +++ b/e2e/features/agent-v2/preflight.feature @@ -52,6 +52,10 @@ Feature: Agent Builder preseeded environment Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E Agent With Dual Retrieval" is available + Scenario: Dual retrieval Agent includes dual retrieval fixture configuration + Given I am signed in as the default E2E admin + And the Agent Builder preseeded Agent "E2E Agent With Dual Retrieval" includes the dual retrieval fixture configuration + Scenario: Published Web app Agent exposes Web app access Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E Agent Published Web App" has published Web app access diff --git a/e2e/features/step-definitions/agent-v2/preflight.steps.ts b/e2e/features/step-definitions/agent-v2/preflight.steps.ts index fa2108ec0d3cb0..f9453bb11c03d8 100644 --- a/e2e/features/step-definitions/agent-v2/preflight.steps.ts +++ b/e2e/features/step-definitions/agent-v2/preflight.steps.ts @@ -11,6 +11,7 @@ import { skipMissingPreseededAgentPublishedWebApp, skipMissingPreseededAgentWorkflowReference, skipMissingPreseededDataset, + skipMissingPreseededDualRetrievalAgentConfiguration, skipMissingPreseededFullConfigAgentCoreConfiguration, skipMissingPreseededTool, skipMissingPreseededToolStatesAgentConfiguration, @@ -123,6 +124,17 @@ Given( }, ) +Given( + 'the Agent Builder preseeded Agent {string} includes the dual retrieval fixture configuration', + async function (this: DifyWorld, agentName: string) { + const resource = await skipMissingPreseededDualRetrievalAgentConfiguration(this, agentName) + if (resource === 'skipped') return resource + + this.agentBuilderPreseededResources[`${agentName} / dual retrieval fixture configuration`] = + resource + }, +) + Given( 'the Agent Builder preseeded Agent {string} includes the file tree fixture files', async function (this: DifyWorld, agentName: string) { diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index 9caf3de9030556..4c205f2f7e0253 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -1,6 +1,7 @@ import type { DifyWorld } from '../features/support/world' import { agentBuilderExpectedTokens, + agentBuilderFixedInputs, agentBuilderPreseededResources, } from './agent-builder-resources' import { createApiContext, expectApiResponseOK } from './api' @@ -294,6 +295,36 @@ const hasKnowledgeDataset = ( }) } +const hasKnowledgeSet = ( + soul: Record, + dataset: NonNullable, + { + queryMode, + queryValue, + }: { + queryMode: 'generated_query' | 'user_query' + queryValue?: string + }, +) => { + const knowledge = asRecord(soul.knowledge) + const sets = asArray(knowledge.sets) + + return sets.some((set) => { + const record = asRecord(set) + const query = asRecord(record.query) + const datasets = asArray(record.datasets) + const hasExpectedDataset = datasets.some((item) => { + const datasetRecord = asRecord(item) + return datasetRecord.id === dataset.id || datasetRecord.name === dataset.name + }) + + if (!hasExpectedDataset || query.mode !== queryMode) return false + if (queryValue === undefined) return true + + return asString(query.value).trim() === queryValue + }) +} + const getPreseededDataset = async (resourceName: string) => { const query = buildQuery({ keyword: resourceName, limit: '20', page: '1' }) @@ -690,6 +721,52 @@ export async function skipMissingPreseededToolStatesAgentConfiguration( } } +export async function skipMissingPreseededDualRetrievalAgentConfiguration( + world: DifyWorld, + agentName: string, +): Promise<'skipped' | NonNullable> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') return agent + + const knowledgeBase = await skipMissingReadyPreseededDataset( + world, + agentBuilderPreseededResources.agentKnowledgeBase, + ) + if (knowledgeBase === 'skipped') return knowledgeBase + + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) + await expectApiResponseOK(response, `Check preseeded Agent dual retrieval ${agentName}`) + const body = (await response.json()) as AgentComposerResponse + const soul = body.agent_soul ?? {} + const missing: string[] = [] + + if (!hasKnowledgeSet(soul, knowledgeBase, { queryMode: 'generated_query' })) + missing.push('Agent decide Knowledge Retrieval') + + if ( + !hasKnowledgeSet(soul, knowledgeBase, { + queryMode: 'user_query', + queryValue: agentBuilderFixedInputs.customKnowledgeQuery, + }) + ) { + missing.push('Custom query Knowledge Retrieval') + } + + if (missing.length > 0) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" is missing dual retrieval fixture configuration: ${missing.join(', ')}.`, + ) + } + + return agent + } finally { + await ctx.dispose() + } +} + export async function skipMissingPreseededAgentFileTreeFixture( world: DifyWorld, agentName: string, From fd3d984162f830cc14d151f6efe5f78623821b5d Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 13:52:22 +0800 Subject: [PATCH 033/185] chore(e2e): align package with root lint --- e2e/AGENTS.md | 10 +- .../agent-v2/access-point.steps.ts | 12 +- .../agent-v2/preflight.steps.ts | 56 +++-- .../apps/duplicate-app.steps.ts | 3 +- .../apps/switch-app-mode.steps.ts | 3 +- e2e/fixtures/auth.ts | 9 +- e2e/package.json | 3 +- e2e/scripts/common.ts | 25 +- e2e/scripts/run-cucumber.ts | 17 +- e2e/scripts/setup.ts | 16 +- e2e/support/api.ts | 27 ++- e2e/support/datasets.ts | 3 +- e2e/support/preflight.ts | 223 ++++++++++-------- e2e/support/process.ts | 18 +- e2e/support/tools.ts | 3 +- e2e/support/web-server.ts | 9 +- 16 files changed, 268 insertions(+), 169 deletions(-) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index dbb7360caf9d8b..9f7409d78186a4 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -26,12 +26,16 @@ Install Playwright browsers once: ```bash pnpm install pnpm -C e2e e2e:install -pnpm -C e2e check ``` `pnpm install` is resolved through the repository workspace and uses the shared root lockfile plus `pnpm-workspace.yaml`. -Use `pnpm -C e2e check` as the default local verification step after editing E2E TypeScript, Cucumber support code, or feature glue. It runs ESLint autofix, linting, and type checks for this package. +Use root lint plus the package type check as the default local verification step after editing E2E TypeScript, Cucumber support code, or feature glue: + +```bash +vpr lint --fix +pnpm -C e2e type-check +``` Common commands: @@ -174,7 +178,7 @@ open cucumber-report/report.html 1. Add step definitions under `features/step-definitions//` 1. Reuse existing steps from `common/` and other definition files before writing new ones 1. Run with `pnpm -C e2e e2e -- --tags @your-tag` to verify -1. Run `pnpm -C e2e check` before committing +1. Run `vpr lint --fix` from the repository root and `pnpm -C e2e type-check` before committing ### Feature file conventions diff --git a/e2e/features/step-definitions/agent-v2/access-point.steps.ts b/e2e/features/step-definitions/agent-v2/access-point.steps.ts index b034ee0d580d67..405f8f06fca8ba 100644 --- a/e2e/features/step-definitions/agent-v2/access-point.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point.steps.ts @@ -5,7 +5,8 @@ import { getAgentAccessPath, setAgentApiAccess } from '../../../support/agent' const getCurrentAgentId = (world: DifyWorld) => { const agentId = world.createdAgentIds.at(-1) - if (!agentId) throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') + if (!agentId) + throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') return agentId } @@ -114,7 +115,8 @@ Then('I should see the newly generated Agent v2 API key once', async function (t await expect(generatedKeyDialog.getByLabel('Copy')).toBeVisible() this.lastGeneratedAgentApiKey = (await generatedKey.textContent())?.trim() - if (!this.lastGeneratedAgentApiKey) throw new Error('Generated Agent v2 API key was empty.') + if (!this.lastGeneratedAgentApiKey) + throw new Error('Generated Agent v2 API key was empty.') }) When('I close the newly generated Agent v2 API key', async function (this: DifyWorld) { @@ -129,7 +131,8 @@ Then( 'the Agent v2 API key list should not expose the full generated secret', async function (this: DifyWorld) { const fullSecret = this.lastGeneratedAgentApiKey - if (!fullSecret) throw new Error('No generated Agent v2 API key found.') + if (!fullSecret) + throw new Error('No generated Agent v2 API key found.') const apiKeyDialog = this.getPage().getByRole('dialog', { name: /API Secret key/i }) @@ -163,7 +166,8 @@ When('I open the Agent v2 API Reference', async function (this: DifyWorld) { Then('the Agent v2 API Reference should open in a new tab', async function (this: DifyWorld) { const apiReferencePage = this.lastAgentApiReferencePage - if (!apiReferencePage) throw new Error('No Agent v2 API Reference page was opened.') + if (!apiReferencePage) + throw new Error('No Agent v2 API Reference page was opened.') await expect(apiReferencePage).toHaveURL(/developing-with-apis/) await apiReferencePage.close() diff --git a/e2e/features/step-definitions/agent-v2/preflight.steps.ts b/e2e/features/step-definitions/agent-v2/preflight.steps.ts index f9453bb11c03d8..fbffb1548babb6 100644 --- a/e2e/features/step-definitions/agent-v2/preflight.steps.ts +++ b/e2e/features/step-definitions/agent-v2/preflight.steps.ts @@ -21,14 +21,16 @@ import { Given('the Agent Builder stable chat model is available', async function (this: DifyWorld) { const stableModel = await skipMissingAgentBuilderStableChatModel(this) - if (stableModel === 'skipped') return stableModel + if (stableModel === 'skipped') + return stableModel this.agentBuilderStableChatModel = stableModel }) Given('the Agent Builder broken chat model is available', async function (this: DifyWorld) { const brokenModel = await skipMissingAgentBuilderBrokenChatModel(this) - if (brokenModel === 'skipped') return brokenModel + if (brokenModel === 'skipped') + return brokenModel this.agentBuilderBrokenChatModel = brokenModel }) @@ -37,7 +39,8 @@ Given( 'the Agent Builder preseeded Agent {string} is available', async function (this: DifyWorld, resourceName: string) { const resource = await skipMissingPreseededAgent(this, resourceName) - if (resource === 'skipped') return resource + if (resource === 'skipped') + return resource this.agentBuilderPreseededResources[resourceName] = resource }, @@ -47,7 +50,8 @@ Given( 'the Agent Builder preseeded workflow {string} is available', async function (this: DifyWorld, resourceName: string) { const resource = await skipMissingPreseededWorkflow(this, resourceName) - if (resource === 'skipped') return resource + if (resource === 'skipped') + return resource this.agentBuilderPreseededResources[resourceName] = resource }, @@ -57,7 +61,8 @@ Given( 'the Agent Builder preseeded dataset {string} is available', async function (this: DifyWorld, resourceName: string) { const resource = await skipMissingPreseededDataset(this, resourceName) - if (resource === 'skipped') return resource + if (resource === 'skipped') + return resource this.agentBuilderPreseededResources[resourceName] = resource }, @@ -67,7 +72,8 @@ Given( 'the Agent Builder preseeded dataset {string} is indexed and ready', async function (this: DifyWorld, resourceName: string) { const resource = await skipMissingReadyPreseededDataset(this, resourceName) - if (resource === 'skipped') return resource + if (resource === 'skipped') + return resource this.agentBuilderPreseededResources[resourceName] = resource }, @@ -77,7 +83,8 @@ Given( 'the Agent Builder preseeded dataset {string} is indexing', async function (this: DifyWorld, resourceName: string) { const resource = await skipMissingIndexingPreseededDataset(this, resourceName) - if (resource === 'skipped') return resource + if (resource === 'skipped') + return resource this.agentBuilderPreseededResources[resourceName] = resource }, @@ -87,7 +94,8 @@ Given( 'the Agent Builder preseeded tool {string} is available', async function (this: DifyWorld, resourceName: string) { const resource = await skipMissingPreseededTool(this, resourceName) - if (resource === 'skipped') return resource + if (resource === 'skipped') + return resource this.agentBuilderPreseededResources[resourceName] = resource }, @@ -97,7 +105,8 @@ Given( 'the Agent Builder preseeded Agent {string} includes drive skill {string}', async function (this: DifyWorld, agentName: string, skillName: string) { const resource = await skipMissingPreseededAgentDriveSkill(this, agentName, skillName) - if (resource === 'skipped') return resource + if (resource === 'skipped') + return resource this.agentBuilderPreseededResources[`${agentName} / ${skillName}`] = resource }, @@ -107,7 +116,8 @@ Given( 'the Agent Builder preseeded Agent {string} includes the core fixture configuration', async function (this: DifyWorld, agentName: string) { const resource = await skipMissingPreseededFullConfigAgentCoreConfiguration(this, agentName) - if (resource === 'skipped') return resource + if (resource === 'skipped') + return resource this.agentBuilderPreseededResources[`${agentName} / core fixture configuration`] = resource }, @@ -117,10 +127,11 @@ Given( 'the Agent Builder preseeded Agent {string} includes the tool state fixture configuration', async function (this: DifyWorld, agentName: string) { const resource = await skipMissingPreseededToolStatesAgentConfiguration(this, agentName) - if (resource === 'skipped') return resource + if (resource === 'skipped') + return resource - this.agentBuilderPreseededResources[`${agentName} / tool state fixture configuration`] = - resource + this.agentBuilderPreseededResources[`${agentName} / tool state fixture configuration`] + = resource }, ) @@ -128,10 +139,11 @@ Given( 'the Agent Builder preseeded Agent {string} includes the dual retrieval fixture configuration', async function (this: DifyWorld, agentName: string) { const resource = await skipMissingPreseededDualRetrievalAgentConfiguration(this, agentName) - if (resource === 'skipped') return resource + if (resource === 'skipped') + return resource - this.agentBuilderPreseededResources[`${agentName} / dual retrieval fixture configuration`] = - resource + this.agentBuilderPreseededResources[`${agentName} / dual retrieval fixture configuration`] + = resource }, ) @@ -139,7 +151,8 @@ Given( 'the Agent Builder preseeded Agent {string} includes the file tree fixture files', async function (this: DifyWorld, agentName: string) { const resource = await skipMissingPreseededAgentFileTreeFixture(this, agentName) - if (resource === 'skipped') return resource + if (resource === 'skipped') + return resource this.agentBuilderPreseededResources[`${agentName} / file tree fixture`] = resource }, @@ -149,7 +162,8 @@ Given( 'the Agent Builder preseeded Agent {string} has Backend service API access with an API key', async function (this: DifyWorld, agentName: string) { const resource = await skipMissingPreseededAgentBackendApiKey(this, agentName) - if (resource === 'skipped') return resource + if (resource === 'skipped') + return resource this.agentBuilderPreseededResources[`${agentName} / Backend service API key`] = resource }, @@ -159,7 +173,8 @@ Given( 'the Agent Builder preseeded Agent {string} has published Web app access', async function (this: DifyWorld, agentName: string) { const resource = await skipMissingPreseededAgentPublishedWebApp(this, agentName) - if (resource === 'skipped') return resource + if (resource === 'skipped') + return resource this.agentBuilderPreseededResources[`${agentName} / Web app`] = resource }, @@ -169,7 +184,8 @@ Given( 'the Agent Builder preseeded Agent {string} is referenced by workflow {string}', async function (this: DifyWorld, agentName: string, workflowName: string) { const resource = await skipMissingPreseededAgentWorkflowReference(this, agentName, workflowName) - if (resource === 'skipped') return resource + if (resource === 'skipped') + return resource this.agentBuilderPreseededResources[`${agentName} / ${workflowName}`] = resource }, diff --git a/e2e/features/step-definitions/apps/duplicate-app.steps.ts b/e2e/features/step-definitions/apps/duplicate-app.steps.ts index bc5fd2ddee099b..aa8c06f5df8f02 100644 --- a/e2e/features/step-definitions/apps/duplicate-app.steps.ts +++ b/e2e/features/step-definitions/apps/duplicate-app.steps.ts @@ -13,7 +13,8 @@ Given('there is an existing E2E app available for testing', async function (this When('I open the options menu for the last created E2E app', async function (this: DifyWorld) { const appName = this.lastCreatedAppName - if (!appName) throw new Error('No app name stored. Run "I enter a unique E2E app name" first.') + if (!appName) + throw new Error('No app name stored. Run "I enter a unique E2E app name" first.') const page = this.getPage() const appLink = page.getByRole('link', { name: appName, exact: true }) diff --git a/e2e/features/step-definitions/apps/switch-app-mode.steps.ts b/e2e/features/step-definitions/apps/switch-app-mode.steps.ts index 20f513f471bf9c..cfaf21a66bde54 100644 --- a/e2e/features/step-definitions/apps/switch-app-mode.steps.ts +++ b/e2e/features/step-definitions/apps/switch-app-mode.steps.ts @@ -24,5 +24,6 @@ Then('I should land on the switched app', async function (this: DifyWorld) { // Capture the new app's ID so the After hook can clean it up const match = page.url().match(/\/app\/([^/]+)\/workflow/) - if (match?.[1]) this.createdAppIds.push(match[1]) + if (match?.[1]) + this.createdAppIds.push(match[1]) }) diff --git a/e2e/fixtures/auth.ts b/e2e/fixtures/auth.ts index 0356c1d1ceba04..9039f97483de55 100644 --- a/e2e/fixtures/auth.ts +++ b/e2e/fixtures/auth.ts @@ -58,7 +58,8 @@ const getRemainingTimeout = (deadline: number) => Math.max(deadline - Date.now() const encodeField = (value: string) => Buffer.from(value, 'utf8').toString('base64') const assertAPIResponse = async (response: APIResponse, action: string) => { - if (response.ok()) return + if (response.ok()) + return const body = await response.text().catch(() => '') throw new Error( @@ -89,7 +90,8 @@ const postConsoleAPI = async ( const validateInitPasswordIfNeeded = async (context: BrowserContext, deadline: number) => { const initStatus = await getConsoleAPI(context, '/console/api/init', deadline) - if (initStatus.status === 'finished') return false + if (initStatus.status === 'finished') + return false console.warn('[e2e] auth bootstrap: validating init password') await postConsoleAPI(context, '/console/api/init', deadline, { password: initPassword }) @@ -165,7 +167,8 @@ export const ensureAuthenticatedState = async (browser: Browser, configuredBaseU } await writeFile(authMetadataPath, `${JSON.stringify(metadata, null, 2)}\n`, 'utf8') - } finally { + } + finally { await context.close() } } diff --git a/e2e/package.json b/e2e/package.json index 7c83203c27cbb4..d435754376765b 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -1,9 +1,8 @@ { "name": "dify-e2e", - "private": true, "type": "module", + "private": true, "scripts": { - "check": "vp check --fix", "e2e": "tsx ./scripts/run-cucumber.ts", "e2e:full": "tsx ./scripts/run-cucumber.ts --full", "e2e:full:headed": "tsx ./scripts/run-cucumber.ts --full --headed", diff --git a/e2e/scripts/common.ts b/e2e/scripts/common.ts index 21f8f0fe5675e1..20b9ab7fbd830a 100644 --- a/e2e/scripts/common.ts +++ b/e2e/scripts/common.ts @@ -51,7 +51,8 @@ const formatCommand = (command: string, args: string[]) => [command, ...args].jo export const isMainModule = (metaUrl: string) => { const entrypoint = process.argv[1] - if (!entrypoint) return false + if (!entrypoint) + return false return pathToFileURL(entrypoint).href === metaUrl } @@ -110,7 +111,8 @@ export const runCommandOrThrow = async (options: RunCommandOptions) => { const forwardSignalsToChild = (childProcess: ChildProcess) => { const handleSignal = (signal: NodeJS.Signals) => { - if (childProcess.exitCode === null) childProcess.kill(signal) + if (childProcess.exitCode === null) + childProcess.kill(signal) } const onSigint = () => handleSignal('SIGINT') @@ -155,7 +157,8 @@ export const runForegroundProcess = async ({ export const ensureFileExists = async (filePath: string, exampleFilePath: string) => { try { await access(filePath) - } catch { + } + catch { await copyFile(exampleFilePath, filePath) } } @@ -165,9 +168,10 @@ export const ensureLineInFile = async (filePath: string, line: string) => { const lines = fileContent.split(/\r?\n/) const assignmentPrefix = line.includes('=') ? `${line.slice(0, line.indexOf('='))}=` : null - if (lines.includes(line)) return + if (lines.includes(line)) + return - if (assignmentPrefix && lines.some((existingLine) => existingLine.startsWith(assignmentPrefix))) + if (assignmentPrefix && lines.some(existingLine => existingLine.startsWith(assignmentPrefix))) return const normalizedContent = fileContent.endsWith('\n') ? fileContent : `${fileContent}\n` @@ -190,16 +194,16 @@ export const readSimpleDotenv = async (filePath: string) => { const fileContent = await readFile(filePath, 'utf8') const entries = fileContent .split(/\r?\n/) - .map((line) => line.trim()) - .filter((line) => line && !line.startsWith('#')) + .map(line => line.trim()) + .filter(line => line && !line.startsWith('#')) .map<[string, string]>((line) => { const separatorIndex = line.indexOf('=') const key = separatorIndex === -1 ? line : line.slice(0, separatorIndex).trim() const rawValue = separatorIndex === -1 ? '' : line.slice(separatorIndex + 1).trim() if ( - (rawValue.startsWith('"') && rawValue.endsWith('"')) || - (rawValue.startsWith("'") && rawValue.endsWith("'")) + (rawValue.startsWith('"') && rawValue.endsWith('"')) + || (rawValue.startsWith('\'') && rawValue.endsWith('\'')) ) { return [key, rawValue.slice(1, -1)] } @@ -224,7 +228,8 @@ export const waitForCondition = async ({ const deadline = Date.now() + timeoutMs while (Date.now() < deadline) { - if (await check()) return + if (await check()) + return await sleep(intervalMs) } diff --git a/e2e/scripts/run-cucumber.ts b/e2e/scripts/run-cucumber.ts index fb105e60faa7ad..3c8e895e90f569 100644 --- a/e2e/scripts/run-cucumber.ts +++ b/e2e/scripts/run-cucumber.ts @@ -40,16 +40,18 @@ const parseArgs = (argv: string[]): RunOptions => { } const hasCustomTags = (forwardArgs: string[]) => - forwardArgs.some((arg) => arg === '--tags' || arg.startsWith('--tags=')) + forwardArgs.some(arg => arg === '--tags' || arg.startsWith('--tags=')) const main = async () => { const { forwardArgs, full, headed } = parseArgs(process.argv.slice(2)) const startMiddlewareForRun = full const resetStateForRun = full - if (resetStateForRun) await resetState() + if (resetStateForRun) + await resetState() - if (startMiddlewareForRun) await startMiddleware() + if (startMiddlewareForRun) + await startMiddleware() const cucumberReportDir = path.join(e2eDir, 'cucumber-report') const logDir = path.join(e2eDir, '.logs') @@ -84,7 +86,8 @@ const main = async () => { if (startMiddlewareForRun) { try { await stopMiddleware() - } catch { + } + catch { // Cleanup should continue even if middleware shutdown fails. } } @@ -106,7 +109,8 @@ const main = async () => { try { try { await waitForUrl(`${apiURL}/health`, 180_000, 1_000) - } catch { + } + catch { throw new Error(`API did not become ready at ${apiURL}/health.`) } @@ -142,7 +146,8 @@ const main = async () => { }) process.exitCode = result.exitCode - } finally { + } + finally { process.off('SIGINT', onTerminate) process.off('SIGTERM', onTerminate) await cleanup() diff --git a/e2e/scripts/setup.ts b/e2e/scripts/setup.ts index bb37b616e1f72a..3f77a3f72a7e54 100644 --- a/e2e/scripts/setup.ts +++ b/e2e/scripts/setup.ts @@ -79,7 +79,8 @@ const getContainerHealth = async (containerId: string) => { stdio: 'pipe', }) - if (result.exitCode !== 0) return '' + if (result.exitCode !== 0) + return '' return result.stdout.trim() } @@ -105,7 +106,8 @@ const waitForDependency = async ({ try { await wait() - } catch (error) { + } + catch (error) { await printComposeLogs(services) throw error } @@ -134,7 +136,7 @@ export const ensureWebBuild = async () => { .then(() => true) .catch(() => false), readFile(webBuildEnvStampPath, 'utf8') - .then((value) => value.trim()) + .then(value => value.trim()) .catch(() => ''), ]) @@ -142,7 +144,8 @@ export const ensureWebBuild = async () => { console.log('Reusing existing web build artifact.') return } - } catch { + } + catch { // Fall through to rebuild when the existing build cannot be verified. } @@ -237,7 +240,8 @@ export const resetState = async () => { console.log('Stopping middleware services...') try { await stopMiddleware() - } catch { + } + catch { // Reset should continue even if middleware is already stopped. } @@ -251,7 +255,7 @@ export const resetState = async () => { console.log('Removing E2E local state...') await Promise.all( - e2eStatePaths.map((targetPath) => rm(targetPath, { force: true, recursive: true })), + e2eStatePaths.map(targetPath => rm(targetPath, { force: true, recursive: true })), ) console.log('E2E state reset complete.') diff --git a/e2e/support/api.ts b/e2e/support/api.ts index dd0b88b2a31653..f03de5f349b5fb 100644 --- a/e2e/support/api.ts +++ b/e2e/support/api.ts @@ -6,12 +6,12 @@ import { apiURL } from '../test-env' import { createE2EResourceName } from './naming' type StorageState = { - cookies: Array<{ name: string; value: string }> + cookies: Array<{ name: string, value: string }> } export async function createApiContext() { const state = JSON.parse(await readFile(authStatePath, 'utf8')) as StorageState - const csrfToken = state.cookies.find((c) => c.name.endsWith('csrf_token'))?.value ?? '' + const csrfToken = state.cookies.find(c => c.name.endsWith('csrf_token'))?.value ?? '' return request.newContext({ baseURL: apiURL, @@ -21,7 +21,8 @@ export async function createApiContext() { } export async function expectApiResponseOK(response: APIResponse, action: string): Promise { - if (response.ok()) return + if (response.ok()) + return const body = await response.text().catch(() => '') throw new Error(`${action} failed with ${response.status()} ${response.statusText()}: ${body}`) @@ -49,7 +50,8 @@ export async function createTestApp( }) const body = (await response.json()) as AppSeed return body - } finally { + } + finally { await ctx.dispose() } } @@ -76,7 +78,8 @@ export async function syncMinimalWorkflowDraft(appId: string): Promise { conversation_variables: [], }, }) - } finally { + } + finally { await ctx.dispose() } } @@ -85,7 +88,8 @@ export async function deleteTestApp(id: string): Promise { const ctx = await createApiContext() try { await ctx.delete(`/console/api/apps/${id}`) - } finally { + } + finally { await ctx.dispose() } } @@ -132,7 +136,8 @@ export async function syncRunnableWorkflowDraft(appId: string): Promise { conversation_variables: [], }, }) - } finally { + } + finally { await ctx.dispose() } } @@ -143,14 +148,15 @@ export async function publishWorkflowApp(appId: string): Promise { await ctx.post(`/console/api/apps/${appId}/workflows/publish`, { data: { marked_name: '', marked_comment: '' }, }) - } finally { + } + finally { await ctx.dispose() } } export type AppDetailWithSite = { mode?: string - site: { access_token: string; app_base_url: string; enable_site: boolean } + site: { access_token: string, app_base_url: string, enable_site: boolean } } export function getAppSiteURL({ mode, site }: AppDetailWithSite): string { @@ -176,7 +182,8 @@ export async function setAppSiteEnabled( const detailResponse = await ctx.get(`/console/api/apps/${appId}`) await expectApiResponseOK(detailResponse, `Get app site detail for ${appId}`) return (await detailResponse.json()) as AppDetailWithSite - } finally { + } + finally { await ctx.dispose() } } diff --git a/e2e/support/datasets.ts b/e2e/support/datasets.ts index 196b335af326cc..4c0851db26d5f3 100644 --- a/e2e/support/datasets.ts +++ b/e2e/support/datasets.ts @@ -5,7 +5,8 @@ export async function deleteTestDataset(datasetId: string): Promise { try { const response = await ctx.delete(`/console/api/datasets/${datasetId}`) await expectApiResponseOK(response, `Delete dataset ${datasetId}`) - } finally { + } + finally { await ctx.dispose() } } diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index 4c205f2f7e0253..8d99a15b194417 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -17,22 +17,23 @@ const activeModelStatus = 'active' const defaultStableChatModelType = 'llm' const defaultBrokenChatModelName = agentBuilderPreseededResources.brokenModel -export type E2EResourcePrecondition = - | { - ok: true - value: string - } +export type E2EResourcePrecondition + = | { + ok: true + value: string + } | { - ok: false - reason: string - } + ok: false + reason: string + } export const readRequiredEnvResource = ( envName: string, description: string, ): E2EResourcePrecondition => { const value = process.env[envName]?.trim() - if (value) return { ok: true, value } + if (value) + return { ok: true, value } return { ok: false, @@ -53,7 +54,8 @@ export function skipMissingEnvResource( description: string, ): 'skipped' | string { const resource = readRequiredEnvResource(envName, description) - if (resource.ok) return resource.value + if (resource.ok) + return resource.value return skipBlockedPrecondition(world, resource.reason) } @@ -101,13 +103,13 @@ type NamedResourceListResponse = { data: T[] } -type DocumentIndexingStatus = - | 'cleaning' - | 'completed' - | 'indexing' - | 'parsing' - | 'splitting' - | 'waiting' +type DocumentIndexingStatus + = | 'cleaning' + | 'completed' + | 'indexing' + | 'parsing' + | 'splitting' + | 'waiting' type DatasetIndexingStatusResponse = { data: Array<{ @@ -200,8 +202,9 @@ const findConsoleResourceByName = async - return body.data.find((item) => item.name === resourceName) - } finally { + return body.data.find(item => item.name === resourceName) + } + finally { await ctx.dispose() } } @@ -227,7 +230,7 @@ const hasNamedOrKeyedEntry = (items: unknown[], expectedName: string) => asString, ) - return values.some((value) => value === expectedName || value.endsWith(`/${expectedName}`)) + return values.some(value => value === expectedName || value.endsWith(`/${expectedName}`)) }) const findToolEntry = ( @@ -252,8 +255,8 @@ const findToolEntry = ( const toolValues = [record.tool_name, record.name].map(asString) return ( - providerValues.some((value) => value === providerName || value === providerDisplayName) && - toolValues.some((value) => value === toolName || value === toolDisplayName) + providerValues.some(value => value === providerName || value === providerDisplayName) + && toolValues.some(value => value === toolName || value === toolDisplayName) ) }) @@ -273,8 +276,8 @@ const hasToolCredentialReference = (item: unknown) => { const credentialType = asString(record.credential_type) return ( - (credentialType === 'api-key' || credentialType === 'oauth2') && - (Boolean(asString(credentialRef.id)) || Boolean(asString(record.credential_id))) + (credentialType === 'api-key' || credentialType === 'oauth2') + && (Boolean(asString(credentialRef.id)) || Boolean(asString(record.credential_id))) ) } @@ -318,8 +321,10 @@ const hasKnowledgeSet = ( return datasetRecord.id === dataset.id || datasetRecord.name === dataset.name }) - if (!hasExpectedDataset || query.mode !== queryMode) return false - if (queryValue === undefined) return true + if (!hasExpectedDataset || query.mode !== queryMode) + return false + if (queryValue === undefined) + return true return asString(query.value).trim() === queryValue }) @@ -343,7 +348,8 @@ const getDatasetIndexingStatuses = async (datasetId: string, resourceName: strin const body = (await response.json()) as DatasetIndexingStatusResponse return body.data - } finally { + } + finally { await ctx.dispose() } } @@ -357,7 +363,7 @@ const toDatasetResource = ( }) const splitToolDisplayName = (resourceName: string) => { - const [providerName, toolName] = resourceName.split('/').map((item) => item.trim()) + const [providerName, toolName] = resourceName.split('/').map(item => item.trim()) if (!providerName || !toolName) { return { @@ -456,7 +462,7 @@ export async function skipMissingReadyPreseededDataset( } const incompleteStatus = statuses.find( - (item) => item.indexing_status !== completedDocumentIndexingStatus, + item => item.indexing_status !== completedDocumentIndexingStatus, ) if (incompleteStatus) { return skipBlockedPrecondition( @@ -478,13 +484,13 @@ export async function skipMissingIndexingPreseededDataset( return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`) const statuses = await getDatasetIndexingStatuses(resource.id, resourceName) - const indexingStatus = statuses.find((item) => + const indexingStatus = statuses.find(item => activeDocumentIndexingStatuses.has(item.indexing_status ?? ''), ) if (!indexingStatus) { - const actualStatuses = - statuses.map((item) => item.indexing_status ?? 'missing').join(', ') || 'none' + const actualStatuses + = statuses.map(item => item.indexing_status ?? 'missing').join(', ') || 'none' return skipBlockedPrecondition( world, @@ -500,17 +506,18 @@ export async function skipMissingPreseededTool( resourceName: string, ): Promise<'skipped' | NonNullable> { const parsed = splitToolDisplayName(resourceName) - if (!parsed.ok) return skipBlockedPrecondition(world, parsed.reason) + if (!parsed.ok) + return skipBlockedPrecondition(world, parsed.reason) const ctx = await createApiContext() try { const response = await ctx.get('/console/api/workspaces/current/tools/builtin') await expectApiResponseOK(response, `Check preseeded tool ${resourceName}`) const providers = (await response.json()) as BuiltinToolProvider[] - const provider = providers.find((item) => + const provider = providers.find(item => matchesNameOrLabel(parsed.providerName, item.name, item.label), ) - const tool = provider?.tools.find((item) => + const tool = provider?.tools.find(item => matchesNameOrLabel(parsed.toolName, item.name, item.label), ) @@ -522,7 +529,8 @@ export async function skipMissingPreseededTool( kind: 'tool', name: resourceName, } - } finally { + } + finally { await ctx.dispose() } } @@ -533,14 +541,15 @@ export async function skipMissingPreseededAgentDriveSkill( skillName: string, ): Promise<'skipped' | NonNullable> { const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') return agent + if (agent === 'skipped') + return agent const ctx = await createApiContext() try { const response = await ctx.get(`/console/api/agent/${agent.id}/drive/skills`) await expectApiResponseOK(response, `Check preseeded Agent skill ${skillName}`) const body = (await response.json()) as AgentDriveSkillListResponse - const skill = body.items.find((item) => item.name === skillName) + const skill = body.items.find(item => item.name === skillName) if (!skill) { return skipBlockedPrecondition( @@ -554,7 +563,8 @@ export async function skipMissingPreseededAgentDriveSkill( kind: 'skill', name: skill.name, } - } finally { + } + finally { await ctx.dispose() } } @@ -564,29 +574,34 @@ export async function skipMissingPreseededFullConfigAgentCoreConfiguration( agentName: string, ): Promise<'skipped' | NonNullable> { const stableModel = await skipMissingAgentBuilderStableChatModel(world) - if (stableModel === 'skipped') return stableModel + if (stableModel === 'skipped') + return stableModel const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') return agent + if (agent === 'skipped') + return agent const summarySkill = await skipMissingPreseededAgentDriveSkill( world, agentName, agentBuilderPreseededResources.summarySkill, ) - if (summarySkill === 'skipped') return summarySkill + if (summarySkill === 'skipped') + return summarySkill const jsonTool = await skipMissingPreseededTool( world, agentBuilderPreseededResources.jsonReplaceTool, ) - if (jsonTool === 'skipped') return jsonTool + if (jsonTool === 'skipped') + return jsonTool const knowledgeBase = await skipMissingReadyPreseededDataset( world, agentBuilderPreseededResources.agentKnowledgeBase, ) - if (knowledgeBase === 'skipped') return knowledgeBase + if (knowledgeBase === 'skipped') + return knowledgeBase const ctx = await createApiContext() try { @@ -609,14 +624,15 @@ export async function skipMissingPreseededFullConfigAgentCoreConfiguration( agentBuilderTestMaterials.smallFile, agentBuilderTestMaterials.specialFilename, ]) { - if (!hasNamedOrKeyedEntry(files, fileName)) missing.push(`file ${fileName}`) + if (!hasNamedOrKeyedEntry(files, fileName)) + missing.push(`file ${fileName}`) } const [providerName = '', toolName = ''] = jsonTool.id.split('/') const parsedTool = splitToolDisplayName(agentBuilderPreseededResources.jsonReplaceTool) if ( - parsedTool.ok && - !hasToolEntry(asArray(asRecord(soul.tools).dify_tools), { + parsedTool.ok + && !hasToolEntry(asArray(asRecord(soul.tools).dify_tools), { providerDisplayName: parsedTool.providerName, providerName, toolDisplayName: parsedTool.toolName, @@ -637,7 +653,8 @@ export async function skipMissingPreseededFullConfigAgentCoreConfiguration( } return agent - } finally { + } + finally { await ctx.dispose() } } @@ -647,26 +664,30 @@ export async function skipMissingPreseededToolStatesAgentConfiguration( agentName: string, ): Promise<'skipped' | NonNullable> { const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') return agent + if (agent === 'skipped') + return agent const summarySkill = await skipMissingPreseededAgentDriveSkill( world, agentName, agentBuilderPreseededResources.summarySkill, ) - if (summarySkill === 'skipped') return summarySkill + if (summarySkill === 'skipped') + return summarySkill const jsonTool = await skipMissingPreseededTool( world, agentBuilderPreseededResources.jsonReplaceTool, ) - if (jsonTool === 'skipped') return jsonTool + if (jsonTool === 'skipped') + return jsonTool const tavilyTool = await skipMissingPreseededTool( world, agentBuilderPreseededResources.tavilySearchTool, ) - if (tavilyTool === 'skipped') return tavilyTool + if (tavilyTool === 'skipped') + return tavilyTool const ctx = await createApiContext() try { @@ -680,8 +701,8 @@ export async function skipMissingPreseededToolStatesAgentConfiguration( const [jsonProviderName = '', jsonToolName = ''] = jsonTool.id.split('/') const parsedJsonTool = splitToolDisplayName(agentBuilderPreseededResources.jsonReplaceTool) if ( - parsedJsonTool.ok && - !findToolEntry(toolItems, { + parsedJsonTool.ok + && !findToolEntry(toolItems, { providerDisplayName: parsedJsonTool.providerName, providerName: jsonProviderName, toolDisplayName: parsedJsonTool.toolName, @@ -704,7 +725,8 @@ export async function skipMissingPreseededToolStatesAgentConfiguration( if (!tavilyEntry) { missing.push(agentBuilderPreseededResources.tavilySearchTool) - } else if (!hasToolCredentialReference(tavilyEntry)) { + } + else if (!hasToolCredentialReference(tavilyEntry)) { missing.push(`${agentBuilderPreseededResources.tavilySearchTool} credential reference`) } @@ -716,7 +738,8 @@ export async function skipMissingPreseededToolStatesAgentConfiguration( } return agent - } finally { + } + finally { await ctx.dispose() } } @@ -726,13 +749,15 @@ export async function skipMissingPreseededDualRetrievalAgentConfiguration( agentName: string, ): Promise<'skipped' | NonNullable> { const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') return agent + if (agent === 'skipped') + return agent const knowledgeBase = await skipMissingReadyPreseededDataset( world, agentBuilderPreseededResources.agentKnowledgeBase, ) - if (knowledgeBase === 'skipped') return knowledgeBase + if (knowledgeBase === 'skipped') + return knowledgeBase const ctx = await createApiContext() try { @@ -762,7 +787,8 @@ export async function skipMissingPreseededDualRetrievalAgentConfiguration( } return agent - } finally { + } + finally { await ctx.dispose() } } @@ -772,7 +798,8 @@ export async function skipMissingPreseededAgentFileTreeFixture( agentName: string, ): Promise<'skipped' | NonNullable> { const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') return agent + if (agent === 'skipped') + return agent const ctx = await createApiContext() try { @@ -780,10 +807,10 @@ export async function skipMissingPreseededAgentFileTreeFixture( const response = await ctx.get(`/console/api/agent/${agent.id}/drive/files?${query}`) await expectApiResponseOK(response, `Check preseeded Agent file tree ${agentName}`) const body = (await response.json()) as AgentDriveFileListResponse - const keys = (body.items ?? []).map((item) => item.key) + const keys = (body.items ?? []).map(item => item.key) const missingFiles = agentBuilderFileTreeFixtureFiles.filter( - (filePath) => - !keys.some((key) => key === `files/${filePath}` || key.endsWith(`/${filePath}`)), + filePath => + !keys.some(key => key === `files/${filePath}` || key.endsWith(`/${filePath}`)), ) if (missingFiles.length > 0) { @@ -798,7 +825,8 @@ export async function skipMissingPreseededAgentFileTreeFixture( kind: 'agent', name: agent.name, } - } finally { + } + finally { await ctx.dispose() } } @@ -808,7 +836,8 @@ export async function skipMissingPreseededAgentBackendApiKey( agentName: string, ): Promise<'skipped' | NonNullable> { const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') return agent + if (agent === 'skipped') + return agent const ctx = await createApiContext() try { @@ -838,7 +867,8 @@ export async function skipMissingPreseededAgentBackendApiKey( kind: 'api-key', name: `${agentName} Backend service API key`, } - } finally { + } + finally { await ctx.dispose() } } @@ -848,7 +878,8 @@ export async function skipMissingPreseededAgentPublishedWebApp( agentName: string, ): Promise<'skipped' | NonNullable> { const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') return agent + if (agent === 'skipped') + return agent const ctx = await createApiContext() try { @@ -879,7 +910,8 @@ export async function skipMissingPreseededAgentPublishedWebApp( kind: 'agent', name: agent.name, } - } finally { + } + finally { await ctx.dispose() } } @@ -890,10 +922,12 @@ export async function skipMissingPreseededAgentWorkflowReference( workflowName: string, ): Promise<'skipped' | NonNullable> { const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') return agent + if (agent === 'skipped') + return agent const workflow = await skipMissingPreseededWorkflow(world, workflowName) - if (workflow === 'skipped') return workflow + if (workflow === 'skipped') + return workflow const ctx = await createApiContext() try { @@ -901,7 +935,7 @@ export async function skipMissingPreseededAgentWorkflowReference( await expectApiResponseOK(response, `Check preseeded Agent workflow reference ${agentName}`) const references = (await response.json()) as AgentReferencingWorkflowsResponse const reference = references.data.find( - (item) => item.app_id === workflow.id || item.app_name === workflow.name, + item => item.app_id === workflow.id || item.app_name === workflow.name, ) if (!reference) { @@ -923,23 +957,24 @@ export async function skipMissingPreseededAgentWorkflowReference( kind: 'workflow', name: workflow.name, } - } finally { + } + finally { await ctx.dispose() } } -type ModelPreflightConfig = - | { - ok: true - provider: string - resourceName: string - type: string - value: string - } +type ModelPreflightConfig + = | { + ok: true + provider: string + resourceName: string + type: string + value: string + } | { - ok: false - reason: string - } + ok: false + reason: string + } export function readAgentBuilderStableChatModelConfig(): ModelPreflightConfig { const provider = process.env[stableChatModelProviderEnv]?.trim() @@ -947,8 +982,10 @@ export function readAgentBuilderStableChatModelConfig(): ModelPreflightConfig { const type = process.env[stableChatModelTypeEnv]?.trim() || defaultStableChatModelType const missing: string[] = [] - if (!provider) missing.push(stableChatModelProviderEnv) - if (!name) missing.push(stableChatModelNameEnv) + if (!provider) + missing.push(stableChatModelProviderEnv) + if (!name) + missing.push(stableChatModelNameEnv) if (!provider || !name) { return { @@ -996,7 +1033,8 @@ async function skipMissingAgentBuilderModel( requireActive: boolean }, ): Promise<'skipped' | NonNullable> { - if (!config.ok) return skipBlockedPrecondition(world, config.reason) + if (!config.ok) + return skipBlockedPrecondition(world, config.reason) const ctx = await createApiContext() try { @@ -1005,12 +1043,12 @@ async function skipMissingAgentBuilderModel( ) await expectApiResponseOK(response, `Check ${config.resourceName}`) const body = (await response.json()) as ModelTypeListResponse - const provider = body.data.find((item) => item.provider === config.provider) + const provider = body.data.find(item => item.provider === config.provider) const model = provider?.models.find( - (item) => - item.model === config.value || - item.label?.en_US === config.value || - item.label?.zh_Hans === config.value, + item => + item.model === config.value + || item.label?.en_US === config.value + || item.label?.zh_Hans === config.value, ) if (!provider || !model) { @@ -1032,7 +1070,8 @@ async function skipMissingAgentBuilderModel( provider: provider.provider, type: config.type, } - } finally { + } + finally { await ctx.dispose() } } diff --git a/e2e/support/process.ts b/e2e/support/process.ts index 84994bc615b640..b54108e6855390 100644 --- a/e2e/support/process.ts +++ b/e2e/support/process.ts @@ -64,11 +64,14 @@ export const waitForUrl = async ( const response = await fetch(url, { signal: controller.signal, }) - if (response.ok) return - } finally { + if (response.ok) + return + } + finally { clearTimeout(timeout) } - } catch { + } + catch { // Keep polling until timeout. } @@ -141,7 +144,8 @@ const waitForProcessExit = (childProcess: ChildProcess, timeoutMs: number) => const signalManagedProcess = (childProcess: ChildProcess, signal: NodeJS.Signals) => { const { pid } = childProcess - if (!pid) return + if (!pid) + return try { if (process.platform !== 'win32') { @@ -150,13 +154,15 @@ const signalManagedProcess = (childProcess: ChildProcess, signal: NodeJS.Signals } childProcess.kill(signal) - } catch { + } + catch { // Best-effort shutdown. Cleanup continues even when the process is already gone. } } export const stopManagedProcess = async (managedProcess?: ManagedProcess) => { - if (!managedProcess) return + if (!managedProcess) + return const { childProcess, logStream } = managedProcess diff --git a/e2e/support/tools.ts b/e2e/support/tools.ts index 11ea82d4dcd8e0..c4bee2278bb3ad 100644 --- a/e2e/support/tools.ts +++ b/e2e/support/tools.ts @@ -16,7 +16,8 @@ export async function deleteBuiltinToolCredential( response, `Delete built-in tool credential ${credentialId} for ${provider}`, ) - } finally { + } + finally { await ctx.dispose() } } diff --git a/e2e/support/web-server.ts b/e2e/support/web-server.ts index ad5d5d916a1daa..819f7effe38b9f 100644 --- a/e2e/support/web-server.ts +++ b/e2e/support/web-server.ts @@ -34,7 +34,8 @@ export const startWebServer = async ({ }: WebServerStartOptions) => { const { host, port } = getUrlHostAndPort(baseURL) - if (reuseExistingServer && (await isPortReachable(host, port))) return + if (reuseExistingServer && (await isPortReachable(host, port))) + return activeProcess = await startLoggedProcess({ command, @@ -49,7 +50,8 @@ export const startWebServer = async ({ startupError = error }) activeProcess.childProcess.once('exit', (code, signal) => { - if (startupError) return + if (startupError) + return startupError = new Error( `Web server exited before readiness (code: ${code ?? 'unknown'}, signal: ${signal ?? 'none'}).`, @@ -67,7 +69,8 @@ export const startWebServer = async ({ try { await waitForUrl(baseURL, 1_000, 250, 1_000) return - } catch { + } + catch { // Continue polling until timeout or child exit. } } From f8646a87410d7f59c24e8dd33ac9f30300b9da93 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 14:19:50 +0800 Subject: [PATCH 034/185] test(e2e): verify agent file upload persistence --- e2e/features/agent-v2/files.feature | 11 +++ .../agent-v2/configure.steps.ts | 63 ++++++++++++- e2e/features/support/hooks.ts | 14 +-- e2e/features/support/world.ts | 18 +++- e2e/support/agent.ts | 89 ++++++++++++++----- 5 files changed, 164 insertions(+), 31 deletions(-) create mode 100644 e2e/features/agent-v2/files.feature diff --git a/e2e/features/agent-v2/files.feature b/e2e/features/agent-v2/files.feature new file mode 100644 index 00000000000000..db24f03f48baa3 --- /dev/null +++ b/e2e/features/agent-v2/files.feature @@ -0,0 +1,11 @@ +@agent-v2 @authenticated @files @core +Feature: Agent v2 files + Scenario: Uploading a small file keeps it in the Agent configuration + Given I am signed in as the default E2E admin + And a basic configured Agent v2 test agent has been created via API + When I open the Agent v2 configure page + And I upload the small Agent v2 file from the Files section + Then I should see the small Agent v2 file in the Files section + And the small Agent v2 file should be saved in the Agent v2 draft + When I refresh the current page + Then I should see the small Agent v2 file in the Files section diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index ab59200dd4fa30..c569c87ace00ba 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -5,6 +5,7 @@ import { createAgentSoulConfigWithModel, createConfiguredTestAgent, createTestAgent, + getAgentComposerDraft, getAgentConfigurePath, getTestAgent, normalAgentPrompt, @@ -15,10 +16,12 @@ import { updatedAgentSoulConfig, } from '../../../support/agent' import { waitForAgentConfigureAutosaved } from '../../../support/agent-configure' +import { agentBuilderTestMaterials, getAgentBuilderTestMaterialPath } from '../../../support/test-materials' const getCurrentAgentId = (world: DifyWorld) => { const agentId = world.createdAgentIds.at(-1) - if (!agentId) throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') + if (!agentId) + throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') return agentId } @@ -77,7 +80,8 @@ When('I open the Agent v2 configure page from the Agent Roster', async function const page = this.getPage() const agentId = getCurrentAgentId(this) const agentName = this.lastCreatedAgentName - if (!agentName) throw new Error('No Agent v2 name found. Create an Agent v2 test agent first.') + if (!agentName) + throw new Error('No Agent v2 name found. Create an Agent v2 test agent first.') await page.goto('/roster') await page.getByRole('link', { name: agentName }).click() @@ -97,6 +101,38 @@ When('I publish the Agent v2 draft', async function (this: DifyWorld) { await publishButton.click() }) +When('I upload the small Agent v2 file from the Files section', async function (this: DifyWorld) { + const page = this.getPage() + const agentId = getCurrentAgentId(this) + const fileName = agentBuilderTestMaterials.smallFile + const filePath = getAgentBuilderTestMaterialPath('smallFile') + + await page.getByRole('button', { name: 'Add file' }).click() + const dialog = page.getByRole('dialog', { name: 'Upload file' }) + await expect(dialog).toBeVisible() + + const fileChooserPromise = page.waitForEvent('filechooser') + await dialog.getByRole('button', { name: 'browse' }).click() + await (await fileChooserPromise).setFiles(filePath) + await expect(dialog.getByText(fileName)).toBeVisible() + + const commitResponsePromise = page.waitForResponse(response => ( + response.request().method() === 'POST' + && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/config/files`) + )) + await dialog.getByRole('button', { name: 'Upload' }).click() + const commitResponse = await commitResponsePromise + expect(commitResponse.status()).toBe(201) + const committed = await commitResponse.json() as { file?: { name?: string } } + await expect(dialog).not.toBeVisible({ timeout: 30_000 }) + + const committedName = committed.file?.name + if (!committedName) + throw new Error('Agent config file upload response did not include a file name.') + + this.createdAgentConfigFiles.push({ agentId, name: committedName }) +}) + Then('I should be on the Agent v2 configure page', async function (this: DifyWorld) { const agentId = getCurrentAgentId(this) @@ -143,6 +179,29 @@ Then( }, ) +Then('I should see the small Agent v2 file in the Files section', async function (this: DifyWorld) { + const fileName = agentBuilderTestMaterials.smallFile + await expect( + this.getPage().getByRole('button', { exact: true, name: fileName }), + ).toBeVisible({ timeout: 30_000 }) +}) + +Then( + 'the small Agent v2 file should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + const fileName = agentBuilderTestMaterials.smallFile + + await expect + .poll(async () => ( + await getAgentComposerDraft(agentId) + ).agent_soul?.config_files?.map(file => file.name) ?? [], { + timeout: 30_000, + }) + .toContain(fileName) + }, +) + Then('I should see the Agent v2 Build draft pending changes', async function (this: DifyWorld) { const page = this.getPage() diff --git a/e2e/features/support/hooks.ts b/e2e/features/support/hooks.ts index 117b3b7e8ca29a..65bb4b8c2a86df 100644 --- a/e2e/features/support/hooks.ts +++ b/e2e/features/support/hooks.ts @@ -7,7 +7,7 @@ import { fileURLToPath } from 'node:url' import { After, AfterAll, Before, BeforeAll, setDefaultTimeout, Status } from '@cucumber/cucumber' import { chromium } from '@playwright/test' import { AUTH_BOOTSTRAP_TIMEOUT_MS, ensureAuthenticatedState } from '../../fixtures/auth' -import { deleteAgentDriveFile, deleteTestAgent } from '../../support/agent' +import { deleteAgentConfigFile, deleteAgentDriveFile, deleteTestAgent } from '../../support/agent' import { deleteTestApp } from '../../support/api' import { deleteTestDataset } from '../../support/datasets' import { deleteBuiltinToolCredential } from '../../support/tools' @@ -50,16 +50,18 @@ BeforeAll({ timeout: AUTH_BOOTSTRAP_TIMEOUT_MS }, async () => { }) Before(async function (this: DifyWorld, { pickle }) { - if (!browser) throw new Error('Shared Playwright browser is not available.') + if (!browser) + throw new Error('Shared Playwright browser is not available.') - const isUnauthenticatedScenario = pickle.tags.some((tag) => tag.name === '@unauthenticated') + const isUnauthenticatedScenario = pickle.tags.some(tag => tag.name === '@unauthenticated') - if (isUnauthenticatedScenario) await this.startUnauthenticatedSession(browser) + if (isUnauthenticatedScenario) + await this.startUnauthenticatedSession(browser) else await this.startAuthenticatedSession(browser) this.scenarioStartedAt = Date.now() - const tags = pickle.tags.map((tag) => tag.name).join(' ') + const tags = pickle.tags.map(tag => tag.name).join(' ') console.warn(`[e2e] start ${pickle.name}${tags ? ` ${tags}` : ''}`) }) @@ -91,6 +93,8 @@ After(async function (this: DifyWorld, { pickle, result }) { `[e2e] end ${pickle.name} status=${status}${elapsedMs ? ` durationMs=${elapsedMs}` : ''}`, ) + for (const file of this.createdAgentConfigFiles.toReversed()) + await deleteAgentConfigFile(file.agentId, file.name).catch(() => {}) for (const file of this.createdAgentDriveFiles.toReversed()) await deleteAgentDriveFile(file.agentId, file.key).catch(() => {}) for (const id of this.createdAgentIds) await deleteTestAgent(id).catch(() => {}) diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index a5c089048d9d15..a84be25868bc00 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -10,6 +10,10 @@ export type CreatedAgentDriveFile = { agentId: string key: string } +export type CreatedAgentConfigFile = { + agentId: string + name: string +} export type CreatedBuiltinToolCredential = { credentialId: string provider: string @@ -41,6 +45,7 @@ export class DifyWorld extends World { createdAppIds: string[] = [] createdAgentIds: string[] = [] createdDatasetIds: string[] = [] + createdAgentConfigFiles: CreatedAgentConfigFile[] = [] createdAgentDriveFiles: CreatedAgentDriveFile[] = [] createdBuiltinToolCredentials: CreatedBuiltinToolCredential[] = [] agentBuilderBrokenChatModel: AgentBuilderStableChatModel | undefined @@ -67,6 +72,7 @@ export class DifyWorld extends World { this.createdAppIds = [] this.createdAgentIds = [] this.createdDatasetIds = [] + this.createdAgentConfigFiles = [] this.createdAgentDriveFiles = [] this.createdBuiltinToolCredentials = [] this.agentBuilderBrokenChatModel = undefined @@ -89,7 +95,8 @@ export class DifyWorld extends World { this.page.setDefaultTimeout(30_000) this.page.on('console', (message: ConsoleMessage) => { - if (message.type() === 'error') this.consoleErrors.push(message.text()) + if (message.type() === 'error') + this.consoleErrors.push(message.text()) }) this.page.on('pageerror', (error) => { this.pageErrors.push(error.message) @@ -108,7 +115,8 @@ export class DifyWorld extends World { } getPage() { - if (!this.page) throw new Error('Playwright page has not been initialized for this scenario.') + if (!this.page) + throw new Error('Playwright page has not been initialized for this scenario.') return this.page } @@ -128,12 +136,14 @@ export class DifyWorld extends World { for (const cleanup of this.scenarioCleanups.toReversed()) { try { await cleanup() - } catch (error) { + } + catch (error) { errors.push(error instanceof Error ? error.message : String(error)) } } - if (errors.length > 0) this.attach(`Cleanup errors:\n${errors.join('\n')}`, 'text/plain') + if (errors.length > 0) + this.attach(`Cleanup errors:\n${errors.join('\n')}`, 'text/plain') } async closeSession() { diff --git a/e2e/support/agent.ts b/e2e/support/agent.ts index f6ca42bc8f62ac..765e2ee3eb649e 100644 --- a/e2e/support/agent.ts +++ b/e2e/support/agent.ts @@ -17,7 +17,18 @@ export type AgentSeed = { } | null } -export type AgentSoulConfig = Record +export type AgentComposerConfigFile = { + file_id?: string | null + file_kind?: string | null + hash?: string | null + mime_type?: string | null + name: string + size?: number | null +} + +export type AgentSoulConfig = Record & { + config_files?: AgentComposerConfigFile[] +} export type AgentModelSelection = { name: string provider: string @@ -57,11 +68,11 @@ export const defaultAgentSoulConfig: AgentSoulConfig = { }, } -export const normalAgentPrompt = - 'You are a Dify Agent E2E test assistant. Reply briefly to every user message, and always include AGENT_E2E_PASS in your response.' +export const normalAgentPrompt + = 'You are a Dify Agent E2E test assistant. Reply briefly to every user message, and always include AGENT_E2E_PASS in your response.' -export const updatedAgentPrompt = - 'You are a Dify Agent E2E test assistant. Every response must start with E2E_AGENT_UPDATED.' +export const updatedAgentPrompt + = 'You are a Dify Agent E2E test assistant. Every response must start with E2E_AGENT_UPDATED.' export const normalAgentSoulConfig: AgentSoulConfig = { prompt: { @@ -81,7 +92,8 @@ export const getAgentAccessPath = (agentId: string) => `/roster/agent/${agentId} const getAgentModelPluginId = (provider: string) => { const [organization, pluginName] = provider.split('/').filter(Boolean) - if (organization && pluginName) return `${organization}/${pluginName}` + if (organization && pluginName) + return `${organization}/${pluginName}` return provider ? `langgenius/${provider}` : '' } @@ -133,7 +145,8 @@ export async function createTestAgent({ }) await expectApiResponseOK(response, 'Create Agent v2 test agent') return (await response.json()) as AgentSeed - } finally { + } + finally { await ctx.dispose() } } @@ -156,7 +169,8 @@ export async function getTestAgent(agentId: string): Promise { const response = await ctx.get(`/console/api/agent/${agentId}`) await expectApiResponseOK(response, `Get Agent v2 test agent ${agentId}`) return (await response.json()) as AgentSeed - } finally { + } + finally { await ctx.dispose() } } @@ -166,7 +180,8 @@ export async function deleteTestAgent(agentId: string): Promise { try { const response = await ctx.delete(`/console/api/agent/${agentId}`) await expectApiResponseOK(response, `Delete Agent v2 test agent ${agentId}`) - } finally { + } + finally { await ctx.dispose() } } @@ -186,7 +201,20 @@ export async function saveAgentComposerDraft( }) await expectApiResponseOK(response, `Save Agent v2 composer draft for ${agentId}`) return (await response.json()) as AgentComposerResponse - } finally { + } + finally { + await ctx.dispose() + } +} + +export async function getAgentComposerDraft(agentId: string): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agentId}/composer`) + await expectApiResponseOK(response, `Get Agent v2 composer draft for ${agentId}`) + return (await response.json()) as AgentComposerResponse + } + finally { await ctx.dispose() } } @@ -199,7 +227,8 @@ export async function checkoutAgentBuildDraft(agentId: string): Promise { try { const response = await ctx.delete(`/console/api/agent/${agentId}/build-draft`) await expectApiResponseOK(response, `Discard Agent v2 build draft for ${agentId}`) - } finally { + } + finally { await ctx.dispose() } } @@ -241,7 +272,8 @@ export async function publishAgent(agentId: string, versionNote = 'E2E publish') data: { version_note: versionNote }, }) await expectApiResponseOK(response, `Publish Agent v2 test agent ${agentId}`) - } finally { + } + finally { await ctx.dispose() } } @@ -249,7 +281,8 @@ export async function publishAgent(agentId: string, versionNote = 'E2E publish') export async function enableAgentSiteAndGetURL(agentId: string): Promise { const agent = await getTestAgent(agentId) const appId = agent.app_id ?? agent.backing_app_id - if (!appId) throw new Error(`Agent v2 ${agentId} does not expose a backing app ID.`) + if (!appId) + throw new Error(`Agent v2 ${agentId} does not expose a backing app ID.`) const appDetail = await setAppSiteEnabled(appId, true) const token = agent.site?.access_token ?? agent.site?.code ?? appDetail.site.access_token @@ -264,7 +297,8 @@ export async function getAgentApiAccess(agentId: string): Promise { const response = await ctx.post(`/console/api/agent/${agentId}/api-keys`) await expectApiResponseOK(response, `Create Agent v2 API key for ${agentId}`) return (await response.json()) as AgentApiKey - } finally { + } + finally { await ctx.dispose() } } @@ -304,7 +340,19 @@ export async function deleteAgentApiKey(agentId: string, apiKeyId: string): Prom try { const response = await ctx.delete(`/console/api/agent/${agentId}/api-keys/${apiKeyId}`) await expectApiResponseOK(response, `Delete Agent v2 API key ${apiKeyId} for ${agentId}`) - } finally { + } + finally { + await ctx.dispose() + } +} + +export async function deleteAgentConfigFile(agentId: string, name: string): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.delete(`/console/api/agent/${agentId}/config/files/${encodeURIComponent(name)}`) + await expectApiResponseOK(response, `Delete Agent v2 config file ${name} for ${agentId}`) + } + finally { await ctx.dispose() } } @@ -315,7 +363,8 @@ export async function deleteAgentDriveFile(agentId: string, key: string): Promis const searchParams = new URLSearchParams({ key }) const response = await ctx.delete(`/console/api/agent/${agentId}/files?${searchParams}`) await expectApiResponseOK(response, `Delete Agent v2 drive file ${key} for ${agentId}`) - } finally { + } + finally { await ctx.dispose() } } From 0ddcf533e70a441c8d121b7116e264f0e2d2e5ad Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 14:54:03 +0800 Subject: [PATCH 035/185] fix(web): keep agent access cards within viewport --- .../agent-detail/access/components/access-surface-card.tsx | 4 ++-- .../access/components/workflow-references-table.tsx | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/web/features/agent-v2/agent-detail/access/components/access-surface-card.tsx b/web/features/agent-v2/agent-detail/access/components/access-surface-card.tsx index 146a0812077393..d546c318313f3b 100644 --- a/web/features/agent-v2/agent-detail/access/components/access-surface-card.tsx +++ b/web/features/agent-v2/agent-detail/access/components/access-surface-card.tsx @@ -63,8 +63,8 @@ export function AccessSurfaceCard({ return (
-
-
+
+
diff --git a/web/features/agent-v2/agent-detail/access/components/workflow-references-table.tsx b/web/features/agent-v2/agent-detail/access/components/workflow-references-table.tsx index 7dffef6ebeeea4..41029edc93a20f 100644 --- a/web/features/agent-v2/agent-detail/access/components/workflow-references-table.tsx +++ b/web/features/agent-v2/agent-detail/access/components/workflow-references-table.tsx @@ -33,7 +33,7 @@ export function WorkflowReferencesTable({ const workflowReferences = workflowReferencesQuery.data?.data ?? [] return ( -
+
From 36c9baa1fc5a4b9dc0f14d831442f663d2505289 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 14:54:24 +0800 Subject: [PATCH 036/185] test(e2e): cover agent env editor persistence --- .../agent-v2/advanced-settings.feature | 10 +++ .../agent-v2/access-point.steps.ts | 13 ++++ .../agent-v2/configure.steps.ts | 65 +++++++++++++++++++ e2e/support/agent-configure.ts | 4 +- e2e/support/agent.ts | 11 ++++ 5 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 e2e/features/agent-v2/advanced-settings.feature diff --git a/e2e/features/agent-v2/advanced-settings.feature b/e2e/features/agent-v2/advanced-settings.feature new file mode 100644 index 00000000000000..b4e85b1b4e3363 --- /dev/null +++ b/e2e/features/agent-v2/advanced-settings.feature @@ -0,0 +1,10 @@ +@agent-v2 @authenticated @advanced-settings @core +Feature: Agent v2 advanced settings + Scenario: Plain environment variables are saved and restored + Given I am signed in as the default E2E admin + And a basic configured Agent v2 test agent has been created via API + When I open the Agent v2 configure page + And I add the plain Agent v2 environment variable from Advanced Settings + Then the plain Agent v2 environment variable should be saved in the Agent v2 draft + When I refresh the current page + Then I should see the plain Agent v2 environment variable in Advanced Settings diff --git a/e2e/features/step-definitions/agent-v2/access-point.steps.ts b/e2e/features/step-definitions/agent-v2/access-point.steps.ts index 405f8f06fca8ba..5f1df3c3de0323 100644 --- a/e2e/features/step-definitions/agent-v2/access-point.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point.steps.ts @@ -40,7 +40,20 @@ Then('I should see the Agent v2 Access Point overview', async function (this: Di await expect(accessRegion).toBeVisible({ timeout: 30_000 }) await expect(accessRegion.getByRole('heading', { name: 'Access Point' })).toBeVisible() await expect(accessRegion.getByRole('heading', { name: 'Web app' })).toBeVisible() + await expect(accessRegion.getByText('Access URL')).toBeVisible() + await expect(accessRegion.getByLabel('Copy access URL')).toBeVisible() + await expect(accessRegion.getByLabel('Toggle Web app access')).toBeVisible() + await expect(accessRegion.getByRole('button', { name: 'Launch' })).toBeVisible() + await expect(accessRegion.getByRole('button', { name: 'Embedded' })).toBeVisible() + await expect(accessRegion.getByRole('button', { name: 'Customize' })).toBeVisible() + await expect(accessRegion.getByRole('button', { name: 'Settings' })).toBeVisible() await expect(accessRegion.getByRole('heading', { name: 'Backend service API' })).toBeVisible() + await expect(accessRegion.getByText('Service API Endpoint')).toBeVisible() + await expect(accessRegion.getByLabel('Copy service API endpoint')).toBeVisible() + await expect(accessRegion.getByLabel('Toggle Backend service API access')).toBeVisible() + await expect(accessRegion.getByRole('button', { name: /^API Key\b/ })).toBeVisible() + await expect(accessRegion.getByRole('link', { name: 'API Reference' })).toBeVisible() + await expect(accessRegion.getByText(/^(?:In|Out of) service$/i)).toHaveCount(2) await expect(accessRegion.getByRole('heading', { name: 'Workflow access' })).toBeVisible() await expect(accessRegion.getByRole('columnheader', { name: 'Name' })).toBeVisible() await expect(accessRegion.getByRole('columnheader', { name: 'Version' })).toBeVisible() diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index c569c87ace00ba..733bae098b8943 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -15,6 +15,7 @@ import { updatedAgentPrompt, updatedAgentSoulConfig, } from '../../../support/agent' +import { agentBuilderFixedInputs } from '../../../support/agent-builder-resources' import { waitForAgentConfigureAutosaved } from '../../../support/agent-configure' import { agentBuilderTestMaterials, getAgentBuilderTestMaterialPath } from '../../../support/test-materials' @@ -133,6 +134,25 @@ When('I upload the small Agent v2 file from the Files section', async function ( this.createdAgentConfigFiles.push({ agentId, name: committedName }) }) +When( + 'I add the plain Agent v2 environment variable from Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() + + await advancedSettings + .getByRole('textbox', { name: 'Key' }) + .fill(agentBuilderFixedInputs.envPlainKey) + await advancedSettings + .getByRole('textbox', { name: 'Value' }) + .fill(agentBuilderFixedInputs.envPlainValue) + await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() + }, +) + Then('I should be on the Agent v2 configure page', async function (this: DifyWorld) { const agentId = getCurrentAgentId(this) @@ -202,6 +222,51 @@ Then( }, ) +Then( + 'the plain Agent v2 environment variable should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect + .poll(async () => { + const env = (await getAgentComposerDraft(agentId)).agent_soul?.env + const variable = env?.variables?.find((item) => { + const key = item.key ?? item.name ?? item.variable + + return key === agentBuilderFixedInputs.envPlainKey + }) + + return { + secretCount: env?.secret_refs?.length ?? 0, + value: variable?.value, + } + }, { + timeout: 30_000, + }) + .toEqual({ + secretCount: 0, + value: agentBuilderFixedInputs.envPlainValue, + }) + }, +) + +Then( + 'I should see the plain Agent v2 environment variable in Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() + await expect(advancedSettings.getByRole('textbox', { name: 'Key' })) + .toHaveValue(agentBuilderFixedInputs.envPlainKey) + await expect(advancedSettings.getByRole('textbox', { name: 'Value' })) + .toHaveValue(agentBuilderFixedInputs.envPlainValue) + await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() + await expect(page.getByRole('button', { name: /^Preview$/i })).toBeVisible() + }, +) + Then('I should see the Agent v2 Build draft pending changes', async function (this: DifyWorld) { const page = this.getPage() diff --git a/e2e/support/agent-configure.ts b/e2e/support/agent-configure.ts index 8c883378a2c031..f04f429081820a 100644 --- a/e2e/support/agent-configure.ts +++ b/e2e/support/agent-configure.ts @@ -2,5 +2,7 @@ import type { Page } from '@playwright/test' import { expect } from '@playwright/test' export async function waitForAgentConfigureAutosaved(page: Page) { - await expect(page.getByText(/^Saved(?:\s|$)/).first()).toBeVisible({ timeout: 30_000 }) + await expect( + page.getByText(/^Saved(?:\s|$)/).filter({ visible: true }).first(), + ).toBeVisible({ timeout: 30_000 }) } diff --git a/e2e/support/agent.ts b/e2e/support/agent.ts index 765e2ee3eb649e..918289e277663b 100644 --- a/e2e/support/agent.ts +++ b/e2e/support/agent.ts @@ -25,9 +25,20 @@ export type AgentComposerConfigFile = { name: string size?: number | null } +export type AgentComposerEnvVariable = { + id?: string | null + key?: string | null + name?: string | null + value?: unknown + variable?: string | null +} export type AgentSoulConfig = Record & { config_files?: AgentComposerConfigFile[] + env?: { + secret_refs?: unknown[] + variables?: AgentComposerEnvVariable[] + } } export type AgentModelSelection = { name: string From a52c5c83af236689fa6240762cdf0930e241f8e9 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 15:16:04 +0800 Subject: [PATCH 037/185] fix(agent-v2): report invalid env imports --- .../agent-v2/advanced-settings.feature | 12 +++ .../agent-v2/configure-validation.feature | 2 +- .../agent-v2/configure.steps.ts | 75 ++++++++++++++++++- e2e/support/agent-builder-resources.ts | 2 + .../advanced/__tests__/env.spec.tsx | 39 +++++++++- .../orchestrate/advanced/env-utils.ts | 34 ++++++--- .../components/orchestrate/advanced/env.tsx | 14 +++- web/i18n/en-US/agent-v-2.json | 2 + web/i18n/zh-Hans/agent-v-2.json | 2 + 9 files changed, 167 insertions(+), 15 deletions(-) diff --git a/e2e/features/agent-v2/advanced-settings.feature b/e2e/features/agent-v2/advanced-settings.feature index b4e85b1b4e3363..0613dfd1b66133 100644 --- a/e2e/features/agent-v2/advanced-settings.feature +++ b/e2e/features/agent-v2/advanced-settings.feature @@ -8,3 +8,15 @@ Feature: Agent v2 advanced settings Then the plain Agent v2 environment variable should be saved in the Agent v2 draft When I refresh the current page Then I should see the plain Agent v2 environment variable in Advanced Settings + + Scenario: Invalid environment imports report skipped lines and keep existing variables + Given I am signed in as the default E2E admin + And a basic configured Agent v2 test agent has been created via API + When I open the Agent v2 configure page + And I add the plain Agent v2 environment variable from Advanced Settings + Then the plain Agent v2 environment variable should be saved in the Agent v2 draft + When I import the invalid Agent v2 environment file from Advanced Settings + Then the invalid Agent v2 environment import should report skipped lines + And the Agent v2 environment variables from the invalid import should be saved in the Agent v2 draft + When I refresh the current page + Then I should see the Agent v2 environment variables from the invalid import in Advanced Settings diff --git a/e2e/features/agent-v2/configure-validation.feature b/e2e/features/agent-v2/configure-validation.feature index e053b38fe41e8f..e15756eba9d6d3 100644 --- a/e2e/features/agent-v2/configure-validation.feature +++ b/e2e/features/agent-v2/configure-validation.feature @@ -1,4 +1,4 @@ -@agent-v2 @authenticated @core +@agent-v2 @authenticated @preview Feature: Agent v2 configure validation Scenario: Preview is unavailable until a required model is configured Given I am signed in as the default E2E admin diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 733bae098b8943..91ca99139ff02e 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -153,6 +153,18 @@ When( }, ) +When( + 'I import the invalid Agent v2 environment file from Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + const fileChooserPromise = page.waitForEvent('filechooser') + await advancedSettings.getByRole('button', { name: 'Import .env' }).click() + await (await fileChooserPromise).setFiles(getAgentBuilderTestMaterialPath('invalidEnv')) + }, +) + Then('I should be on the Agent v2 configure page', async function (this: DifyWorld) { const agentId = getCurrentAgentId(this) @@ -250,6 +262,44 @@ Then( }, ) +Then( + 'the invalid Agent v2 environment import should report skipped lines', + async function (this: DifyWorld) { + await expect(this.getPage().getByText('2 invalid .env lines were skipped.')).toBeVisible() + }, +) + +Then( + 'the Agent v2 environment variables from the invalid import should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect + .poll(async () => { + const variables = (await getAgentComposerDraft(agentId)).agent_soul?.env?.variables ?? [] + + return { + importedValue: variables.find((item) => { + const key = item.key ?? item.name ?? item.variable + + return key === agentBuilderFixedInputs.envAfterInvalidImportKey + })?.value, + plainValue: variables.find((item) => { + const key = item.key ?? item.name ?? item.variable + + return key === agentBuilderFixedInputs.envPlainKey + })?.value, + } + }, { + timeout: 30_000, + }) + .toEqual({ + importedValue: agentBuilderFixedInputs.envAfterInvalidImportValue, + plainValue: agentBuilderFixedInputs.envPlainValue, + }) + }, +) + Then( 'I should see the plain Agent v2 environment variable in Advanced Settings', async function (this: DifyWorld) { @@ -263,7 +313,30 @@ Then( await expect(advancedSettings.getByRole('textbox', { name: 'Value' })) .toHaveValue(agentBuilderFixedInputs.envPlainValue) await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() - await expect(page.getByRole('button', { name: /^Preview$/i })).toBeVisible() + await expect(page.getByRole('button', { name: /^Build$/i })).toBeVisible() + }, +) + +Then( + 'I should see the Agent v2 environment variables from the invalid import in Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() + await expect.poll( + async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => + inputs.map(input => (input as HTMLInputElement).value), + ), + { timeout: 30_000 }, + ).toEqual(expect.arrayContaining([ + agentBuilderFixedInputs.envPlainKey, + agentBuilderFixedInputs.envPlainValue, + agentBuilderFixedInputs.envAfterInvalidImportKey, + agentBuilderFixedInputs.envAfterInvalidImportValue, + ])) + await expect(page.getByRole('button', { name: /^Build$/i })).toBeVisible() }, ) diff --git a/e2e/support/agent-builder-resources.ts b/e2e/support/agent-builder-resources.ts index fa6b0376c233a6..a1f795aa5cf382 100644 --- a/e2e/support/agent-builder-resources.ts +++ b/e2e/support/agent-builder-resources.ts @@ -26,6 +26,8 @@ export const agentBuilderFixedInputs = { customKnowledgeQuery: 'Dify Agent E2E 测试暗号', envPlainKey: 'E2E_AGENT_FLAG', envPlainValue: 'enabled', + envAfterInvalidImportKey: 'E2E_AGENT_AFTER_INVALID', + envAfterInvalidImportValue: 'still-valid', inputModerationReply: 'E2E_INPUT_BLOCKED_REPLY', outputModerationReply: 'E2E_OUTPUT_BLOCKED_REPLY', previewSuccessQuery: '请回复测试成功', diff --git a/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/__tests__/env.spec.tsx b/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/__tests__/env.spec.tsx index 27d923f7379cfd..e9cdf8034e61a1 100644 --- a/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/__tests__/env.spec.tsx +++ b/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/__tests__/env.spec.tsx @@ -6,7 +6,7 @@ import { defaultAgentSoulConfigFormState } from '@/features/agent-v2/agent-compo import { AgentComposerProvider } from '@/features/agent-v2/agent-composer/provider' import { AgentOrchestrateReadOnlyContext } from '../../read-only-context' import { AgentEnvEditor, EnvVariablesTable } from '../env' -import { getEnvImportPlatform, parseEnvVariables } from '../env-utils' +import { getEnvImportPlatform, parseEnvImport, parseEnvVariables } from '../env-utils' vi.mock('@langgenius/dify-ui/toast', () => ({ toast: { @@ -65,6 +65,22 @@ describe('AgentEnvEditor', () => { { key: 'MULTILINE', value: 'first\nsecond' }, ]) }) + + it('should report invalid dotenv lines without blocking valid entries', () => { + expect(parseEnvImport([ + '# ignored', + 'API_KEY=abc123', + 'INVALID_LINE', + '=missing_key', + 'SECOND_KEY=enabled', + ].join('\n'))).toEqual({ + invalidLineCount: 2, + variables: [ + { key: 'API_KEY', value: 'abc123' }, + { key: 'SECOND_KEY', value: 'enabled' }, + ], + }) + }) }) describe('Platform Detection', () => { @@ -145,6 +161,27 @@ describe('AgentEnvEditor', () => { expect(screen.getByDisplayValue('https://example.com')).toBeInTheDocument() }) + it('should show a visible error when imported dotenv content includes invalid lines', async () => { + const user = userEvent.setup() + const { container } = renderAgentEnvEditor() + const input = container.querySelector('input[type="file"]') as HTMLInputElement + + const file = new File([ + 'API_KEY=abc123\n', + 'INVALID_LINE\n', + '=missing_key\n', + ], '.env', { type: 'text/plain' }) + + await user.upload(input, file) + + await waitFor(() => { + expect(screen.getByDisplayValue('API_KEY')).toBeInTheDocument() + }) + expect(mockToastError).toHaveBeenCalledWith( + 'agentV2.agentDetail.configure.advancedSettings.envEditor.importSkippedInvalidLines:{"count":2}', + ) + }) + it('should hide import, add, edit, and delete controls when readonly', () => { renderReadonlyAgentEnvEditor() diff --git a/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/env-utils.ts b/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/env-utils.ts index 4a6b0242e7229b..bdcbe635a74541 100644 --- a/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/env-utils.ts +++ b/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/env-utils.ts @@ -31,33 +31,47 @@ const parseEnvValue = (rawValue: string) => { return stripInlineComment(value).trim() } -export const parseEnvVariables = (content: string) => { - return content.split(/\r?\n/).flatMap((line) => { +export const parseEnvImport = (content: string) => { + const variables: Array<{ key: string, value: string }> = [] + let invalidLineCount = 0 + + for (const line of content.split(/\r?\n/)) { const trimmedLine = line.trim() if (!trimmedLine || trimmedLine.startsWith('#')) - return [] + continue const envLine = trimmedLine.startsWith('export ') ? trimmedLine.slice('export '.length).trimStart() : trimmedLine const separatorIndex = envLine.indexOf('=') - if (separatorIndex <= 0) - return [] + if (separatorIndex <= 0) { + invalidLineCount += 1 + continue + } const key = envLine.slice(0, separatorIndex).trim() - if (!/^[\w.-]+$/.test(key)) - return [] + if (!/^[\w.-]+$/.test(key)) { + invalidLineCount += 1 + continue + } - return [{ + variables.push({ key, value: parseEnvValue(envLine.slice(separatorIndex + 1)), - }] - }) + }) + } + + return { + invalidLineCount, + variables, + } } +export const parseEnvVariables = (content: string) => parseEnvImport(content).variables + export type EnvImportPlatform = 'mac' | 'windows' | 'other' export const getEnvImportPlatform = ({ diff --git a/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/env.tsx b/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/env.tsx index e989cd9e3650cc..555c8e5185d168 100644 --- a/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/env.tsx +++ b/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/env.tsx @@ -15,7 +15,7 @@ import { checkKeys } from '@/utils/var' import { ConfigureSection } from '../common/section' import { AgentConfigureTipContent } from '../common/tip-content' import { useAgentOrchestrateReadOnly } from '../read-only-context' -import { getEnvImportPlatform, parseEnvVariables } from './env-utils' +import { getEnvImportPlatform, parseEnvImport } from './env-utils' const scopeLabelKeys: Record> = { plain: 'agentDetail.configure.advancedSettings.envEditor.scopePlain', @@ -455,7 +455,17 @@ export function AgentEnvEditor() { setFocusedVariable({ id: variable.id, field: focusField }) } const importEnvVariables = async (file: File) => { - const importedVariables = parseEnvVariables(await file.text()).map(createEnvVariableFromEntry) + const { + invalidLineCount, + variables, + } = parseEnvImport(await file.text()) + const importedVariables = variables.map(createEnvVariableFromEntry) + + if (invalidLineCount > 0) { + toast.error(t('agentDetail.configure.advancedSettings.envEditor.importSkippedInvalidLines', { + count: invalidLineCount, + })) + } if (importedVariables.length === 0) return diff --git a/web/i18n/en-US/agent-v-2.json b/web/i18n/en-US/agent-v-2.json index f884399fc2f6e7..1f51d53acf02f9 100644 --- a/web/i18n/en-US/agent-v-2.json +++ b/web/i18n/en-US/agent-v-2.json @@ -45,6 +45,8 @@ "agentDetail.configure.advancedSettings.envEditor.importEnvTip.mac": "To show hidden .env files, press Command + Shift + . in the file picker.", "agentDetail.configure.advancedSettings.envEditor.importEnvTip.other": "If your .env file is hidden, enable hidden files in your system file picker.", "agentDetail.configure.advancedSettings.envEditor.importEnvTip.windows": "To show hidden .env files, enable Hidden items in File Explorer.", + "agentDetail.configure.advancedSettings.envEditor.importSkippedInvalidLines": "{{count}} invalid .env line was skipped.", + "agentDetail.configure.advancedSettings.envEditor.importSkippedInvalidLines_other": "{{count}} invalid .env lines were skipped.", "agentDetail.configure.advancedSettings.envEditor.keyColumn": "Key", "agentDetail.configure.advancedSettings.envEditor.keyPlaceholder": "Key", "agentDetail.configure.advancedSettings.envEditor.label": "Env Editor", diff --git a/web/i18n/zh-Hans/agent-v-2.json b/web/i18n/zh-Hans/agent-v-2.json index 9a3862943d9788..ed1071f2b3b82c 100644 --- a/web/i18n/zh-Hans/agent-v-2.json +++ b/web/i18n/zh-Hans/agent-v-2.json @@ -45,6 +45,8 @@ "agentDetail.configure.advancedSettings.envEditor.importEnvTip.mac": "如需显示隐藏的 .env 文件，请在文件选择器中按 Command + Shift + .。", "agentDetail.configure.advancedSettings.envEditor.importEnvTip.other": "如果 .env 文件被隐藏，请在系统文件选择器中开启隐藏文件显示。", "agentDetail.configure.advancedSettings.envEditor.importEnvTip.windows": "如需显示隐藏的 .env 文件，请在文件资源管理器中开启“隐藏的项目”。", + "agentDetail.configure.advancedSettings.envEditor.importSkippedInvalidLines": "已跳过 {{count}} 行非法 .env 内容。", + "agentDetail.configure.advancedSettings.envEditor.importSkippedInvalidLines_other": "已跳过 {{count}} 行非法 .env 内容。", "agentDetail.configure.advancedSettings.envEditor.keyColumn": "Key", "agentDetail.configure.advancedSettings.envEditor.keyPlaceholder": "Key", "agentDetail.configure.advancedSettings.envEditor.label": "Env Editor", From 16d00bd7f502331939817adb0360c963f742a57a Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 15:22:49 +0800 Subject: [PATCH 038/185] test(e2e): verify build draft isolation --- e2e/features/agent-v2/build-draft.feature | 1 + .../step-definitions/agent-v2/configure.steps.ts | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/e2e/features/agent-v2/build-draft.feature b/e2e/features/agent-v2/build-draft.feature index ab4b2d72bd97d5..b69e97fc46de30 100644 --- a/e2e/features/agent-v2/build-draft.feature +++ b/e2e/features/agent-v2/build-draft.feature @@ -8,6 +8,7 @@ Feature: Agent v2 build draft When I open the Agent v2 configure page Then I should see the Agent v2 Build draft pending changes And I should see the updated E2E prompt in the Agent v2 prompt editor + And the normal Agent v2 draft should still use the normal E2E prompt When I discard the Agent v2 Build draft Then I should see the normal E2E prompt in the Agent v2 prompt editor And the Agent v2 Build draft should no longer be active diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 91ca99139ff02e..700db9232e25db 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -348,6 +348,16 @@ Then('I should see the Agent v2 Build draft pending changes', async function (th await expect(page.getByRole('button', { name: 'Discard' })).toBeVisible() }) +Then( + 'the normal Agent v2 draft should still use the normal E2E prompt', + async function (this: DifyWorld) { + await expect.poll( + async () => (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul?.prompt, + { timeout: 30_000 }, + ).toEqual({ system_prompt: normalAgentPrompt }) + }, +) + Then('the Agent v2 Build draft should no longer be active', async function (this: DifyWorld) { const page = this.getPage() From 362cc31b1a8c31167cde161ddb829b12280413c5 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 15:24:51 +0800 Subject: [PATCH 039/185] fix(web): remove unused agent env parser export --- .../advanced/__tests__/env.spec.tsx | 18 +----------------- .../orchestrate/advanced/env-utils.ts | 2 -- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/__tests__/env.spec.tsx b/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/__tests__/env.spec.tsx index e9cdf8034e61a1..7e823298fe8bba 100644 --- a/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/__tests__/env.spec.tsx +++ b/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/__tests__/env.spec.tsx @@ -6,7 +6,7 @@ import { defaultAgentSoulConfigFormState } from '@/features/agent-v2/agent-compo import { AgentComposerProvider } from '@/features/agent-v2/agent-composer/provider' import { AgentOrchestrateReadOnlyContext } from '../../read-only-context' import { AgentEnvEditor, EnvVariablesTable } from '../env' -import { getEnvImportPlatform, parseEnvImport, parseEnvVariables } from '../env-utils' +import { getEnvImportPlatform, parseEnvImport } from '../env-utils' vi.mock('@langgenius/dify-ui/toast', () => ({ toast: { @@ -50,22 +50,6 @@ describe('AgentEnvEditor', () => { }) describe('Env parsing', () => { - it('should parse dotenv entries from supported line formats', () => { - expect(parseEnvVariables([ - '# ignored', - 'API_KEY=abc123', - 'export BASE_URL="https://example.com"', - 'PASSWORD=secret # inline comment', - 'MULTILINE="first\\nsecond"', - 'INVALID_LINE', - ].join('\n'))).toEqual([ - { key: 'API_KEY', value: 'abc123' }, - { key: 'BASE_URL', value: 'https://example.com' }, - { key: 'PASSWORD', value: 'secret' }, - { key: 'MULTILINE', value: 'first\nsecond' }, - ]) - }) - it('should report invalid dotenv lines without blocking valid entries', () => { expect(parseEnvImport([ '# ignored', diff --git a/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/env-utils.ts b/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/env-utils.ts index bdcbe635a74541..1445aedd8ae668 100644 --- a/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/env-utils.ts +++ b/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/env-utils.ts @@ -70,8 +70,6 @@ export const parseEnvImport = (content: string) => { } } -export const parseEnvVariables = (content: string) => parseEnvImport(content).variables - export type EnvImportPlatform = 'mac' | 'windows' | 'other' export const getEnvImportPlatform = ({ From 01055f9669ae4f8e5446e4557c65895477b20bc0 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 15:31:10 +0800 Subject: [PATCH 040/185] test(e2e): improve api startup diagnostics --- e2e/scripts/common.ts | 17 +++++++++++ e2e/scripts/run-cucumber.ts | 57 ++++++++++++++++++++++++++++++++++--- e2e/scripts/setup.ts | 18 ++++++++++-- e2e/support/process.ts | 2 +- 4 files changed, 87 insertions(+), 7 deletions(-) diff --git a/e2e/scripts/common.ts b/e2e/scripts/common.ts index 20b9ab7fbd830a..5e5edca70ce84e 100644 --- a/e2e/scripts/common.ts +++ b/e2e/scripts/common.ts @@ -109,6 +109,23 @@ export const runCommandOrThrow = async (options: RunCommandOptions) => { return result } +export const getTcpPortListenerDescription = async (port: number) => { + if (process.platform === 'win32') + return '' + + const result = await runCommand({ + command: 'lsof', + args: ['-nP', `-iTCP:${port}`, '-sTCP:LISTEN'], + cwd: rootDir, + stdio: 'pipe', + }) + + if (result.exitCode !== 0) + return '' + + return result.stdout.trim() +} + const forwardSignalsToChild = (childProcess: ChildProcess) => { const handleSignal = (signal: NodeJS.Signals) => { if (childProcess.exitCode === null) diff --git a/e2e/scripts/run-cucumber.ts b/e2e/scripts/run-cucumber.ts index 3c8e895e90f569..887e01580903ba 100644 --- a/e2e/scripts/run-cucumber.ts +++ b/e2e/scripts/run-cucumber.ts @@ -1,4 +1,5 @@ -import { mkdir, rm } from 'node:fs/promises' +import type { ManagedProcess } from '../support/process' +import { mkdir, readFile, rm } from 'node:fs/promises' import path from 'node:path' import { startLoggedProcess, stopManagedProcess, waitForUrl } from '../support/process' import { startWebServer, stopWebServer } from '../support/web-server' @@ -42,6 +43,44 @@ const parseArgs = (argv: string[]): RunOptions => { const hasCustomTags = (forwardArgs: string[]) => forwardArgs.some(arg => arg === '--tags' || arg.startsWith('--tags=')) +const readLogTail = async (logFilePath: string) => { + const content = await readFile(logFilePath, 'utf8').catch(() => '') + + return content + .trim() + .split(/\r?\n/) + .slice(-20) + .join('\n') +} + +const waitForUnexpectedProcessExit = async ( + managedProcess: ManagedProcess, + shouldIgnoreExit: () => boolean, +) => { + const { childProcess, label, logFilePath } = managedProcess + + await new Promise((resolve) => { + if (childProcess.exitCode !== null) { + resolve() + return + } + + childProcess.once('exit', () => resolve()) + }) + + if (shouldIgnoreExit()) + return + + const logTail = await readLogTail(logFilePath) + const logTailMessage = logTail + ? `\n\nLast ${label} log lines:\n${logTail}` + : '' + + throw new Error( + `${label} exited before becoming ready. See ${logFilePath}.${logTailMessage}`, + ) +} + const main = async () => { const { forwardArgs, full, headed } = parseArgs(process.argv.slice(2)) const startMiddlewareForRun = full @@ -107,11 +146,21 @@ const main = async () => { process.once('SIGTERM', onTerminate) try { + let waitingForApi = true try { - await waitForUrl(`${apiURL}/health`, 180_000, 1_000) + await Promise.race([ + waitForUrl(`${apiURL}/health`, 180_000, 1_000), + waitForUnexpectedProcessExit(apiProcess, () => !waitingForApi), + ]) + } + catch (error) { + if (error instanceof Error && error.message.includes('exited before becoming ready')) + throw error + + throw new Error(`API did not become ready at ${apiURL}/health. See ${apiProcess.logFilePath}.`) } - catch { - throw new Error(`API did not become ready at ${apiURL}/health.`) + finally { + waitingForApi = false } await startWebServer({ diff --git a/e2e/scripts/setup.ts b/e2e/scripts/setup.ts index 3f77a3f72a7e54..e780feaf0e02f4 100644 --- a/e2e/scripts/setup.ts +++ b/e2e/scripts/setup.ts @@ -9,6 +9,7 @@ import { e2eWebEnvOverrides, ensureFileExists, ensureLineInFile, + getTcpPortListenerDescription, getWebEnvLocalHash, isMainModule, isTcpPortReachable, @@ -25,6 +26,8 @@ import { const buildIdPath = path.join(webDir, '.next', 'BUILD_ID') const webBuildEnvStampPath = path.join(webDir, '.next', 'e2e-web-env.sha256') +const apiHost = '127.0.0.1' +const apiPort = 5001 const middlewareDataPaths = [ path.join(dockerDir, 'volumes', 'db', 'data'), @@ -174,6 +177,17 @@ export const startWeb = async () => { } export const startApi = async () => { + if (await isTcpPortReachable(apiHost, apiPort)) { + const listenerDescription = await getTcpPortListenerDescription(apiPort) + const listenerMessage = listenerDescription + ? `\n\nPort listener:\n${listenerDescription}` + : '' + + throw new Error( + `Cannot start the E2E API server because ${apiHost}:${apiPort} is already in use.${listenerMessage}`, + ) + } + const env = await getApiEnvironment() await runCommandOrThrow({ @@ -193,9 +207,9 @@ export const startApi = async () => { 'flask', 'run', '--host', - '127.0.0.1', + apiHost, '--port', - '5001', + String(apiPort), ], cwd: apiDir, env, diff --git a/e2e/support/process.ts b/e2e/support/process.ts index b54108e6855390..965f94e3be644b 100644 --- a/e2e/support/process.ts +++ b/e2e/support/process.ts @@ -91,7 +91,7 @@ export const startLoggedProcess = async ({ }: ManagedProcessOptions): Promise => { await mkdir(dirname(logFilePath), { recursive: true }) - const logStream = createWriteStream(logFilePath, { flags: 'a' }) + const logStream = createWriteStream(logFilePath, { flags: 'w' }) const childProcess = spawn(command, args, { cwd, env: { From dae2c1db2e4ec8e3d66f98a21efdee480452675f Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 15:38:09 +0800 Subject: [PATCH 041/185] test(e2e): cover build draft apply path --- e2e/features/agent-v2/build-draft.feature | 17 +++++++ .../agent-v2/configure.steps.ts | 44 +++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/e2e/features/agent-v2/build-draft.feature b/e2e/features/agent-v2/build-draft.feature index b69e97fc46de30..6af3ca50151222 100644 --- a/e2e/features/agent-v2/build-draft.feature +++ b/e2e/features/agent-v2/build-draft.feature @@ -15,3 +15,20 @@ Feature: Agent v2 build draft When I refresh the current page Then I should see the normal E2E prompt in the Agent v2 prompt editor And the Agent v2 Build draft should no longer be active + + Scenario: Applying a pending Build draft updates the normal Agent configuration + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API + And an Agent v2 Build draft uses the updated E2E prompt with the stable E2E model + When I open the Agent v2 configure page + Then I should see the Agent v2 Build draft pending changes + And I should see the updated E2E prompt in the Agent v2 prompt editor + And the normal Agent v2 draft should still use the normal E2E prompt + When I apply the Agent v2 Build draft + Then I should see the updated E2E prompt in the Agent v2 prompt editor + And the normal Agent v2 draft should use the updated E2E prompt + And the Agent v2 Build draft should no longer be active + When I refresh the current page + Then I should see the updated E2E prompt in the Agent v2 prompt editor + And the Agent v2 Build draft should no longer be active diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 700db9232e25db..9958789ae67e57 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -73,6 +73,19 @@ Given('an Agent v2 Build draft uses the updated E2E prompt', async function (thi await saveAgentBuildDraft(getCurrentAgentId(this), updatedAgentSoulConfig) }) +Given( + 'an Agent v2 Build draft uses the updated E2E prompt with the stable E2E model', + async function (this: DifyWorld) { + if (!this.agentBuilderStableChatModel) + throw new Error('Create an Agent v2 Build draft with a stable model after stable model preflight.') + + await saveAgentBuildDraft( + getCurrentAgentId(this), + createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilderStableChatModel), + ) + }, +) + When('I open the Agent v2 configure page', async function (this: DifyWorld) { await this.getPage().goto(getAgentConfigurePath(getCurrentAgentId(this))) }) @@ -94,6 +107,27 @@ When('I discard the Agent v2 Build draft', async function (this: DifyWorld) { await this.getPage().getByRole('button', { name: 'Discard' }).click() }) +When('I apply the Agent v2 Build draft', async function (this: DifyWorld) { + const page = this.getPage() + const agentId = getCurrentAgentId(this) + const applyButton = page.getByRole('button', { name: 'Apply' }) + + await expect(applyButton).toBeEnabled({ timeout: 30_000 }) + const finalizeResponsePromise = page.waitForResponse(response => ( + response.request().method() === 'POST' + && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/build-chat/finalize`) + ), { timeout: 120_000 }) + const applyResponsePromise = page.waitForResponse(response => ( + response.request().method() === 'POST' + && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/build-draft/apply`) + ), { timeout: 120_000 }) + + await applyButton.click() + expect((await finalizeResponsePromise).ok()).toBe(true) + expect((await applyResponsePromise).ok()).toBe(true) + await expect(page.getByText('Action succeeded')).toBeVisible() +}) + When('I publish the Agent v2 draft', async function (this: DifyWorld) { const page = this.getPage() const publishButton = page.getByRole('button', { name: /^Publish(?: update)?$/ }) @@ -358,6 +392,16 @@ Then( }, ) +Then( + 'the normal Agent v2 draft should use the updated E2E prompt', + async function (this: DifyWorld) { + await expect.poll( + async () => (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul?.prompt, + { timeout: 30_000 }, + ).toEqual({ system_prompt: updatedAgentPrompt }) + }, +) + Then('the Agent v2 Build draft should no longer be active', async function (this: DifyWorld) { const page = this.getPage() From 90a7a0fb9ef2aa4a1ecb896a18bd9d3a5cdb0920 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 15:43:03 +0800 Subject: [PATCH 042/185] test(e2e): enforce e2e resource names --- e2e/support/agent.ts | 3 ++- e2e/support/api.ts | 4 +++- e2e/support/naming.ts | 7 +++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/e2e/support/agent.ts b/e2e/support/agent.ts index 918289e277663b..49af5dc0d23b1c 100644 --- a/e2e/support/agent.ts +++ b/e2e/support/agent.ts @@ -1,5 +1,5 @@ import { createApiContext, expectApiResponseOK, setAppSiteEnabled } from './api' -import { createE2EResourceName } from './naming' +import { assertE2EResourceName, createE2EResourceName } from './naming' export type AgentSeed = { active_config_is_published?: boolean @@ -142,6 +142,7 @@ export async function createTestAgent({ name = createE2EResourceName('Agent'), role = 'E2E test assistant', }: CreateTestAgentOptions = {}): Promise { + assertE2EResourceName(name, 'Agent') const ctx = await createApiContext() try { const response = await ctx.post('/console/api/agent', { diff --git a/e2e/support/api.ts b/e2e/support/api.ts index f03de5f349b5fb..e11bab1691318d 100644 --- a/e2e/support/api.ts +++ b/e2e/support/api.ts @@ -3,7 +3,7 @@ import { readFile } from 'node:fs/promises' import { request } from '@playwright/test' import { authStatePath } from '../fixtures/auth' import { apiURL } from '../test-env' -import { createE2EResourceName } from './naming' +import { assertE2EResourceName, createE2EResourceName } from './naming' type StorageState = { cookies: Array<{ name: string, value: string }> @@ -37,6 +37,7 @@ export async function createTestApp( name = createE2EResourceName('App'), mode = 'workflow', ): Promise { + assertE2EResourceName(name, 'App') const ctx = await createApiContext() try { const response = await ctx.post('/console/api/apps', { @@ -48,6 +49,7 @@ export async function createTestApp( icon_background: '#FFEAD5', }, }) + await expectApiResponseOK(response, `Create ${mode} app ${name}`) const body = (await response.json()) as AppSeed return body } diff --git a/e2e/support/naming.ts b/e2e/support/naming.ts index 7a973c204af035..b830e599d2fb5e 100644 --- a/e2e/support/naming.ts +++ b/e2e/support/naming.ts @@ -2,3 +2,10 @@ export const createE2EResourceName = (resource: string, qualifier?: string) => { const nonce = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}` return ['E2E', qualifier, resource, nonce].filter(Boolean).join(' ') } + +export function assertE2EResourceName(name: string, resource: string) { + if (name.startsWith('E2E ')) + return + + throw new Error(`${resource} test resources must use an "E2E " name prefix: ${name}`) +} From 740c4eb0f457133855a3ba98f6f96184c90ae9d8 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 16:08:12 +0800 Subject: [PATCH 043/185] test(e2e): cover agent v2 output variables --- .../agent-v2/output-variables.feature | 17 ++ .../agent-v2/workflow-node.steps.ts | 150 ++++++++++++++++++ e2e/features/support/world.ts | 6 + e2e/support/api.ts | 70 ++++++++ .../nodes/_base/__tests__/node.spec.tsx | 23 ++- .../components/workflow/nodes/_base/node.tsx | 35 ++-- 6 files changed, 285 insertions(+), 16 deletions(-) create mode 100644 e2e/features/agent-v2/output-variables.feature create mode 100644 e2e/features/step-definitions/agent-v2/workflow-node.steps.ts diff --git a/e2e/features/agent-v2/output-variables.feature b/e2e/features/agent-v2/output-variables.feature new file mode 100644 index 00000000000000..7ece4d68b9ec92 --- /dev/null +++ b/e2e/features/agent-v2/output-variables.feature @@ -0,0 +1,17 @@ +@agent-v2 @authenticated @output-variables @core +Feature: Agent v2 output variables + Scenario: Workflow Agent v2 output variables persist after refresh + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a workflow app with an Agent v2 node has been created via API + When I open the app from the app list + And I open the Agent v2 workflow node panel + And I add these Agent v2 workflow node output variables + | name | type | + | e2e_summary | string | + | e2e_report_pdf | file | + | e2e_topics | array[string] | + Then the Agent v2 workflow node output variables should be saved in the workflow draft + When I refresh the current page + And I open the Agent v2 workflow node panel + Then I should see the Agent v2 workflow node output variables diff --git a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts new file mode 100644 index 00000000000000..67e5c4e82aa9a8 --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts @@ -0,0 +1,150 @@ +import type { DataTable } from '@cucumber/cucumber' +import type { AgentV2WorkflowOutputVariable, DifyWorld } from '../../support/world' +import { Given, Then, When } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { + createAgentSoulConfigWithModel, + createConfiguredTestAgent, + normalAgentSoulConfig, +} from '../../../support/agent' +import { + createTestApp, + getWorkflowDraft, + syncAgentV2WorkflowDraft, +} from '../../../support/api' +import { createE2EResourceName } from '../../../support/naming' + +const agentV2WorkflowNodeId = 'agent-v2' + +const getCurrentAppId = (world: DifyWorld) => { + const appId = world.createdAppIds.at(-1) + if (!appId) + throw new Error('No app ID found. Create a workflow app first.') + + return appId +} + +const getOutputVariablesFromDraft = async (appId: string) => { + const draft = await getWorkflowDraft(appId) + const agentNode = draft.graph.nodes.find(node => node.id === agentV2WorkflowNodeId) + if (!agentNode) + throw new Error(`Workflow draft ${appId} does not include Agent v2 node ${agentV2WorkflowNodeId}.`) + + const outputs = agentNode.data?.agent_declared_outputs + if (!Array.isArray(outputs)) + return [] + + return outputs as Array<{ + array_item?: { type?: string } + name?: string + type?: string + }> +} + +Given( + 'a workflow app with an Agent v2 node has been created via API', + async function (this: DifyWorld) { + if (!this.agentBuilderStableChatModel) + throw new Error('Create an Agent v2 workflow node after stable model preflight.') + + const agent = await createConfiguredTestAgent({ + agentSoul: createAgentSoulConfigWithModel( + normalAgentSoulConfig, + this.agentBuilderStableChatModel, + ), + }) + this.createdAgentIds.push(agent.id) + this.lastCreatedAgentName = agent.name + this.lastCreatedAgentRole = agent.role + + const app = await createTestApp(createE2EResourceName('App', 'workflow-agent-v2'), 'workflow') + this.createdAppIds.push(app.id) + this.lastCreatedAppName = app.name + + await syncAgentV2WorkflowDraft(app.id, agent.id) + }, +) + +When('I open the Agent v2 workflow node panel', async function (this: DifyWorld) { + const page = this.getPage() + const workflowCanvas = page.locator('#workflow-container') + const agentNode = workflowCanvas.getByRole('button', { name: 'Agent' }).first() + + await expect(agentNode).toBeVisible({ timeout: 30_000 }) + await agentNode.click() + await expect(page.getByRole('button', { name: 'Output Variables' })).toBeVisible() +}) + +When( + 'I add these Agent v2 workflow node output variables', + async function (this: DifyWorld, table: DataTable) { + const page = this.getPage() + const appId = getCurrentAppId(this) + const rows = table.hashes() as AgentV2WorkflowOutputVariable[] + this.agentV2WorkflowOutputVariables = rows + + await page.getByRole('button', { name: 'Output Variables' }).click() + + for (const row of rows) { + await page.getByRole('button', { name: 'New output' }).click() + const editor = page.getByRole('form', { name: 'Output variable editor' }) + await expect(editor).toBeVisible() + + await editor.getByRole('textbox', { name: 'Field name' }).fill(row.name) + if (row.type !== 'string') { + await editor.getByRole('button', { name: 'Output type' }).click() + await page.getByRole('option', { name: row.type }).click() + } + + const saveResponse = page.waitForResponse(response => ( + response.request().method() === 'POST' + && new URL(response.url()).pathname.endsWith(`/console/api/apps/${appId}/workflows/draft`) + )) + await editor.getByRole('button', { name: 'Confirm' }).click() + expect((await saveResponse).ok()).toBe(true) + await expect(editor).not.toBeVisible() + } + }, +) + +Then( + 'the Agent v2 workflow node output variables should be saved in the workflow draft', + async function (this: DifyWorld) { + const appId = getCurrentAppId(this) + const expectedOutputVariables = this.agentV2WorkflowOutputVariables + if (expectedOutputVariables.length === 0) + throw new Error('No Agent v2 workflow output variables were recorded for this scenario.') + + await expect + .poll(async () => { + const outputs = await getOutputVariablesFromDraft(appId) + + return expectedOutputVariables.map((expected) => { + const output = outputs.find(item => item.name === expected.name) + return { + name: output?.name, + type: output?.type === 'array' + ? `array[${output.array_item?.type ?? 'object'}]` + : output?.type, + } + }) + }, { + timeout: 30_000, + }) + .toEqual(expectedOutputVariables) + }, +) + +Then('I should see the Agent v2 workflow node output variables', async function (this: DifyWorld) { + const page = this.getPage() + const expectedOutputVariables = this.agentV2WorkflowOutputVariables + if (expectedOutputVariables.length === 0) + throw new Error('No Agent v2 workflow output variables were recorded for this scenario.') + + await page.getByRole('button', { name: 'Output Variables' }).click() + + for (const output of expectedOutputVariables) { + await expect(page.getByText(output.name, { exact: true })).toBeVisible() + await expect(page.getByText(output.type, { exact: true })).toBeVisible() + } +}) diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index a84be25868bc00..58ff2e01b710e4 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -28,6 +28,10 @@ export type AgentBuilderPreseededResource = { kind: 'agent' | 'api-key' | 'dataset' | 'skill' | 'tool' | 'workflow' name: string } +export type AgentV2WorkflowOutputVariable = { + name: string + type: string +} export class DifyWorld extends World { context: BrowserContext | undefined @@ -51,6 +55,7 @@ export class DifyWorld extends World { agentBuilderBrokenChatModel: AgentBuilderStableChatModel | undefined agentBuilderStableChatModel: AgentBuilderStableChatModel | undefined agentBuilderPreseededResources: Record = {} + agentV2WorkflowOutputVariables: AgentV2WorkflowOutputVariable[] = [] scenarioCleanups: ScenarioCleanup[] = [] capturedDownloads: Download[] = [] shareURL: string | undefined @@ -78,6 +83,7 @@ export class DifyWorld extends World { this.agentBuilderBrokenChatModel = undefined this.agentBuilderStableChatModel = undefined this.agentBuilderPreseededResources = {} + this.agentV2WorkflowOutputVariables = [] this.scenarioCleanups = [] this.capturedDownloads = [] this.shareURL = undefined diff --git a/e2e/support/api.ts b/e2e/support/api.ts index e11bab1691318d..ae5706837a209c 100644 --- a/e2e/support/api.ts +++ b/e2e/support/api.ts @@ -33,6 +33,18 @@ export type AppSeed = { name: string } +export type WorkflowDraft = { + graph: { + edges: Array> + nodes: Array<{ + data?: Record + id: string + type: string + }> + viewport?: Record + } +} + export async function createTestApp( name = createE2EResourceName('App'), mode = 'workflow', @@ -58,6 +70,18 @@ export async function createTestApp( } } +export async function getWorkflowDraft(appId: string): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/apps/${appId}/workflows/draft`) + await expectApiResponseOK(response, `Get workflow draft for ${appId}`) + return (await response.json()) as WorkflowDraft + } + finally { + await ctx.dispose() + } +} + export async function syncMinimalWorkflowDraft(appId: string): Promise { const ctx = await createApiContext() try { @@ -86,6 +110,52 @@ export async function syncMinimalWorkflowDraft(appId: string): Promise { } } +export async function syncAgentV2WorkflowDraft(appId: string, agentId: string): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.post(`/console/api/apps/${appId}/workflows/draft`, { + data: { + graph: { + nodes: [ + { + id: 'start', + type: 'custom', + position: { x: 80, y: 282 }, + data: { id: 'start', type: 'start', title: 'Start', variables: [] }, + }, + { + id: 'agent-v2', + type: 'custom', + position: { x: 420, y: 282 }, + data: { + id: 'agent-v2', + type: 'agent', + title: 'Agent', + desc: '', + agent_binding: { + binding_type: 'roster_agent', + agent_id: agentId, + }, + agent_node_kind: 'dify_agent', + version: '2', + }, + }, + ], + edges: [], + viewport: { x: 0, y: 0, zoom: 1 }, + }, + features: {}, + environment_variables: [], + conversation_variables: [], + }, + }) + await expectApiResponseOK(response, `Sync Agent v2 workflow draft for ${appId}`) + } + finally { + await ctx.dispose() + } +} + export async function deleteTestApp(id: string): Promise { const ctx = await createApiContext() try { diff --git a/web/app/components/workflow/nodes/_base/__tests__/node.spec.tsx b/web/app/components/workflow/nodes/_base/__tests__/node.spec.tsx index e6ddbdc44206f3..bbc2d3c688a782 100644 --- a/web/app/components/workflow/nodes/_base/__tests__/node.spec.tsx +++ b/web/app/components/workflow/nodes/_base/__tests__/node.spec.tsx @@ -81,6 +81,9 @@ vi.mock('@/app/components/workflow/block-icon', () => ({ vi.mock('@/app/components/workflow/nodes/tool/components/copy-id', () => ({ default: ({ content }: { content: string }) =>
{content}
, })) +vi.mock('@/app/components/workflow/utils/node-navigation', () => ({ + selectWorkflowNode: vi.fn(), +})) const createData = (overrides: Record = {}) => ({ type: BlockEnum.Tool, @@ -127,6 +130,22 @@ describe('BaseNode', () => { expect(screen.getByTestId('node-target-handle')).toBeInTheDocument() }) + it('should expose the node header as a selectable button', async () => { + const { selectWorkflowNode } = await import('@/app/components/workflow/utils/node-navigation') + + renderWorkflowComponent( + +
Body
+ , + ) + + const node = screen.getByRole('button', { name: 'Node title' }) + + fireEvent.click(node) + + expect(selectWorkflowNode).toHaveBeenCalledWith('node-1') + }) + it('should render entry nodes inside the entry container', () => { renderWorkflowComponent( @@ -152,9 +171,9 @@ describe('BaseNode', () => { , ) - const overlay = screen.getByTestId('workflow-node-install-overlay') + const overlay = screen.getByRole('button', { name: 'plugin.installPlugin' }) expect(overlay).toBeInTheDocument() - fireEvent.click(overlay) + expect(overlay).toBeDisabled() }) it('should render running status indicators for loop nodes', () => { diff --git a/web/app/components/workflow/nodes/_base/node.tsx b/web/app/components/workflow/nodes/_base/node.tsx index 680a8894d0edcf..a688e2030ab697 100644 --- a/web/app/components/workflow/nodes/_base/node.tsx +++ b/web/app/components/workflow/nodes/_base/node.tsx @@ -29,6 +29,7 @@ import { } from '@/app/components/workflow/types' import { hasErrorHandleNode, hasRetryNode } from '@/app/components/workflow/utils' import { useAppContext } from '@/context/app-context' +import { selectWorkflowNode } from '../../utils/node-navigation' import AddVariablePopupWithPosition from './components/add-variable-popup-with-position' import EntryNodeContainer, { StartNodeTypeEnum } from './components/entry-node-container' import ErrorHandleOnNode from './components/error-handle/error-handle-on-node' @@ -168,15 +169,17 @@ const BaseNode: FC = ({ height: isContainerNode(data.type) ? data.height : 'auto', }} > - {(data._dimmed || pluginDimmed || pluginInstallLocked) && ( + {pluginInstallLocked && ( + )} /> { hasRetryNode(data.type) && ( From ac495ebc755470e85e7d75eebea817e48783c625 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Wed, 1 Jul 2026 08:12:03 +0000 Subject: [PATCH 044/185] [autofix.ci] apply automated fixes --- eslint-suppressions.json | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/eslint-suppressions.json b/eslint-suppressions.json index d9d06836644ab5..73a52c346f0d98 100644 --- a/eslint-suppressions.json +++ b/eslint-suppressions.json @@ -5227,17 +5227,6 @@ "count": 2 } }, - "web/app/components/workflow/nodes/_base/node.tsx": { - "jsx-a11y/click-events-have-key-events": { - "count": 1 - }, - "jsx-a11y/no-static-element-interactions": { - "count": 1 - }, - "ts/no-explicit-any": { - "count": 1 - } - }, "web/app/components/workflow/nodes/_base/types.ts": { "erasable-syntax-only/enums": { "count": 1 From 47742145063b4d0bca1bd95c82f6ec5e31406199 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 16:14:09 +0800 Subject: [PATCH 045/185] test(e2e): verify agent lifecycle preflight --- e2e/features/agent-v2/preflight.feature | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/e2e/features/agent-v2/preflight.feature b/e2e/features/agent-v2/preflight.feature index f693d39bc59985..76e2b539d5ba34 100644 --- a/e2e/features/agent-v2/preflight.feature +++ b/e2e/features/agent-v2/preflight.feature @@ -1,5 +1,14 @@ @agent-v2 @authenticated @infra @agent-v2-preflight Feature: Agent Builder preseeded environment + @agent-lifecycle + Scenario: Agent lifecycle permissions are available + Given I am signed in as the default E2E admin + And an Agent v2 test agent has been created via API + And the Agent v2 composer draft uses the normal E2E prompt + When I open the Agent v2 configure page + And I publish the Agent v2 draft + Then the Agent v2 draft should be published and up to date + Scenario: Stable chat model is available Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available From 30c7db7e84ab2cebdfe8c4aa4ad567a20d52f221 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 16:24:38 +0800 Subject: [PATCH 046/185] test(e2e): cover agent advanced settings entries --- .../agent-v2/advanced-settings.feature | 8 ++++ .../agent-v2/configure.steps.ts | 41 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/e2e/features/agent-v2/advanced-settings.feature b/e2e/features/agent-v2/advanced-settings.feature index 0613dfd1b66133..65da1bd70460e5 100644 --- a/e2e/features/agent-v2/advanced-settings.feature +++ b/e2e/features/agent-v2/advanced-settings.feature @@ -1,5 +1,13 @@ @agent-v2 @authenticated @advanced-settings @core Feature: Agent v2 advanced settings + Scenario: Advanced Settings exposes supported configuration entries + Given I am signed in as the default E2E admin + And a basic configured Agent v2 test agent has been created via API + When I open the Agent v2 configure page + Then Agent v2 Advanced Settings should describe supported entries while collapsed + When I expand Agent v2 Advanced Settings + Then I should see the supported Agent v2 Advanced Settings entries + Scenario: Plain environment variables are saved and restored Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 9958789ae67e57..bae56345f10971 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -199,6 +199,14 @@ When( }, ) +When('I expand Agent v2 Advanced Settings', async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() +}) + Then('I should be on the Agent v2 configure page', async function (this: DifyWorld) { const agentId = getCurrentAgentId(this) @@ -268,6 +276,22 @@ Then( }, ) +Then( + 'Agent v2 Advanced Settings should describe supported entries while collapsed', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await expect(advancedSettings).toBeVisible() + await expect( + advancedSettings.getByText('For power users. Env vars, sandbox & memory.'), + ).toBeVisible() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })) + .not + .toBeVisible() + }, +) + Then( 'the plain Agent v2 environment variable should be saved in the Agent v2 draft', async function (this: DifyWorld) { @@ -334,6 +358,23 @@ Then( }, ) +Then( + 'I should see the supported Agent v2 Advanced Settings entries', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + const envEditor = advancedSettings.getByRole('region', { name: 'Env Editor' }) + + await expect(envEditor).toBeVisible() + await expect(envEditor.getByRole('button', { name: 'Import .env' })).toBeVisible() + await expect(envEditor.getByRole('button', { name: 'Add environment variable' })) + .toBeVisible() + await expect(envEditor.getByText('Key', { exact: true })).toBeVisible() + await expect(envEditor.getByText('Value', { exact: true })).toBeVisible() + await expect(envEditor.getByText('Scope', { exact: true })).toBeVisible() + }, +) + Then( 'I should see the plain Agent v2 environment variable in Advanced Settings', async function (this: DifyWorld) { From 969bdd85a4051d1b810e146e44f1cf074f017fee Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 16:30:34 +0800 Subject: [PATCH 047/185] test(e2e): cover agent env imports --- .../agent-v2/advanced-settings.feature | 9 ++ .../agent-v2/configure.steps.ts | 98 ++++++++++++++++--- e2e/support/agent-builder-resources.ts | 2 + 3 files changed, 93 insertions(+), 16 deletions(-) diff --git a/e2e/features/agent-v2/advanced-settings.feature b/e2e/features/agent-v2/advanced-settings.feature index 65da1bd70460e5..8cdc319a81f67f 100644 --- a/e2e/features/agent-v2/advanced-settings.feature +++ b/e2e/features/agent-v2/advanced-settings.feature @@ -17,6 +17,15 @@ Feature: Agent v2 advanced settings When I refresh the current page Then I should see the plain Agent v2 environment variable in Advanced Settings + Scenario: Valid environment imports are saved and restored + Given I am signed in as the default E2E admin + And a basic configured Agent v2 test agent has been created via API + When I open the Agent v2 configure page + And I import the valid Agent v2 environment file from Advanced Settings + Then the valid Agent v2 environment import should be saved in the Agent v2 draft + When I refresh the current page + Then I should see the Agent v2 environment variables from the valid import in Advanced Settings + Scenario: Invalid environment imports report skipped lines and keep existing variables Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index bae56345f10971..74e85bcfa7819b 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -1,3 +1,4 @@ +import type { AgentComposerEnvVariable } from '../../../support/agent' import type { DifyWorld } from '../../support/world' import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' @@ -27,6 +28,17 @@ const getCurrentAgentId = (world: DifyWorld) => { return agentId } +const getEnvVariableKey = (variable: AgentComposerEnvVariable) => + variable.key ?? variable.name ?? variable.variable + +const getAgentEnvVariableValue = ( + variables: AgentComposerEnvVariable[], + key: string, +) => variables.find(variable => getEnvVariableKey(variable) === key)?.value + +const getAgentEnvVariables = async (agentId: string) => + (await getAgentComposerDraft(agentId)).agent_soul?.env?.variables ?? [] + Given('an Agent v2 test agent has been created via API', async function (this: DifyWorld) { const agent = await createTestAgent() this.createdAgentIds.push(agent.id) @@ -199,6 +211,21 @@ When( }, ) +When( + 'I import the valid Agent v2 environment file from Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() + + const fileChooserPromise = page.waitForEvent('filechooser') + await advancedSettings.getByRole('button', { name: 'Import .env' }).click() + await (await fileChooserPromise).setFiles(getAgentBuilderTestMaterialPath('validEnv')) + }, +) + When('I expand Agent v2 Advanced Settings', async function (this: DifyWorld) { const page = this.getPage() const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) @@ -300,11 +327,9 @@ Then( await expect .poll(async () => { const env = (await getAgentComposerDraft(agentId)).agent_soul?.env - const variable = env?.variables?.find((item) => { - const key = item.key ?? item.name ?? item.variable - - return key === agentBuilderFixedInputs.envPlainKey - }) + const variable = env?.variables?.find(item => + getEnvVariableKey(item) === agentBuilderFixedInputs.envPlainKey, + ) return { secretCount: env?.secret_refs?.length ?? 0, @@ -320,6 +345,29 @@ Then( }, ) +Then( + 'the valid Agent v2 environment import should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect + .poll(async () => { + const variables = await getAgentEnvVariables(agentId) + + return { + modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey), + plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), + } + }, { + timeout: 30_000, + }) + .toEqual({ + modeValue: agentBuilderFixedInputs.envModeValue, + plainValue: agentBuilderFixedInputs.envPlainValue, + }) + }, +) + Then( 'the invalid Agent v2 environment import should report skipped lines', async function (this: DifyWorld) { @@ -334,19 +382,14 @@ Then( await expect .poll(async () => { - const variables = (await getAgentComposerDraft(agentId)).agent_soul?.env?.variables ?? [] + const variables = await getAgentEnvVariables(agentId) return { - importedValue: variables.find((item) => { - const key = item.key ?? item.name ?? item.variable - - return key === agentBuilderFixedInputs.envAfterInvalidImportKey - })?.value, - plainValue: variables.find((item) => { - const key = item.key ?? item.name ?? item.variable - - return key === agentBuilderFixedInputs.envPlainKey - })?.value, + importedValue: getAgentEnvVariableValue( + variables, + agentBuilderFixedInputs.envAfterInvalidImportKey, + ), + plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), } }, { timeout: 30_000, @@ -375,6 +418,29 @@ Then( }, ) +Then( + 'I should see the Agent v2 environment variables from the valid import in Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() + await expect.poll( + async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => + inputs.map(input => (input as HTMLInputElement).value), + ), + { timeout: 30_000 }, + ).toEqual(expect.arrayContaining([ + agentBuilderFixedInputs.envPlainKey, + agentBuilderFixedInputs.envPlainValue, + agentBuilderFixedInputs.envModeKey, + agentBuilderFixedInputs.envModeValue, + ])) + await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(2) + }, +) + Then( 'I should see the plain Agent v2 environment variable in Advanced Settings', async function (this: DifyWorld) { diff --git a/e2e/support/agent-builder-resources.ts b/e2e/support/agent-builder-resources.ts index a1f795aa5cf382..16cff4ba563b57 100644 --- a/e2e/support/agent-builder-resources.ts +++ b/e2e/support/agent-builder-resources.ts @@ -26,6 +26,8 @@ export const agentBuilderFixedInputs = { customKnowledgeQuery: 'Dify Agent E2E 测试暗号', envPlainKey: 'E2E_AGENT_FLAG', envPlainValue: 'enabled', + envModeKey: 'E2E_AGENT_MODE', + envModeValue: 'plain', envAfterInvalidImportKey: 'E2E_AGENT_AFTER_INVALID', envAfterInvalidImportValue: 'still-valid', inputModerationReply: 'E2E_INPUT_BLOCKED_REPLY', From 764f971551a7e6942b88289968ed38e8b89cf32f Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 16:34:12 +0800 Subject: [PATCH 048/185] test(e2e): cover agent env deletion --- .../agent-v2/advanced-settings.feature | 12 ++ .../agent-v2/configure.steps.ts | 104 ++++++++++++++++++ 2 files changed, 116 insertions(+) diff --git a/e2e/features/agent-v2/advanced-settings.feature b/e2e/features/agent-v2/advanced-settings.feature index 8cdc319a81f67f..a14aaee97f0636 100644 --- a/e2e/features/agent-v2/advanced-settings.feature +++ b/e2e/features/agent-v2/advanced-settings.feature @@ -26,6 +26,18 @@ Feature: Agent v2 advanced settings When I refresh the current page Then I should see the Agent v2 environment variables from the valid import in Advanced Settings + Scenario: Deleted environment variables are removed after refresh + Given I am signed in as the default E2E admin + And a basic configured Agent v2 test agent has been created via API + When I open the Agent v2 configure page + And I add the plain Agent v2 environment variable from Advanced Settings + And I add the secondary plain Agent v2 environment variable from Advanced Settings + Then the Agent v2 environment variables for deletion should be saved in the Agent v2 draft + When I delete the plain Agent v2 environment variable from Advanced Settings + Then the plain Agent v2 environment variable should be removed from the Agent v2 draft + When I refresh the current page + Then I should not see the deleted Agent v2 environment variable in Advanced Settings + Scenario: Invalid environment imports report skipped lines and keep existing variables Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 74e85bcfa7819b..f254379ff0923a 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -211,6 +211,37 @@ When( }, ) +When( + 'I add the secondary plain Agent v2 environment variable from Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await advancedSettings.getByRole('button', { name: 'Add environment variable' }).click() + await advancedSettings + .getByRole('textbox', { name: 'Key' }) + .last() + .fill(agentBuilderFixedInputs.envModeKey) + await advancedSettings + .getByRole('textbox', { name: 'Value' }) + .last() + .fill(agentBuilderFixedInputs.envModeValue) + await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(2) + }, +) + +When( + 'I delete the plain Agent v2 environment variable from Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await advancedSettings + .getByRole('button', { name: `Delete ${agentBuilderFixedInputs.envPlainKey}` }) + .click() + }, +) + When( 'I import the valid Agent v2 environment file from Advanced Settings', async function (this: DifyWorld) { @@ -345,6 +376,52 @@ Then( }, ) +Then( + 'the Agent v2 environment variables for deletion should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect + .poll(async () => { + const variables = await getAgentEnvVariables(agentId) + + return { + modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey), + plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), + } + }, { + timeout: 30_000, + }) + .toEqual({ + modeValue: agentBuilderFixedInputs.envModeValue, + plainValue: agentBuilderFixedInputs.envPlainValue, + }) + }, +) + +Then( + 'the plain Agent v2 environment variable should be removed from the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect + .poll(async () => { + const variables = await getAgentEnvVariables(agentId) + + return { + modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey), + plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), + } + }, { + timeout: 30_000, + }) + .toEqual({ + modeValue: agentBuilderFixedInputs.envModeValue, + plainValue: undefined, + }) + }, +) + Then( 'the valid Agent v2 environment import should be saved in the Agent v2 draft', async function (this: DifyWorld) { @@ -441,6 +518,33 @@ Then( }, ) +Then( + 'I should not see the deleted Agent v2 environment variable in Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() + await expect.poll( + async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => + inputs.map(input => (input as HTMLInputElement).value), + ), + { timeout: 30_000 }, + ).toEqual(expect.arrayContaining([ + agentBuilderFixedInputs.envModeKey, + agentBuilderFixedInputs.envModeValue, + ])) + await expect.poll( + async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => + inputs.map(input => (input as HTMLInputElement).value), + ), + { timeout: 30_000 }, + ).not.toContain(agentBuilderFixedInputs.envPlainKey) + await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(1) + }, +) + Then( 'I should see the plain Agent v2 environment variable in Advanced Settings', async function (this: DifyWorld) { From 35e577a0d41d63d1be84b57be40bd11fe2d08b56 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 16:40:03 +0800 Subject: [PATCH 049/185] test(e2e): cover agent special filename upload --- e2e/features/agent-v2/files.feature | 10 ++ .../agent-v2/configure.steps.ts | 125 ++++++++++++------ 2 files changed, 93 insertions(+), 42 deletions(-) diff --git a/e2e/features/agent-v2/files.feature b/e2e/features/agent-v2/files.feature index db24f03f48baa3..8b352f9d86548a 100644 --- a/e2e/features/agent-v2/files.feature +++ b/e2e/features/agent-v2/files.feature @@ -9,3 +9,13 @@ Feature: Agent v2 files And the small Agent v2 file should be saved in the Agent v2 draft When I refresh the current page Then I should see the small Agent v2 file in the Files section + + Scenario: Uploading a special-name file keeps the filename readable + Given I am signed in as the default E2E admin + And a basic configured Agent v2 test agent has been created via API + When I open the Agent v2 configure page + And I upload the special-name Agent v2 file from the Files section + Then I should see the special-name Agent v2 file in the Files section + And the special-name Agent v2 file should be saved in the Agent v2 draft + When I refresh the current page + Then I should see the special-name Agent v2 file in the Files section diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index f254379ff0923a..57e06d16c1609e 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -18,7 +18,10 @@ import { } from '../../../support/agent' import { agentBuilderFixedInputs } from '../../../support/agent-builder-resources' import { waitForAgentConfigureAutosaved } from '../../../support/agent-configure' -import { agentBuilderTestMaterials, getAgentBuilderTestMaterialPath } from '../../../support/test-materials' +import { + agentBuilderTestMaterials, + getAgentBuilderTestMaterialPath, +} from '../../../support/test-materials' const getCurrentAgentId = (world: DifyWorld) => { const agentId = world.createdAgentIds.at(-1) @@ -39,6 +42,69 @@ const getAgentEnvVariableValue = ( const getAgentEnvVariables = async (agentId: string) => (await getAgentComposerDraft(agentId)).agent_soul?.env?.variables ?? [] +const uploadAgentConfigFile = async ( + world: DifyWorld, + material: keyof typeof agentBuilderTestMaterials, +) => { + const page = world.getPage() + const agentId = getCurrentAgentId(world) + const fileName = agentBuilderTestMaterials[material] + const filePath = getAgentBuilderTestMaterialPath(material) + + await page.getByRole('button', { name: 'Add file' }).click() + const dialog = page.getByRole('dialog', { name: 'Upload file' }) + await expect(dialog).toBeVisible() + + const fileChooserPromise = page.waitForEvent('filechooser') + await dialog.getByRole('button', { name: 'browse' }).click() + await (await fileChooserPromise).setFiles(filePath) + await expect(dialog.getByText(fileName)).toBeVisible() + + const commitResponsePromise = page.waitForResponse(response => ( + response.request().method() === 'POST' + && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/config/files`) + )) + await dialog.getByRole('button', { name: 'Upload' }).click() + const commitResponse = await commitResponsePromise + expect(commitResponse.status()).toBe(201) + const committed = await commitResponse.json() as { file?: { name?: string } } + await expect(dialog).not.toBeVisible({ timeout: 30_000 }) + + const committedName = committed.file?.name + if (!committedName) + throw new Error('Agent config file upload response did not include a file name.') + + world.createdAgentConfigFiles.push({ agentId, name: committedName }) +} + +const expectAgentConfigFileVisible = async ( + world: DifyWorld, + material: keyof typeof agentBuilderTestMaterials, +) => { + await expect( + world.getPage().getByRole('button', { + exact: true, + name: agentBuilderTestMaterials[material], + }), + ).toBeVisible({ timeout: 30_000 }) +} + +const expectAgentConfigFileSaved = async ( + world: DifyWorld, + material: keyof typeof agentBuilderTestMaterials, +) => { + const agentId = getCurrentAgentId(world) + const fileName = agentBuilderTestMaterials[material] + + await expect + .poll(async () => ( + await getAgentComposerDraft(agentId) + ).agent_soul?.config_files?.map(file => file.name) ?? [], { + timeout: 30_000, + }) + .toContain(fileName) +} + Given('an Agent v2 test agent has been created via API', async function (this: DifyWorld) { const agent = await createTestAgent() this.createdAgentIds.push(agent.id) @@ -149,35 +215,11 @@ When('I publish the Agent v2 draft', async function (this: DifyWorld) { }) When('I upload the small Agent v2 file from the Files section', async function (this: DifyWorld) { - const page = this.getPage() - const agentId = getCurrentAgentId(this) - const fileName = agentBuilderTestMaterials.smallFile - const filePath = getAgentBuilderTestMaterialPath('smallFile') - - await page.getByRole('button', { name: 'Add file' }).click() - const dialog = page.getByRole('dialog', { name: 'Upload file' }) - await expect(dialog).toBeVisible() - - const fileChooserPromise = page.waitForEvent('filechooser') - await dialog.getByRole('button', { name: 'browse' }).click() - await (await fileChooserPromise).setFiles(filePath) - await expect(dialog.getByText(fileName)).toBeVisible() - - const commitResponsePromise = page.waitForResponse(response => ( - response.request().method() === 'POST' - && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/config/files`) - )) - await dialog.getByRole('button', { name: 'Upload' }).click() - const commitResponse = await commitResponsePromise - expect(commitResponse.status()).toBe(201) - const committed = await commitResponse.json() as { file?: { name?: string } } - await expect(dialog).not.toBeVisible({ timeout: 30_000 }) - - const committedName = committed.file?.name - if (!committedName) - throw new Error('Agent config file upload response did not include a file name.') + await uploadAgentConfigFile(this, 'smallFile') +}) - this.createdAgentConfigFiles.push({ agentId, name: committedName }) +When('I upload the special-name Agent v2 file from the Files section', async function (this: DifyWorld) { + await uploadAgentConfigFile(this, 'specialFilename') }) When( @@ -312,25 +354,24 @@ Then( ) Then('I should see the small Agent v2 file in the Files section', async function (this: DifyWorld) { - const fileName = agentBuilderTestMaterials.smallFile - await expect( - this.getPage().getByRole('button', { exact: true, name: fileName }), - ).toBeVisible({ timeout: 30_000 }) + await expectAgentConfigFileVisible(this, 'smallFile') +}) + +Then('I should see the special-name Agent v2 file in the Files section', async function (this: DifyWorld) { + await expectAgentConfigFileVisible(this, 'specialFilename') }) Then( 'the small Agent v2 file should be saved in the Agent v2 draft', async function (this: DifyWorld) { - const agentId = getCurrentAgentId(this) - const fileName = agentBuilderTestMaterials.smallFile + await expectAgentConfigFileSaved(this, 'smallFile') + }, +) - await expect - .poll(async () => ( - await getAgentComposerDraft(agentId) - ).agent_soul?.config_files?.map(file => file.name) ?? [], { - timeout: 30_000, - }) - .toContain(fileName) +Then( + 'the special-name Agent v2 file should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + await expectAgentConfigFileSaved(this, 'specialFilename') }, ) From a7bd1b13f0388d475aadc5f6120602262f8ae68f Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 16:52:55 +0800 Subject: [PATCH 050/185] test(e2e): rebuild web when sources change --- e2e/scripts/setup.ts | 68 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 6 deletions(-) diff --git a/e2e/scripts/setup.ts b/e2e/scripts/setup.ts index e780feaf0e02f4..b1f623f923b5b6 100644 --- a/e2e/scripts/setup.ts +++ b/e2e/scripts/setup.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto' import { access, mkdir, readFile, rm, writeFile } from 'node:fs/promises' import path from 'node:path' import { waitForUrl } from '../support/process' @@ -17,6 +18,7 @@ import { middlewareEnvExampleFile, middlewareEnvFile, readSimpleDotenv, + rootDir, runCommand, runCommandOrThrow, runForegroundProcess, @@ -25,7 +27,7 @@ import { } from './common' const buildIdPath = path.join(webDir, '.next', 'BUILD_ID') -const webBuildEnvStampPath = path.join(webDir, '.next', 'e2e-web-env.sha256') +const webBuildStampPath = path.join(webDir, '.next', 'e2e-web-build.sha256') const apiHost = '127.0.0.1' const apiPort = 5001 @@ -116,8 +118,62 @@ const waitForDependency = async ({ } } +const webBuildSourcePaths = [ + 'package.json', + 'pnpm-lock.yaml', + 'pnpm-workspace.yaml', + 'packages', + 'web', +] + +const getWebBuildSourceHash = async () => { + const hash = createHash('sha256') + const gitArgsSuffix = ['--', ...webBuildSourcePaths] + const commands = [ + ['rev-parse', 'HEAD'], + ['diff', '--binary', ...gitArgsSuffix], + ['diff', '--cached', '--binary', ...gitArgsSuffix], + ] + + for (const args of commands) { + const result = await runCommandOrThrow({ + command: 'git', + args, + cwd: rootDir, + stdio: 'pipe', + }) + + hash.update(args.join(' ')) + hash.update('\n') + hash.update(result.stdout) + hash.update('\n') + } + + const untrackedFiles = await runCommandOrThrow({ + command: 'git', + args: ['ls-files', '--others', '--exclude-standard', '-z', ...gitArgsSuffix], + cwd: rootDir, + stdio: 'pipe', + }) + + for (const file of untrackedFiles.stdout.split('\0').filter(Boolean)) { + hash.update(file) + hash.update('\0') + hash.update(await readFile(path.join(rootDir, file))) + hash.update('\0') + } + + return hash.digest('hex') +} + export const ensureWebBuild = async () => { const envHash = await getWebEnvLocalHash() + const sourceHash = await getWebBuildSourceHash() + const buildStamp = createHash('sha256') + .update(envHash) + .update('\n') + .update(sourceHash) + .digest('hex') const buildEnv = { ...e2eWebEnvOverrides, } @@ -129,21 +185,21 @@ export const ensureWebBuild = async () => { cwd: webDir, env: buildEnv, }) - await writeFile(webBuildEnvStampPath, `${envHash}\n`, 'utf8') + await writeFile(webBuildStampPath, `${buildStamp}\n`, 'utf8') return } try { - const [buildExists, previousEnvHash] = await Promise.all([ + const [buildExists, previousBuildStamp] = await Promise.all([ access(buildIdPath) .then(() => true) .catch(() => false), - readFile(webBuildEnvStampPath, 'utf8') + readFile(webBuildStampPath, 'utf8') .then(value => value.trim()) .catch(() => ''), ]) - if (buildExists && previousEnvHash === envHash) { + if (buildExists && previousBuildStamp === buildStamp) { console.log('Reusing existing web build artifact.') return } @@ -158,7 +214,7 @@ export const ensureWebBuild = async () => { cwd: webDir, env: buildEnv, }) - await writeFile(webBuildEnvStampPath, `${envHash}\n`, 'utf8') + await writeFile(webBuildStampPath, `${buildStamp}\n`, 'utf8') } export const startWeb = async () => { From 6b658232c0e72343b2a98d4af120d4e267cb8213 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 17:11:50 +0800 Subject: [PATCH 051/185] fix(agent-v2): expose configure autosave status --- .../base/prompt-editor/__tests__/index.spec.tsx | 2 ++ web/app/components/base/prompt-editor/index.tsx | 8 +++++++- .../base/prompt-editor/prompt-editor-content.tsx | 8 +++++++- .../components/__tests__/agent-prompt-editor.spec.tsx | 8 ++++++++ .../components/orchestrate/prompt-editor/index.tsx | 1 + .../components/orchestrate/publish-bar/index.tsx | 8 ++++++-- 6 files changed, 31 insertions(+), 4 deletions(-) diff --git a/web/app/components/base/prompt-editor/__tests__/index.spec.tsx b/web/app/components/base/prompt-editor/__tests__/index.spec.tsx index df89f1cb1ddb22..ee8fc04ef6d7ad 100644 --- a/web/app/components/base/prompt-editor/__tests__/index.spec.tsx +++ b/web/app/components/base/prompt-editor/__tests__/index.spec.tsx @@ -199,6 +199,7 @@ describe('PromptEditor', () => { render( { expect(screen.getByText('Type prompt')).toBeInTheDocument() expect(screen.getByTestId('content-editable')).toHaveClass('editor-class') expect(screen.getByTestId('content-editable')).toHaveClass('text-[13px]') + expect(screen.getByTestId('content-editable')).toHaveAttribute('aria-labelledby', 'prompt-label') await waitFor(() => { expect(onChange).toHaveBeenCalledWith('first line\nsecond line') diff --git a/web/app/components/base/prompt-editor/index.tsx b/web/app/components/base/prompt-editor/index.tsx index c663f9deb0c5ef..ebd6968dd69196 100644 --- a/web/app/components/base/prompt-editor/index.tsx +++ b/web/app/components/base/prompt-editor/index.tsx @@ -110,7 +110,9 @@ const EditableSyncPlugin: FC<{ editable: boolean }> = ({ editable }) => { return null } -export type PromptEditorProps = { +type PromptEditorAriaProps = Pick + +export type PromptEditorProps = PromptEditorAriaProps & { instanceId?: string children?: React.ReactNode compact?: boolean @@ -148,6 +150,8 @@ export type PromptEditorProps = { } const PromptEditor: FC = ({ + 'aria-label': ariaLabel, + 'aria-labelledby': ariaLabelledBy, instanceId, children, compact, @@ -248,6 +252,8 @@ const PromptEditor: FC = ({
void, onInsert: ShortcutPopupInsertHandler }> } -type PromptEditorContentProps = { +type PromptEditorContentAriaProps = Pick + +type PromptEditorContentProps = PromptEditorContentAriaProps & { compact?: boolean className?: string placeholder?: string | React.ReactNode @@ -111,6 +113,8 @@ type PromptEditorContentProps = { } const PromptEditorContent: FC = ({ + 'aria-label': ariaLabel, + 'aria-labelledby': ariaLabelledBy, compact, className, placeholder, @@ -144,6 +148,8 @@ const PromptEditorContent: FC = ({ { // Prompt actions should expose the designed copy control and copy the current draft prompt. describe('Prompt Actions', () => { + it('should label the editable prompt with the visible prompt heading', () => { + renderAgentPromptEditor('Review these tenders') + + expect(mockPromptEditor).toHaveBeenCalledWith(expect.objectContaining({ + 'aria-labelledby': 'agent-configure-prompt-label', + })) + }) + it('should copy the current prompt when the copy button is clicked', () => { renderAgentPromptEditor('Review these tenders') diff --git a/web/features/agent-v2/agent-detail/configure/components/orchestrate/prompt-editor/index.tsx b/web/features/agent-v2/agent-detail/configure/components/orchestrate/prompt-editor/index.tsx index 2f6f818b95ad3b..aef94ad0593815 100644 --- a/web/features/agent-v2/agent-detail/configure/components/orchestrate/prompt-editor/index.tsx +++ b/web/features/agent-v2/agent-detail/configure/components/orchestrate/prompt-editor/index.tsx @@ -423,6 +423,7 @@ export function AgentPromptEditor() {
-
+
{statusLabel} · - + {metaLabel}
From eca1d60cd6e63a77312f1094d7b39db4a81ee096 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 17:12:03 +0800 Subject: [PATCH 052/185] test(e2e): cover agent prompt autosave --- .../agent-v2/configure-persistence.feature | 6 +++-- .../agent-v2/configure.steps.ts | 27 ++++++++++++++++--- e2e/support/agent-configure.ts | 2 +- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/e2e/features/agent-v2/configure-persistence.feature b/e2e/features/agent-v2/configure-persistence.feature index 0dc4edf5d91fb8..7b525225bad2fa 100644 --- a/e2e/features/agent-v2/configure-persistence.feature +++ b/e2e/features/agent-v2/configure-persistence.feature @@ -1,10 +1,12 @@ @agent-v2 @authenticated @core Feature: Agent v2 configure persistence + @configure-persistence Scenario: Persisted Agent v2 instructions remain visible after refresh Given I am signed in as the default E2E admin And an Agent v2 test agent has been created via API - And the Agent v2 composer draft uses the normal E2E prompt When I open the Agent v2 configure page - Then I should see the normal E2E prompt in the Agent v2 prompt editor + And I fill the Agent v2 prompt editor with the normal E2E prompt + Then the normal Agent v2 draft should use the normal E2E prompt + And the Agent v2 configuration should be saved automatically When I refresh the current page Then I should see the normal E2E prompt in the Agent v2 prompt editor diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 57e06d16c1609e..bcdfae90ebd5c0 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -105,6 +105,13 @@ const expectAgentConfigFileSaved = async ( .toContain(fileName) } +const expectNormalAgentPromptDraft = async (world: DifyWorld) => { + await expect.poll( + async () => (await getAgentComposerDraft(getCurrentAgentId(world))).agent_soul?.prompt, + { timeout: 30_000 }, + ).toEqual({ system_prompt: normalAgentPrompt }) +} + Given('an Agent v2 test agent has been created via API', async function (this: DifyWorld) { const agent = await createTestAgent() this.createdAgentIds.push(agent.id) @@ -181,6 +188,14 @@ When('I open the Agent v2 configure page from the Agent Roster', async function await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) }) +When('I fill the Agent v2 prompt editor with the normal E2E prompt', async function (this: DifyWorld) { + const page = this.getPage() + const promptSection = page.getByRole('region', { name: 'Prompt' }) + + await expect(promptSection).toBeVisible({ timeout: 30_000 }) + await promptSection.getByRole('textbox', { name: 'Prompt' }).fill(normalAgentPrompt) +}) + When('I discard the Agent v2 Build draft', async function (this: DifyWorld) { await this.getPage().getByRole('button', { name: 'Discard' }).click() }) @@ -637,10 +652,14 @@ Then('I should see the Agent v2 Build draft pending changes', async function (th Then( 'the normal Agent v2 draft should still use the normal E2E prompt', async function (this: DifyWorld) { - await expect.poll( - async () => (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul?.prompt, - { timeout: 30_000 }, - ).toEqual({ system_prompt: normalAgentPrompt }) + await expectNormalAgentPromptDraft(this) + }, +) + +Then( + 'the normal Agent v2 draft should use the normal E2E prompt', + async function (this: DifyWorld) { + await expectNormalAgentPromptDraft(this) }, ) diff --git a/e2e/support/agent-configure.ts b/e2e/support/agent-configure.ts index f04f429081820a..9adf51a59eae94 100644 --- a/e2e/support/agent-configure.ts +++ b/e2e/support/agent-configure.ts @@ -3,6 +3,6 @@ import { expect } from '@playwright/test' export async function waitForAgentConfigureAutosaved(page: Page) { await expect( - page.getByText(/^Saved(?:\s|$)/).filter({ visible: true }).first(), + page.getByRole('status', { name: /Saved/i }).first(), ).toBeVisible({ timeout: 30_000 }) } From f5b42f5d7041fb036437c12e1ff80d763a56b5f3 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 17:20:09 +0800 Subject: [PATCH 053/185] test(e2e): require runnable agent for publish --- e2e/features/agent-v2/publish.feature | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/features/agent-v2/publish.feature b/e2e/features/agent-v2/publish.feature index 78bd2218356a15..c3a8fc3f7b794a 100644 --- a/e2e/features/agent-v2/publish.feature +++ b/e2e/features/agent-v2/publish.feature @@ -2,8 +2,8 @@ Feature: Agent v2 publish Scenario: Publish a configured Agent v2 draft Given I am signed in as the default E2E admin - And an Agent v2 test agent has been created via API - And the Agent v2 composer draft uses the normal E2E prompt + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API When I open the Agent v2 configure page And I publish the Agent v2 draft Then the Agent v2 draft should be published and up to date From 4b7fb6000887768e984327ad9a7d2df18dd8d0fb Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 17:30:13 +0800 Subject: [PATCH 054/185] test(e2e): cover agent tool states display --- e2e/features/agent-v2/agent-edit.feature | 8 ++ .../agent-v2/configure.steps.ts | 78 ++++++++++++++++++- e2e/support/preflight.ts | 13 +--- 3 files changed, 89 insertions(+), 10 deletions(-) create mode 100644 e2e/features/agent-v2/agent-edit.feature diff --git a/e2e/features/agent-v2/agent-edit.feature b/e2e/features/agent-v2/agent-edit.feature new file mode 100644 index 00000000000000..52508eb008ef53 --- /dev/null +++ b/e2e/features/agent-v2/agent-edit.feature @@ -0,0 +1,8 @@ +@agent-v2 @authenticated @agent-edit @core +Feature: Agent v2 Agent Edit page + Scenario: Tool states are visible on the Agent Edit page + Given I am signed in as the default E2E admin + And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" is available + And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" includes the tool state fixture configuration + When I open the preseeded Agent v2 configure page for "E2E New Agent Builder Tool States" from the Agent Roster + Then I should see the Agent v2 tool state fixture tools diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index bcdfae90ebd5c0..cc2f8e5dad6ea1 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -16,7 +16,10 @@ import { updatedAgentPrompt, updatedAgentSoulConfig, } from '../../../support/agent' -import { agentBuilderFixedInputs } from '../../../support/agent-builder-resources' +import { + agentBuilderFixedInputs, + agentBuilderPreseededResources, +} from '../../../support/agent-builder-resources' import { waitForAgentConfigureAutosaved } from '../../../support/agent-configure' import { agentBuilderTestMaterials, @@ -31,6 +34,25 @@ const getCurrentAgentId = (world: DifyWorld) => { return agentId } +const getPreseededAgent = (world: DifyWorld, name: string) => { + const resource = world.agentBuilderPreseededResources[name] + if (!resource || resource.kind !== 'agent') { + throw new Error( + `Preseeded Agent "${name}" is not available. Run the matching preflight step first.`, + ) + } + + return resource +} + +const getPreseededToolDisplayParts = (displayName: string) => { + const [providerName, actionName] = displayName.split(' / ') + if (!providerName || !actionName) + throw new Error(`Preseeded tool display name must use "Provider / Action": ${displayName}`) + + return { actionName, providerName } +} + const getEnvVariableKey = (variable: AgentComposerEnvVariable) => variable.key ?? variable.name ?? variable.variable @@ -188,6 +210,19 @@ When('I open the Agent v2 configure page from the Agent Roster', async function await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) }) +When( + 'I open the preseeded Agent v2 configure page for {string} from the Agent Roster', + async function (this: DifyWorld, agentName: string) { + const page = this.getPage() + const agent = getPreseededAgent(this, agentName) + + await page.goto('/roster') + await page.getByRole('link', { name: agentName }).click() + await expect(page).toHaveURL(new RegExp(`/roster/agent/${agent.id}/configure(?:\\?.*)?$`)) + await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) + }, +) + When('I fill the Agent v2 prompt editor with the normal E2E prompt', async function (this: DifyWorld) { const page = this.getPage() const promptSection = page.getByRole('region', { name: 'Prompt' }) @@ -338,6 +373,47 @@ Then('I should see the Agent v2 configure workspace', async function (this: Dify await expect(page.getByText(this.lastCreatedAgentName!)).toBeVisible() }) +Then('I should see the Agent v2 tool state fixture tools', async function (this: DifyWorld) { + const page = this.getPage() + const toolsSection = page.getByRole('region', { name: 'Tools' }) + const jsonTool = getPreseededToolDisplayParts(agentBuilderPreseededResources.jsonReplaceTool) + const tavilyTool = getPreseededToolDisplayParts(agentBuilderPreseededResources.tavilySearchTool) + + await expect(toolsSection).toBeVisible({ timeout: 30_000 }) + await expect(toolsSection.getByRole('button', { exact: true, name: 'Not authorized' })).toBeVisible() + + const jsonProcessProvider = toolsSection.getByRole('button', { + exact: true, + name: jsonTool.providerName, + }) + await expect(jsonProcessProvider).toBeVisible() + + const jsonReplaceAction = toolsSection.getByText(jsonTool.actionName, { exact: true }) + if (!await jsonReplaceAction.isVisible()) + await jsonProcessProvider.click() + await expect(jsonReplaceAction).toBeVisible() + await jsonReplaceAction.hover() + await expect(toolsSection.getByRole('button', { + exact: true, + name: `Edit ${jsonTool.actionName}`, + })).toBeVisible() + await expect(toolsSection.getByRole('button', { + exact: true, + name: `Remove ${jsonTool.actionName}`, + })).toBeVisible() + + const tavilyProvider = toolsSection.getByRole('button', { + exact: true, + name: tavilyTool.providerName, + }) + await expect(tavilyProvider).toBeVisible() + + const tavilySearchAction = toolsSection.getByText(tavilyTool.actionName, { exact: true }) + if (!await tavilySearchAction.isVisible()) + await tavilyProvider.click() + await expect(tavilySearchAction).toBeVisible() +}) + Then( 'I should see the normal E2E prompt in the Agent v2 prompt editor', async function (this: DifyWorld) { diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index 8d99a15b194417..e0924dd5d06810 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -270,15 +270,10 @@ const hasToolEntry = ( }, ) => Boolean(findToolEntry(items, tool)) -const hasToolCredentialReference = (item: unknown) => { +const hasUnauthorizedToolCredentialState = (item: unknown) => { const record = asRecord(item) - const credentialRef = asRecord(record.credential_ref) - const credentialType = asString(record.credential_type) - return ( - (credentialType === 'api-key' || credentialType === 'oauth2') - && (Boolean(asString(credentialRef.id)) || Boolean(asString(record.credential_id))) - ) + return asString(record.credential_type) === 'unauthorized' } const hasKnowledgeDataset = ( @@ -726,8 +721,8 @@ export async function skipMissingPreseededToolStatesAgentConfiguration( if (!tavilyEntry) { missing.push(agentBuilderPreseededResources.tavilySearchTool) } - else if (!hasToolCredentialReference(tavilyEntry)) { - missing.push(`${agentBuilderPreseededResources.tavilySearchTool} credential reference`) + else if (!hasUnauthorizedToolCredentialState(tavilyEntry)) { + missing.push(`${agentBuilderPreseededResources.tavilySearchTool} unauthorized credential state`) } if (missing.length > 0) { From be6bf180a9fede81e0232cadb06b25621278eb38 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 17:38:18 +0800 Subject: [PATCH 055/185] test(e2e): cover agent edit saved configuration --- e2e/features/agent-v2/agent-edit.feature | 8 ++ .../agent-v2/configure.steps.ts | 102 ++++++++++++++---- 2 files changed, 88 insertions(+), 22 deletions(-) diff --git a/e2e/features/agent-v2/agent-edit.feature b/e2e/features/agent-v2/agent-edit.feature index 52508eb008ef53..5f15c76f9c6197 100644 --- a/e2e/features/agent-v2/agent-edit.feature +++ b/e2e/features/agent-v2/agent-edit.feature @@ -1,5 +1,13 @@ @agent-v2 @authenticated @agent-edit @core Feature: Agent v2 Agent Edit page + Scenario: Saved orchestration sections are visible on the Agent Edit page + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" is available + And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" includes the core fixture configuration + When I open the preseeded Agent v2 configure page for "E2E New Agent Builder Full Config" from the Agent Roster + Then I should see the Agent v2 full-config fixture sections + Scenario: Tool states are visible on the Agent Edit page Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" is available diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index cc2f8e5dad6ea1..144d691fb56dd6 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -1,3 +1,4 @@ +import type { Locator } from '@playwright/test' import type { AgentComposerEnvVariable } from '../../../support/agent' import type { DifyWorld } from '../../support/world' import { Given, Then, When } from '@cucumber/cucumber' @@ -17,6 +18,7 @@ import { updatedAgentSoulConfig, } from '../../../support/agent' import { + agentBuilderExpectedTokens, agentBuilderFixedInputs, agentBuilderPreseededResources, } from '../../../support/agent-builder-resources' @@ -134,6 +136,25 @@ const expectNormalAgentPromptDraft = async (world: DifyWorld) => { ).toEqual({ system_prompt: normalAgentPrompt }) } +const expectProviderToolActionVisible = async ( + toolsSection: Locator, + displayName: string, +) => { + const tool = getPreseededToolDisplayParts(displayName) + const provider = toolsSection.getByRole('button', { + exact: true, + name: tool.providerName, + }) + await expect(provider).toBeVisible() + + const action = toolsSection.getByText(tool.actionName, { exact: true }) + if (!await action.isVisible()) + await provider.click() + await expect(action).toBeVisible() + + return { action, tool } +} + Given('an Agent v2 test agent has been created via API', async function (this: DifyWorld) { const agent = await createTestAgent() this.createdAgentIds.push(agent.id) @@ -373,25 +394,68 @@ Then('I should see the Agent v2 configure workspace', async function (this: Dify await expect(page.getByText(this.lastCreatedAgentName!)).toBeVisible() }) +Then('I should see the Agent v2 full-config fixture sections', async function (this: DifyWorld) { + const page = this.getPage() + const stableModel = this.agentBuilderStableChatModel + if (!stableModel) + throw new Error('Stable chat model preflight must run before asserting the full-config Agent.') + + await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) + await expect(page.getByText(agentBuilderPreseededResources.fullConfigAgent, { exact: true })) + .toBeVisible() + await expect(page.getByText(stableModel.name, { exact: true })).toBeVisible() + + const promptSection = page.getByRole('region', { name: 'Prompt' }) + await expect(promptSection).toBeVisible() + await expect(promptSection).toContainText(agentBuilderExpectedTokens.agentReply) + + const skillsSection = page.getByRole('region', { name: 'Skills' }) + await expect(skillsSection).toBeVisible() + await expect(skillsSection.getByRole('button', { + exact: true, + name: agentBuilderPreseededResources.summarySkill, + })).toBeVisible() + + const filesSection = page.getByRole('region', { name: 'Files' }) + await expect(filesSection).toBeVisible() + await expect(filesSection.getByRole('button', { + exact: true, + name: agentBuilderTestMaterials.smallFile, + })).toBeVisible() + await expect(filesSection.getByRole('button', { + exact: true, + name: agentBuilderTestMaterials.specialFilename, + })).toBeVisible() + + const toolsSection = page.getByRole('region', { name: 'Tools' }) + await expect(toolsSection).toBeVisible() + await expectProviderToolActionVisible( + toolsSection, + agentBuilderPreseededResources.jsonReplaceTool, + ) + + const knowledgeSection = page.getByRole('region', { name: 'Knowledge Retrieval' }) + await expect(knowledgeSection).toBeVisible() + await expect(knowledgeSection.getByText('Retrieval 1', { exact: true })).toBeVisible() + + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + await expect(advancedSettings).toBeVisible() + await expect( + advancedSettings.getByText('For power users. Env vars, sandbox & memory.'), + ).toBeVisible() +}) + Then('I should see the Agent v2 tool state fixture tools', async function (this: DifyWorld) { const page = this.getPage() const toolsSection = page.getByRole('region', { name: 'Tools' }) - const jsonTool = getPreseededToolDisplayParts(agentBuilderPreseededResources.jsonReplaceTool) - const tavilyTool = getPreseededToolDisplayParts(agentBuilderPreseededResources.tavilySearchTool) await expect(toolsSection).toBeVisible({ timeout: 30_000 }) await expect(toolsSection.getByRole('button', { exact: true, name: 'Not authorized' })).toBeVisible() - const jsonProcessProvider = toolsSection.getByRole('button', { - exact: true, - name: jsonTool.providerName, - }) - await expect(jsonProcessProvider).toBeVisible() - - const jsonReplaceAction = toolsSection.getByText(jsonTool.actionName, { exact: true }) - if (!await jsonReplaceAction.isVisible()) - await jsonProcessProvider.click() - await expect(jsonReplaceAction).toBeVisible() + const { action: jsonReplaceAction, tool: jsonTool } = await expectProviderToolActionVisible( + toolsSection, + agentBuilderPreseededResources.jsonReplaceTool, + ) await jsonReplaceAction.hover() await expect(toolsSection.getByRole('button', { exact: true, @@ -402,16 +466,10 @@ Then('I should see the Agent v2 tool state fixture tools', async function (this: name: `Remove ${jsonTool.actionName}`, })).toBeVisible() - const tavilyProvider = toolsSection.getByRole('button', { - exact: true, - name: tavilyTool.providerName, - }) - await expect(tavilyProvider).toBeVisible() - - const tavilySearchAction = toolsSection.getByText(tavilyTool.actionName, { exact: true }) - if (!await tavilySearchAction.isVisible()) - await tavilyProvider.click() - await expect(tavilySearchAction).toBeVisible() + await expectProviderToolActionVisible( + toolsSection, + agentBuilderPreseededResources.tavilySearchTool, + ) }) Then( From c98c51dddd7786077b01ccb6ee63ee732bd93bd8 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 17:56:36 +0800 Subject: [PATCH 056/185] test(e2e): cover agent build draft generation --- e2e/features/agent-v2/build-draft.feature | 12 ++ e2e/features/agent-v2/preflight.feature | 4 +- .../agent-v2/configure.steps.ts | 48 +++++- .../agent-build-instruction.txt | 2 +- .../test-materials/e2e-summary-skill/SKILL.md | 10 ++ .../test-materials/e2e-summary.SKILL.md | 5 - e2e/support/agent-builder-resources.ts | 2 +- e2e/support/agent.ts | 147 ++++++++++++++++++ e2e/support/test-materials.ts | 2 +- 9 files changed, 220 insertions(+), 12 deletions(-) create mode 100644 e2e/fixtures/test-materials/e2e-summary-skill/SKILL.md delete mode 100644 e2e/fixtures/test-materials/e2e-summary.SKILL.md diff --git a/e2e/features/agent-v2/build-draft.feature b/e2e/features/agent-v2/build-draft.feature index 6af3ca50151222..671cc8abeb0237 100644 --- a/e2e/features/agent-v2/build-draft.feature +++ b/e2e/features/agent-v2/build-draft.feature @@ -1,5 +1,17 @@ @agent-v2 @authenticated @build @core Feature: Agent v2 build draft + Scenario: Generating a Build draft leaves the normal Agent configuration unchanged + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And the Agent Builder preseeded tool "JSON Process / JSON Replace" is available + And a runnable Agent v2 test agent has been created via API + And the e2e-summary-skill Skill is available to the Agent v2 test agent + When I open the Agent v2 configure page + And I generate an Agent v2 Build draft from the fixed instruction + Then I should see the Agent v2 Build draft pending changes + And I should see the Agent v2 Build mode confirmation state + And the normal Agent v2 draft should still use the normal E2E prompt + Scenario: Discarding a Build draft keeps the original Agent configuration Given I am signed in as the default E2E admin And an Agent v2 test agent has been created via API diff --git a/e2e/features/agent-v2/preflight.feature b/e2e/features/agent-v2/preflight.feature index 76e2b539d5ba34..05f2a27de02a98 100644 --- a/e2e/features/agent-v2/preflight.feature +++ b/e2e/features/agent-v2/preflight.feature @@ -37,9 +37,9 @@ Feature: Agent Builder preseeded environment Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" is available - Scenario: Full config Agent includes the Summary Skill + Scenario: Full config Agent includes the summary Skill Given I am signed in as the default E2E admin - And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" includes drive skill "E2E Summary Skill" + And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" includes drive skill "e2e-summary-skill" Scenario: Full config Agent includes core fixture configuration Given I am signed in as the default E2E admin diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 144d691fb56dd6..b22b23971e7c50 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -1,6 +1,7 @@ import type { Locator } from '@playwright/test' import type { AgentComposerEnvVariable } from '../../../support/agent' import type { DifyWorld } from '../../support/world' +import { readFile } from 'node:fs/promises' import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { @@ -16,6 +17,7 @@ import { saveAgentComposerDraft, updatedAgentPrompt, updatedAgentSoulConfig, + uploadAgentDriveSkill, } from '../../../support/agent' import { agentBuilderExpectedTokens, @@ -197,6 +199,14 @@ Given('the Agent v2 composer draft uses the normal E2E prompt', async function ( await saveAgentComposerDraft(getCurrentAgentId(this), normalAgentSoulConfig) }) +Given('the e2e-summary-skill Skill is available to the Agent v2 test agent', async function (this: DifyWorld) { + await uploadAgentDriveSkill({ + agentId: getCurrentAgentId(this), + fileName: agentBuilderTestMaterials.summarySkill, + filePath: getAgentBuilderTestMaterialPath('summarySkill'), + }) +}) + Given('an Agent v2 Build draft uses the updated E2E prompt', async function (this: DifyWorld) { await saveAgentBuildDraft(getCurrentAgentId(this), updatedAgentSoulConfig) }) @@ -218,6 +228,31 @@ When('I open the Agent v2 configure page', async function (this: DifyWorld) { await this.getPage().goto(getAgentConfigurePath(getCurrentAgentId(this))) }) +When('I generate an Agent v2 Build draft from the fixed instruction', async function (this: DifyWorld) { + const page = this.getPage() + const agentId = getCurrentAgentId(this) + const instruction = (await readFile(getAgentBuilderTestMaterialPath('buildInstruction'), 'utf8')).trim() + + await page.getByRole('button', { name: 'Build' }).click() + await page.getByPlaceholder('Describe what your agent should do').fill(instruction) + + const checkoutResponsePromise = page.waitForResponse(response => ( + response.request().method() === 'POST' + && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/build-draft/checkout`) + )) + const chatResponsePromise = page.waitForResponse(response => ( + response.request().method() === 'POST' + && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/chat-messages`) + )) + + await page.getByRole('button', { name: 'Start build' }).click() + expect((await checkoutResponsePromise).ok()).toBe(true) + expect((await chatResponsePromise).ok()).toBe(true) + await expect(page.getByText('Build draft')).toBeVisible({ timeout: 120_000 }) + await expect(page.getByRole('button', { name: 'Apply' })).toBeEnabled({ timeout: 120_000 }) + await expect(page.getByRole('button', { name: 'Discard' })).toBeEnabled() +}) + When('I open the Agent v2 configure page from the Agent Roster', async function (this: DifyWorld) { const page = this.getPage() const agentId = getCurrentAgentId(this) @@ -779,8 +814,17 @@ Then('I should see the Agent v2 Build draft pending changes', async function (th const page = this.getPage() await expect(page.getByText('Build draft')).toBeVisible({ timeout: 30_000 }) - await expect(page.getByRole('button', { name: 'Apply' })).toBeVisible() - await expect(page.getByRole('button', { name: 'Discard' })).toBeVisible() + await expect(page.getByRole('button', { name: 'Apply' })).toBeEnabled() + await expect(page.getByRole('button', { name: 'Discard' })).toBeEnabled() +}) + +Then('I should see the Agent v2 Build mode confirmation state', async function (this: DifyWorld) { + const page = this.getPage() + + await expect(page.getByText('Build mode', { exact: true })).toBeVisible() + await expect( + page.getByText('You\'re in build mode. Shape this setup through the chat on the right, then Apply.'), + ).toBeVisible() }) Then( diff --git a/e2e/fixtures/test-materials/agent-build-instruction.txt b/e2e/fixtures/test-materials/agent-build-instruction.txt index 45a36462d2f555..d5cf5f83e5e97f 100644 --- a/e2e/fixtures/test-materials/agent-build-instruction.txt +++ b/e2e/fixtures/test-materials/agent-build-instruction.txt @@ -1,3 +1,3 @@ -Ask the Agent to use E2E Summary Skill to summarize user input. +Ask the Agent to use e2e-summary-skill to summarize user input. When replacing JSON strings, use JSON Process / JSON Replace. After applying, include these capabilities in the Agent instructions. diff --git a/e2e/fixtures/test-materials/e2e-summary-skill/SKILL.md b/e2e/fixtures/test-materials/e2e-summary-skill/SKILL.md new file mode 100644 index 00000000000000..f8a8fd91f3c3b5 --- /dev/null +++ b/e2e/fixtures/test-materials/e2e-summary-skill/SKILL.md @@ -0,0 +1,10 @@ +--- +name: e2e-summary-skill +description: Summarize user input for Agent Builder E2E coverage. +--- + +# e2e-summary-skill + +Summarize the user input in one concise paragraph. + +The summary must include E2E_SUMMARY_SKILL_PASS. diff --git a/e2e/fixtures/test-materials/e2e-summary.SKILL.md b/e2e/fixtures/test-materials/e2e-summary.SKILL.md deleted file mode 100644 index 948f6bb143b491..00000000000000 --- a/e2e/fixtures/test-materials/e2e-summary.SKILL.md +++ /dev/null @@ -1,5 +0,0 @@ -# E2E Summary Skill - -Summarize the user input in one concise paragraph. - -The summary must include E2E_SUMMARY_SKILL_PASS. diff --git a/e2e/support/agent-builder-resources.ts b/e2e/support/agent-builder-resources.ts index 16cff4ba563b57..3c87fdc094fa11 100644 --- a/e2e/support/agent-builder-resources.ts +++ b/e2e/support/agent-builder-resources.ts @@ -1,6 +1,6 @@ export const agentBuilderPreseededResources = { stableChatModel: 'E2E Stable Chat Model', - summarySkill: 'E2E Summary Skill', + summarySkill: 'e2e-summary-skill', jsonReplaceTool: 'JSON Process / JSON Replace', tavilySearchTool: 'Tavily / Tavily Search', agentKnowledgeBase: 'E2E Agent Knowledge Base', diff --git a/e2e/support/agent.ts b/e2e/support/agent.ts index 49af5dc0d23b1c..89d5ff1cbcd8ae 100644 --- a/e2e/support/agent.ts +++ b/e2e/support/agent.ts @@ -1,3 +1,6 @@ +import { Buffer } from 'node:buffer' +import { readFile } from 'node:fs/promises' +import path from 'node:path' import { createApiContext, expectApiResponseOK, setAppSiteEnabled } from './api' import { assertE2EResourceName, createE2EResourceName } from './naming' @@ -67,6 +70,16 @@ export type AgentApiKey = { token?: string } +export type AgentDriveSkillUpload = { + skill: { + archive_key?: string | null + description: string + name: string + path: string + skill_md_key: string + } +} + export type CreateTestAgentOptions = { description?: string name?: string @@ -118,6 +131,111 @@ const getExistingModelConfig = (agentSoul: AgentSoulConfig) => { return {} } +const crc32Table = new Uint32Array(256) +for (let i = 0; i < crc32Table.length; i++) { + let c = i + for (let k = 0; k < 8; k++) + c = c & 1 ? 0xEDB88320 ^ (c >>> 1) : c >>> 1 + crc32Table[i] = c >>> 0 +} + +const crc32 = (buffer: Buffer) => { + let crc = 0xFFFFFFFF + for (const byte of buffer) + crc = crc32Table[(crc ^ byte) & 0xFF]! ^ (crc >>> 8) + return (crc ^ 0xFFFFFFFF) >>> 0 +} + +const createSingleFileZip = ({ + content, + entryName, +}: { + content: Buffer + entryName: string +}) => { + const entryNameBuffer = Buffer.from(entryName) + const checksum = crc32(content) + const localHeader = Buffer.alloc(30) + localHeader.writeUInt32LE(0x04034B50, 0) + localHeader.writeUInt16LE(20, 4) + localHeader.writeUInt16LE(0, 6) + localHeader.writeUInt16LE(0, 8) + localHeader.writeUInt16LE(0, 10) + localHeader.writeUInt16LE(0, 12) + localHeader.writeUInt32LE(checksum, 14) + localHeader.writeUInt32LE(content.length, 18) + localHeader.writeUInt32LE(content.length, 22) + localHeader.writeUInt16LE(entryNameBuffer.length, 26) + localHeader.writeUInt16LE(0, 28) + + const centralDirectoryOffset = localHeader.length + entryNameBuffer.length + content.length + const centralDirectoryHeader = Buffer.alloc(46) + centralDirectoryHeader.writeUInt32LE(0x02014B50, 0) + centralDirectoryHeader.writeUInt16LE(20, 4) + centralDirectoryHeader.writeUInt16LE(20, 6) + centralDirectoryHeader.writeUInt16LE(0, 8) + centralDirectoryHeader.writeUInt16LE(0, 10) + centralDirectoryHeader.writeUInt16LE(0, 12) + centralDirectoryHeader.writeUInt16LE(0, 14) + centralDirectoryHeader.writeUInt32LE(checksum, 16) + centralDirectoryHeader.writeUInt32LE(content.length, 20) + centralDirectoryHeader.writeUInt32LE(content.length, 24) + centralDirectoryHeader.writeUInt16LE(entryNameBuffer.length, 28) + centralDirectoryHeader.writeUInt16LE(0, 30) + centralDirectoryHeader.writeUInt16LE(0, 32) + centralDirectoryHeader.writeUInt16LE(0, 34) + centralDirectoryHeader.writeUInt16LE(0, 36) + centralDirectoryHeader.writeUInt32LE(0, 38) + centralDirectoryHeader.writeUInt32LE(0, 42) + + const centralDirectorySize = centralDirectoryHeader.length + entryNameBuffer.length + const endOfCentralDirectory = Buffer.alloc(22) + endOfCentralDirectory.writeUInt32LE(0x06054B50, 0) + endOfCentralDirectory.writeUInt16LE(0, 4) + endOfCentralDirectory.writeUInt16LE(0, 6) + endOfCentralDirectory.writeUInt16LE(1, 8) + endOfCentralDirectory.writeUInt16LE(1, 10) + endOfCentralDirectory.writeUInt32LE(centralDirectorySize, 12) + endOfCentralDirectory.writeUInt32LE(centralDirectoryOffset, 16) + endOfCentralDirectory.writeUInt16LE(0, 20) + + return Buffer.concat([ + localHeader, + entryNameBuffer, + content, + centralDirectoryHeader, + entryNameBuffer, + endOfCentralDirectory, + ]) +} + +const toSkillArchiveUpload = async ({ + fileName, + filePath, +}: { + fileName: string + filePath: string +}) => { + if (fileName.endsWith('.zip') || fileName.endsWith('.skill')) { + return { + buffer: await readFile(filePath), + name: path.basename(fileName), + } + } + const sourceDirName = path.basename(path.dirname(fileName)) + const archiveBaseName = sourceDirName && sourceDirName !== '.' + ? sourceDirName + : path.basename(fileName, path.extname(fileName)) + + return { + buffer: createSingleFileZip({ + content: await readFile(filePath), + entryName: 'SKILL.md', + }), + name: `${archiveBaseName}.skill`, + } +} + export function createAgentSoulConfigWithModel( agentSoul: AgentSoulConfig, model: AgentModelSelection, @@ -219,6 +337,35 @@ export async function saveAgentComposerDraft( } } +export async function uploadAgentDriveSkill({ + agentId, + fileName, + filePath, +}: { + agentId: string + fileName: string + filePath: string +}): Promise { + const ctx = await createApiContext() + try { + const upload = await toSkillArchiveUpload({ fileName, filePath }) + const response = await ctx.post(`/console/api/agent/${agentId}/skills/upload`, { + multipart: { + file: { + buffer: upload.buffer, + mimeType: 'application/zip', + name: upload.name, + }, + }, + }) + await expectApiResponseOK(response, `Upload Agent v2 drive skill ${fileName} for ${agentId}`) + return (await response.json()) as AgentDriveSkillUpload + } + finally { + await ctx.dispose() + } +} + export async function getAgentComposerDraft(agentId: string): Promise { const ctx = await createApiContext() try { diff --git a/e2e/support/test-materials.ts b/e2e/support/test-materials.ts index 76524447a8e405..49f87a2eadf76b 100644 --- a/e2e/support/test-materials.ts +++ b/e2e/support/test-materials.ts @@ -21,7 +21,7 @@ export const agentBuilderTestMaterials = { validEnv: 'agent-valid.env', invalidEnv: 'agent-invalid.env', buildInstruction: 'agent-build-instruction.txt', - summarySkill: 'e2e-summary.SKILL.md', + summarySkill: 'e2e-summary-skill/SKILL.md', fileTreeFixture: 'file_tree_fixture', countBatch5: 'count_batch_5_valid_files', countBatch6: 'count_batch_6_valid_files', From 556f55e4309209c87d992830dcec948728bc5d74 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 18:07:24 +0800 Subject: [PATCH 057/185] test(e2e): verify agent skill fixture upload --- e2e/features/agent-v2/preflight.feature | 7 ++++++ .../agent-v2/configure.steps.ts | 7 ++++++ e2e/support/agent.ts | 23 +++++++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/e2e/features/agent-v2/preflight.feature b/e2e/features/agent-v2/preflight.feature index 05f2a27de02a98..3472f9c2688f61 100644 --- a/e2e/features/agent-v2/preflight.feature +++ b/e2e/features/agent-v2/preflight.feature @@ -25,6 +25,13 @@ Feature: Agent Builder preseeded environment Given I am signed in as the default E2E admin And the Agent Builder preseeded tool "Tavily / Tavily Search" is available + @skill-fixture + Scenario: Summary Skill package fixture uploads to Agent drive + Given I am signed in as the default E2E admin + And an Agent v2 test agent has been created via API + And the e2e-summary-skill Skill is available to the Agent v2 test agent + Then the Agent v2 test agent should include drive skill "e2e-summary-skill" + Scenario: Agent knowledge base is available Given I am signed in as the default E2E admin And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index b22b23971e7c50..7be3e4f0795785 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -10,6 +10,7 @@ import { createTestAgent, getAgentComposerDraft, getAgentConfigurePath, + getAgentDriveSkills, getTestAgent, normalAgentPrompt, normalAgentSoulConfig, @@ -207,6 +208,12 @@ Given('the e2e-summary-skill Skill is available to the Agent v2 test agent', asy }) }) +Then('the Agent v2 test agent should include drive skill {string}', async function (this: DifyWorld, skillName: string) { + const skills = await getAgentDriveSkills(getCurrentAgentId(this)) + + expect(skills.map(skill => skill.name)).toContain(skillName) +}) + Given('an Agent v2 Build draft uses the updated E2E prompt', async function (this: DifyWorld) { await saveAgentBuildDraft(getCurrentAgentId(this), updatedAgentSoulConfig) }) diff --git a/e2e/support/agent.ts b/e2e/support/agent.ts index 89d5ff1cbcd8ae..4467c5fb972072 100644 --- a/e2e/support/agent.ts +++ b/e2e/support/agent.ts @@ -80,6 +80,16 @@ export type AgentDriveSkillUpload = { } } +export type AgentDriveSkill = { + description?: string | null + name: string + path: string +} + +export type AgentDriveSkillListResponse = { + items: AgentDriveSkill[] +} + export type CreateTestAgentOptions = { description?: string name?: string @@ -366,6 +376,19 @@ export async function uploadAgentDriveSkill({ } } +export async function getAgentDriveSkills(agentId: string): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agentId}/drive/skills`) + await expectApiResponseOK(response, `Get Agent v2 drive skills for ${agentId}`) + const body = (await response.json()) as AgentDriveSkillListResponse + return body.items + } + finally { + await ctx.dispose() + } +} + export async function getAgentComposerDraft(agentId: string): Promise { const ctx = await createApiContext() try { From f4e1b32ea1259523d18b1c384062113c961295b2 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 18:16:41 +0800 Subject: [PATCH 058/185] test(e2e): cover agent workflow access references --- e2e/features/agent-v2/access-point.feature | 11 ++ .../agent-v2/access-point.steps.ts | 101 +++++++++++++++++- e2e/features/support/world.ts | 2 + e2e/support/agent.ts | 26 +++++ 4 files changed, 139 insertions(+), 1 deletion(-) diff --git a/e2e/features/agent-v2/access-point.feature b/e2e/features/agent-v2/access-point.feature index 62fcc076a1e3e1..27d590471fdceb 100644 --- a/e2e/features/agent-v2/access-point.feature +++ b/e2e/features/agent-v2/access-point.feature @@ -6,6 +6,17 @@ Feature: Agent v2 Access Point When I open the Agent v2 Access Point page Then I should see the Agent v2 Access Point overview + @workflow-reference + Scenario: Workflow access shows the referencing workflow + Given I am signed in as the default E2E admin + And the Agent Builder preseeded Agent "E2E Agent With Workflow Reference" is available + And the Agent Builder preseeded workflow "E2E Agent Reference Workflow" is available + And the Agent Builder preseeded Agent "E2E Agent With Workflow Reference" is referenced by workflow "E2E Agent Reference Workflow" + When I open the preseeded Agent v2 Access Point page for "E2E Agent With Workflow Reference" from the Agent Roster + Then I should see the Agent v2 Workflow access reference for "E2E Agent Reference Workflow" + When I open the Agent v2 Workflow access reference for "E2E Agent Reference Workflow" + Then the Agent v2 Workflow access reference for "E2E Agent Reference Workflow" should open in Studio + Scenario: Backend service API supports endpoint copy, key creation, and API reference navigation Given I am signed in as the default E2E admin And an Agent v2 test agent has been created via API diff --git a/e2e/features/step-definitions/agent-v2/access-point.steps.ts b/e2e/features/step-definitions/agent-v2/access-point.steps.ts index 5f1df3c3de0323..f9bf56bc861868 100644 --- a/e2e/features/step-definitions/agent-v2/access-point.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point.steps.ts @@ -1,7 +1,12 @@ import type { DifyWorld } from '../../support/world' import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' -import { getAgentAccessPath, setAgentApiAccess } from '../../../support/agent' +import { + getAgentAccessPath, + getAgentReferencingWorkflows, + setAgentApiAccess, +} from '../../../support/agent' +import { agentBuilderPreseededResources } from '../../../support/agent-builder-resources' const getCurrentAgentId = (world: DifyWorld) => { const agentId = world.createdAgentIds.at(-1) @@ -11,6 +16,17 @@ const getCurrentAgentId = (world: DifyWorld) => { return agentId } +const getPreseededResource = (world: DifyWorld, name: string, kind: 'agent' | 'workflow') => { + const resource = world.agentBuilderPreseededResources[name] + if (!resource || resource.kind !== kind) { + throw new Error( + `Preseeded ${kind} "${name}" is not available. Run the matching preflight step first.`, + ) + } + + return resource +} + Given( 'Agent v2 Backend service API access has been enabled via API', async function (this: DifyWorld) { @@ -24,6 +40,23 @@ When('I open the Agent v2 Access Point page', async function (this: DifyWorld) { await this.getPage().goto(getAgentAccessPath(getCurrentAgentId(this))) }) +When( + 'I open the preseeded Agent v2 Access Point page for {string} from the Agent Roster', + async function (this: DifyWorld, agentName: string) { + const page = this.getPage() + const agent = getPreseededResource(this, agentName, 'agent') + + await page.goto('/roster') + await page.getByRole('link', { name: agentName }).click() + await expect(page).toHaveURL(new RegExp(`/roster/agent/${agent.id}/configure(?:\\?.*)?$`)) + await page.getByRole('link', { name: 'Access Point' }).click() + await expect(page).toHaveURL(new RegExp(`/roster/agent/${agent.id}/access(?:\\?.*)?$`)) + await expect(page.getByRole('region', { name: 'Access Point' })).toBeVisible({ + timeout: 30_000, + }) + }, +) + When('I switch to the Agent v2 Access Point section', async function (this: DifyWorld) { const page = this.getPage() const agentId = getCurrentAgentId(this) @@ -63,6 +96,72 @@ Then('I should see the Agent v2 Access Point overview', async function (this: Di await expect(accessRegion.getByText('No workflow references yet.')).toBeVisible() }) +Then( + 'I should see the Agent v2 Workflow access reference for {string}', + async function (this: DifyWorld, workflowName: string) { + const page = this.getPage() + const workflow = getPreseededResource(this, workflowName, 'workflow') + const agent = getPreseededResource( + this, + agentBuilderPreseededResources.workflowReferenceAgent, + 'agent', + ) + const references = await getAgentReferencingWorkflows(agent.id) + const reference = references.find(item => item.app_id === workflow.id || item.app_name === workflow.name) + if (!reference) + throw new Error(`Agent "${agent.name}" does not reference workflow "${workflow.name}".`) + + const accessRegion = page.getByRole('region', { name: 'Access Point' }) + const workflowSection = accessRegion.getByRole('region', { name: 'Workflow access' }) + const row = workflowSection.getByRole('row').filter({ hasText: workflowName }) + const nodeCount = reference.node_ids?.length ?? 0 + + await expect(accessRegion.getByRole('columnheader', { name: 'Name' })).toBeVisible() + await expect(accessRegion.getByRole('columnheader', { name: 'Version' })).toBeVisible() + await expect(accessRegion.getByRole('columnheader', { name: 'Nodes' })).toBeVisible() + await expect(accessRegion.getByRole('columnheader', { name: 'Last updated' })).toBeVisible() + await expect(accessRegion.getByRole('columnheader', { name: 'Actions' })).toBeVisible() + await expect(row).toBeVisible({ timeout: 30_000 }) + await expect(row.getByText(reference.workflow_version, { exact: true })).toBeVisible() + await expect(row.getByText(new RegExp(`^${nodeCount} nodes?$`))).toBeVisible() + if (reference.app_updated_at == null) + await expect(row.getByText('N/A', { exact: true })).toBeVisible() + else + await expect(row.getByText('N/A', { exact: true })).not.toBeVisible() + await expect(row.getByRole('link', { name: `Open ${workflowName} in Studio` })).toBeVisible() + }, +) + +When( + 'I open the Agent v2 Workflow access reference for {string}', + async function (this: DifyWorld, workflowName: string) { + const page = this.getPage() + const workflowLink = page.getByRole('link', { name: `Open ${workflowName} in Studio` }) + + const [workflowPage] = await Promise.all([ + page.waitForEvent('popup'), + workflowLink.click(), + ]) + + this.lastAgentWorkflowReferencePage = workflowPage + }, +) + +Then( + 'the Agent v2 Workflow access reference for {string} should open in Studio', + async function (this: DifyWorld, workflowName: string) { + const workflowPage = this.lastAgentWorkflowReferencePage + if (!workflowPage) + throw new Error('No Agent v2 Workflow access reference page was opened.') + + const workflow = getPreseededResource(this, workflowName, 'workflow') + + await expect(workflowPage).toHaveURL(new RegExp(`/app/${workflow.id}/workflow(?:\\?.*)?$`)) + await workflowPage.close() + this.lastAgentWorkflowReferencePage = undefined + }, +) + Then('I should see the Agent v2 Backend service API endpoint', async function (this: DifyWorld) { const page = this.getPage() diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index 58ff2e01b710e4..f94ec887d2410f 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -46,6 +46,7 @@ export class DifyWorld extends World { lastAgentServiceApiBaseURL: string | undefined lastGeneratedAgentApiKey: string | undefined lastAgentApiReferencePage: Page | undefined + lastAgentWorkflowReferencePage: Page | undefined createdAppIds: string[] = [] createdAgentIds: string[] = [] createdDatasetIds: string[] = [] @@ -74,6 +75,7 @@ export class DifyWorld extends World { this.lastAgentServiceApiBaseURL = undefined this.lastGeneratedAgentApiKey = undefined this.lastAgentApiReferencePage = undefined + this.lastAgentWorkflowReferencePage = undefined this.createdAppIds = [] this.createdAgentIds = [] this.createdDatasetIds = [] diff --git a/e2e/support/agent.ts b/e2e/support/agent.ts index 4467c5fb972072..a8a8829a742b41 100644 --- a/e2e/support/agent.ts +++ b/e2e/support/agent.ts @@ -70,6 +70,19 @@ export type AgentApiKey = { token?: string } +export type AgentReferencingWorkflow = { + app_id: string + app_name: string + app_updated_at?: number | null + node_ids?: string[] + workflow_id: string + workflow_version: string +} + +export type AgentReferencingWorkflowsResponse = { + data: AgentReferencingWorkflow[] +} + export type AgentDriveSkillUpload = { skill: { archive_key?: string | null @@ -389,6 +402,19 @@ export async function getAgentDriveSkills(agentId: string): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agentId}/referencing-workflows`) + await expectApiResponseOK(response, `Get Agent v2 referencing workflows for ${agentId}`) + const body = (await response.json()) as AgentReferencingWorkflowsResponse + return body.data + } + finally { + await ctx.dispose() + } +} + export async function getAgentComposerDraft(agentId: string): Promise { const ctx = await createApiContext() try { From 15af401519985d981e1ed3b12a8e3290e5fc77e6 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 18:20:58 +0800 Subject: [PATCH 059/185] test(e2e): mark agent content moderation preflight --- e2e/features/agent-v2/preflight.feature | 8 ++++++++ .../step-definitions/agent-v2/configure.steps.ts | 16 ++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/e2e/features/agent-v2/preflight.feature b/e2e/features/agent-v2/preflight.feature index 3472f9c2688f61..4453067ba9dc36 100644 --- a/e2e/features/agent-v2/preflight.feature +++ b/e2e/features/agent-v2/preflight.feature @@ -52,6 +52,14 @@ Feature: Agent Builder preseeded environment Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" includes the core fixture configuration + @content-moderation @feature-gated + Scenario: Content Moderation Settings is enabled + Given I am signed in as the default E2E admin + And a basic configured Agent v2 test agent has been created via API + When I open the Agent v2 configure page + And I expand Agent v2 Advanced Settings + Then Agent v2 Content Moderation Settings should be available + Scenario: Tool states Agent is available Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" is available diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 7be3e4f0795785..3232d028df3fdd 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -26,6 +26,7 @@ import { agentBuilderPreseededResources, } from '../../../support/agent-builder-resources' import { waitForAgentConfigureAutosaved } from '../../../support/agent-configure' +import { skipBlockedPrecondition } from '../../../support/preflight' import { agentBuilderTestMaterials, getAgentBuilderTestMaterialPath, @@ -727,6 +728,21 @@ Then( }, ) +Then('Agent v2 Content Moderation Settings should be available', async function (this: DifyWorld) { + const advancedSettings = this.getPage().getByRole('region', { name: 'Advanced Settings' }) + const contentModeration = advancedSettings.getByRole('region', { name: 'Content moderation' }) + + try { + await expect(contentModeration).toBeVisible({ timeout: 3_000 }) + } + catch { + return skipBlockedPrecondition( + this, + 'Feature not enabled: Agent v2 Content Moderation Settings is not available in this build.', + ) + } +}) + Then( 'I should see the Agent v2 environment variables from the valid import in Advanced Settings', async function (this: DifyWorld) { From d1a2fd1e58488126193a71b2cc945f7bbafdfd8e Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 18:28:40 +0800 Subject: [PATCH 060/185] test(e2e): mark build tool writeback blocked --- e2e/features/agent-v2/build-draft.feature | 7 +++++++ .../step-definitions/agent-v2/configure.steps.ts | 11 +++++++++++ 2 files changed, 18 insertions(+) diff --git a/e2e/features/agent-v2/build-draft.feature b/e2e/features/agent-v2/build-draft.feature index 671cc8abeb0237..95eed4e043794e 100644 --- a/e2e/features/agent-v2/build-draft.feature +++ b/e2e/features/agent-v2/build-draft.feature @@ -44,3 +44,10 @@ Feature: Agent v2 build draft When I refresh the current page Then I should see the updated E2E prompt in the Agent v2 prompt editor And the Agent v2 Build draft should no longer be active + + @build-tool-writeback @feature-gated + Scenario: Applying a Build draft can add Dify Tools to the Agent configuration + Given I am signed in as the default E2E admin + And a basic configured Agent v2 test agent has been created via API + When I open the Agent v2 configure page + Then Agent v2 Build chat Dify Tool writeback should be available diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 3232d028df3fdd..9975bb00518a64 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -743,6 +743,17 @@ Then('Agent v2 Content Moderation Settings should be available', async function } }) +Then('Agent v2 Build chat Dify Tool writeback should be available', async function (this: DifyWorld) { + const toolsSection = this.getPage().getByRole('region', { name: 'Tools' }) + + await expect(toolsSection).toBeVisible({ timeout: 30_000 }) + + return skipBlockedPrecondition( + this, + 'Build draft Dify Tool writeback is not available: Build draft currently supports files, skills, and env only.', + ) +}) + Then( 'I should see the Agent v2 environment variables from the valid import in Advanced Settings', async function (this: DifyWorld) { From d72bb98feb1e7a0b6c8568759fa0327cdb609f68 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 18:33:28 +0800 Subject: [PATCH 061/185] test(e2e): mark standalone output variables blocked --- e2e/features/agent-v2/output-variables.feature | 7 +++++++ .../step-definitions/agent-v2/configure.steps.ts | 11 +++++++++++ 2 files changed, 18 insertions(+) diff --git a/e2e/features/agent-v2/output-variables.feature b/e2e/features/agent-v2/output-variables.feature index 7ece4d68b9ec92..46716b894efec5 100644 --- a/e2e/features/agent-v2/output-variables.feature +++ b/e2e/features/agent-v2/output-variables.feature @@ -1,5 +1,12 @@ @agent-v2 @authenticated @output-variables @core Feature: Agent v2 output variables + @standalone-output-variables @feature-gated + Scenario: Standalone Agent configure exposes Output Variables + Given I am signed in as the default E2E admin + And a basic configured Agent v2 test agent has been created via API + When I open the Agent v2 configure page + Then Agent v2 standalone Output Variables should be available + Scenario: Workflow Agent v2 output variables persist after refresh Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 9975bb00518a64..15404ebaa214cf 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -754,6 +754,17 @@ Then('Agent v2 Build chat Dify Tool writeback should be available', async functi ) }) +Then('Agent v2 standalone Output Variables should be available', async function (this: DifyWorld) { + const page = this.getPage() + + await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) + + return skipBlockedPrecondition( + this, + 'Standalone Agent Output Variables are not available: output variables currently belong to Workflow Agent v2 nodes.', + ) +}) + Then( 'I should see the Agent v2 environment variables from the valid import in Advanced Settings', async function (this: DifyWorld) { From d32050b12655975d5ebec0cdffaf052105b3ab33 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 18:45:12 +0800 Subject: [PATCH 062/185] fix(workflow): avoid nested node header buttons --- .../nodes/_base/__tests__/node.spec.tsx | 22 +++++++- .../components/workflow/nodes/_base/node.tsx | 56 ++++++++++--------- 2 files changed, 51 insertions(+), 27 deletions(-) diff --git a/web/app/components/workflow/nodes/_base/__tests__/node.spec.tsx b/web/app/components/workflow/nodes/_base/__tests__/node.spec.tsx index bbc2d3c688a782..b6f5c32d025ca2 100644 --- a/web/app/components/workflow/nodes/_base/__tests__/node.spec.tsx +++ b/web/app/components/workflow/nodes/_base/__tests__/node.spec.tsx @@ -130,7 +130,7 @@ describe('BaseNode', () => { expect(screen.getByTestId('node-target-handle')).toBeInTheDocument() }) - it('should expose the node header as a selectable button', async () => { + it('should expose the node title area as a selectable button', async () => { const { selectWorkflowNode } = await import('@/app/components/workflow/utils/node-navigation') renderWorkflowComponent( @@ -146,6 +146,26 @@ describe('BaseNode', () => { expect(selectWorkflowNode).toHaveBeenCalledWith('node-1') }) + it('should keep header metadata outside the selectable button', () => { + renderWorkflowComponent( + +
Iteration body
+ , + ) + + const titleButton = screen.getByRole('button', { name: 'Node title' }) + const parallelButton = screen.getByRole('button', { name: /workflow\.nodes\.iteration\.parallelModeUpper/ }) + + expect(titleButton).not.toContainElement(parallelButton) + expect(titleButton.querySelector('button')).toBeNull() + }) + it('should render entry nodes inside the entry container', () => { renderWorkflowComponent( diff --git a/web/app/components/workflow/nodes/_base/node.tsx b/web/app/components/workflow/nodes/_base/node.tsx index a688e2030ab697..9fd1fb6f97e497 100644 --- a/web/app/components/workflow/nodes/_base/node.tsx +++ b/web/app/components/workflow/nodes/_base/node.tsx @@ -260,37 +260,41 @@ const BaseNode: FC = ({ /> ) } -
= ({ t={t} />
- +
)} From 6ce486a44955ac65d4207cd86b222353115539d7 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 18:45:25 +0800 Subject: [PATCH 063/185] fix(agent-v2): preserve compact publish bar layout --- .../configure/components/orchestrate/publish-bar/index.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/features/agent-v2/agent-detail/configure/components/orchestrate/publish-bar/index.tsx b/web/features/agent-v2/agent-detail/configure/components/orchestrate/publish-bar/index.tsx index cdba0ab1d99e42..e526dc8e015170 100644 --- a/web/features/agent-v2/agent-detail/configure/components/orchestrate/publish-bar/index.tsx +++ b/web/features/agent-v2/agent-detail/configure/components/orchestrate/publish-bar/index.tsx @@ -349,14 +349,14 @@ function PublishBarActions({
{statusLabel} · - + {metaLabel}
From c8e6ff3ebfb9eb8c4de2b20c3cb1ecbef1eb6403 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 18:57:55 +0800 Subject: [PATCH 064/185] fix(agent-v2): show build draft env changes --- .../components/orchestrate/advanced/index.tsx | 1 + .../orchestrate/build-draft-changes-context.ts | 2 ++ .../common/__tests__/section.spec.tsx | 16 ++++++++++++++-- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/index.tsx b/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/index.tsx index 7f71c52fd6c32f..a6d2a14ccccbb9 100644 --- a/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/index.tsx +++ b/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/index.tsx @@ -17,6 +17,7 @@ export function AgentAdvancedSettings() { panelId={advancedSettingsPanelId} description={t('agentDetail.configure.advancedSettings.description')} defaultOpen={false} + buildDraftChangeSection="advancedSettings" rootClassName="gap-2 pt-1 pb-3" headerClassName="mb-0 pt-2" titleRowClassName="min-h-6" diff --git a/web/features/agent-v2/agent-detail/configure/components/orchestrate/build-draft-changes-context.ts b/web/features/agent-v2/agent-detail/configure/components/orchestrate/build-draft-changes-context.ts index 3ba01f16da1297..111952903cd05d 100644 --- a/web/features/agent-v2/agent-detail/configure/components/orchestrate/build-draft-changes-context.ts +++ b/web/features/agent-v2/agent-detail/configure/components/orchestrate/build-draft-changes-context.ts @@ -9,10 +9,12 @@ export type AgentBuildDraftChangedKey = keyof AgentSoulConfigFormState export type AgentBuildDraftChangeSection = | 'skills' | 'files' + | 'advancedSettings' const changedKeysBySection: Record = { skills: ['skills'], files: ['files'], + advancedSettings: ['envVariables'], } const AgentBuildDraftChangedKeysContext = createContext>(new Set()) diff --git a/web/features/agent-v2/agent-detail/configure/components/orchestrate/common/__tests__/section.spec.tsx b/web/features/agent-v2/agent-detail/configure/components/orchestrate/common/__tests__/section.spec.tsx index fabcecab16f22f..0052bab0b05b78 100644 --- a/web/features/agent-v2/agent-detail/configure/components/orchestrate/common/__tests__/section.spec.tsx +++ b/web/features/agent-v2/agent-detail/configure/components/orchestrate/common/__tests__/section.spec.tsx @@ -7,13 +7,19 @@ function renderSection({ section = 'skills', changedKeys, }: { - section?: 'skills' | 'files' + section?: 'skills' | 'files' | 'advancedSettings' changedKeys: AgentBuildDraftChangedKey[] }) { + const label = { + advancedSettings: 'Advanced Settings', + files: 'Files', + skills: 'Skills', + }[section] + return render( @@ -36,6 +42,12 @@ describe('ConfigureSection', () => { expect(screen.getByRole('heading', { name: 'Files' }).querySelector('.bg-text-warning-secondary')).toBeInTheDocument() }) + it('should show a build draft change dot when Advanced Settings changed', () => { + renderSection({ section: 'advancedSettings', changedKeys: ['envVariables'] }) + + expect(screen.getByRole('heading', { name: 'Advanced Settings' }).querySelector('.bg-text-warning-secondary')).toBeInTheDocument() + }) + it('should not show a build draft change dot when only another key changed', () => { renderSection({ section: 'skills', changedKeys: ['prompt'] }) From daeecef1dc57e5be8d28151fbe30eab31b7065df Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 18:58:01 +0800 Subject: [PATCH 065/185] test(e2e): cover build draft supported config --- e2e/features/agent-v2/build-draft.feature | 26 ++++ .../agent-v2/configure.steps.ts | 139 +++++++++++++++++- e2e/support/agent.ts | 102 +++++++++++++ 3 files changed, 264 insertions(+), 3 deletions(-) diff --git a/e2e/features/agent-v2/build-draft.feature b/e2e/features/agent-v2/build-draft.feature index 95eed4e043794e..48e3f93bdcf6d9 100644 --- a/e2e/features/agent-v2/build-draft.feature +++ b/e2e/features/agent-v2/build-draft.feature @@ -45,6 +45,32 @@ Feature: Agent v2 build draft Then I should see the updated E2E prompt in the Agent v2 prompt editor And the Agent v2 Build draft should no longer be active + Scenario: Applying a Build draft updates supported configuration sections + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API + And an Agent v2 Build draft adds the supported E2E files, skills, and env + When I open the Agent v2 configure page + Then I should see the Agent v2 Build draft pending changes + And I should see the updated E2E prompt in the Agent v2 prompt editor + And I should see the small Agent v2 file in the Files section + And I should see the e2e-summary-skill Skill in the Skills section + And I should see the supported E2E environment variable in Advanced Settings + And the normal Agent v2 draft should still use the normal E2E prompt + When I apply the Agent v2 Build draft + Then I should see the updated E2E prompt in the Agent v2 prompt editor + And I should see the small Agent v2 file in the Files section + And I should see the e2e-summary-skill Skill in the Skills section + And I should see the supported E2E environment variable in Advanced Settings + And the Agent v2 draft should include the supported Build draft config + And the Agent v2 Build draft should no longer be active + When I refresh the current page + Then I should see the updated E2E prompt in the Agent v2 prompt editor + And I should see the small Agent v2 file in the Files section + And I should see the e2e-summary-skill Skill in the Skills section + And I should see the supported E2E environment variable in Advanced Settings + And the Agent v2 Build draft should no longer be active + @build-tool-writeback @feature-gated Scenario: Applying a Build draft can add Dify Tools to the Agent configuration Given I am signed in as the default E2E admin diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 15404ebaa214cf..92c104652f2984 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -18,6 +18,8 @@ import { saveAgentComposerDraft, updatedAgentPrompt, updatedAgentSoulConfig, + uploadAgentConfigFileToDraft, + uploadAgentConfigSkillToDraft, uploadAgentDriveSkill, } from '../../../support/agent' import { @@ -133,6 +135,45 @@ const expectAgentConfigFileSaved = async ( .toContain(fileName) } +const openAgentAdvancedSettings = async (page: ReturnType) => { + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + const envEditorHeading = advancedSettings.getByRole('heading', { name: 'Env Editor' }) + + if (!await envEditorHeading.isVisible().catch(() => false)) + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + + await expect(envEditorHeading).toBeVisible() + + return advancedSettings +} + +const expectAgentEnvVariableVisible = async ( + world: DifyWorld, + key: string, + value: string, +) => { + const advancedSettings = await openAgentAdvancedSettings(world.getPage()) + + await expect.poll( + async () => { + const text = await advancedSettings.textContent() + const inputValues = await advancedSettings.getByRole('textbox').evaluateAll(inputs => + inputs.map(input => (input as HTMLInputElement).value), + ) + + return { + hasKey: inputValues.includes(key) || !!text?.includes(key), + hasValue: inputValues.includes(value) || !!text?.includes(value), + } + }, + { timeout: 30_000 }, + ).toEqual({ + hasKey: true, + hasValue: true, + }) + await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() +} + const expectNormalAgentPromptDraft = async (world: DifyWorld) => { await expect.poll( async () => (await getAgentComposerDraft(getCurrentAgentId(world))).agent_soul?.prompt, @@ -215,6 +256,55 @@ Then('the Agent v2 test agent should include drive skill {string}', async functi expect(skills.map(skill => skill.name)).toContain(skillName) }) +Given( + 'an Agent v2 Build draft adds the supported E2E files, skills, and env', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + const configFile = await uploadAgentConfigFileToDraft({ + agentId, + fileName: agentBuilderTestMaterials.smallFile, + filePath: getAgentBuilderTestMaterialPath('smallFile'), + }) + const skill = await uploadAgentConfigSkillToDraft({ + agentId, + fileName: agentBuilderTestMaterials.summarySkill, + filePath: getAgentBuilderTestMaterialPath('summarySkill'), + }) + + if (!configFile.file_id) + throw new Error('Agent v2 build draft config file fixture did not return a file_id.') + if (!skill.file_id) + throw new Error('Agent v2 build draft Skill fixture did not return a file_id.') + + const normalConfig = this.agentBuilderStableChatModel + ? createAgentSoulConfigWithModel(normalAgentSoulConfig, this.agentBuilderStableChatModel) + : normalAgentSoulConfig + const updatedConfig = this.agentBuilderStableChatModel + ? createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilderStableChatModel) + : updatedAgentSoulConfig + + await saveAgentComposerDraft(agentId, normalConfig) + await saveAgentBuildDraft(agentId, { + ...updatedConfig, + config_files: [configFile], + config_skills: [{ + ...skill, + file_kind: skill.file_kind ?? 'tool_file', + }], + env: { + secret_refs: [], + variables: [{ + id: agentBuilderFixedInputs.envPlainKey, + key: agentBuilderFixedInputs.envPlainKey, + name: agentBuilderFixedInputs.envPlainKey, + value: agentBuilderFixedInputs.envPlainValue, + variable: agentBuilderFixedInputs.envPlainKey, + }], + }, + }) + }, +) + Given('an Agent v2 Build draft uses the updated E2E prompt', async function (this: DifyWorld) { await saveAgentBuildDraft(getCurrentAgentId(this), updatedAgentSoulConfig) }) @@ -553,6 +643,15 @@ Then('I should see the special-name Agent v2 file in the Files section', async f await expectAgentConfigFileVisible(this, 'specialFilename') }) +Then('I should see the e2e-summary-skill Skill in the Skills section', async function (this: DifyWorld) { + const skillsSection = this.getPage().getByRole('region', { name: 'Skills' }) + + await expect(skillsSection.getByRole('button', { + exact: true, + name: agentBuilderPreseededResources.summarySkill, + })).toBeVisible({ timeout: 30_000 }) +}) + Then( 'the small Agent v2 file should be saved in the Agent v2 draft', async function (this: DifyWorld) { @@ -819,10 +918,8 @@ Then( 'I should see the plain Agent v2 environment variable in Advanced Settings', async function (this: DifyWorld) { const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + const advancedSettings = await openAgentAdvancedSettings(page) - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() - await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() await expect(advancedSettings.getByRole('textbox', { name: 'Key' })) .toHaveValue(agentBuilderFixedInputs.envPlainKey) await expect(advancedSettings.getByRole('textbox', { name: 'Value' })) @@ -832,6 +929,17 @@ Then( }, ) +Then( + 'I should see the supported E2E environment variable in Advanced Settings', + async function (this: DifyWorld) { + await expectAgentEnvVariableVisible( + this, + agentBuilderFixedInputs.envPlainKey, + agentBuilderFixedInputs.envPlainValue, + ) + }, +) + Then( 'I should see the Agent v2 environment variables from the invalid import in Advanced Settings', async function (this: DifyWorld) { @@ -896,6 +1004,31 @@ Then( }, ) +Then( + 'the Agent v2 draft should include the supported Build draft config', + async function (this: DifyWorld) { + await expect.poll( + async () => { + const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul + const variables = agentSoul?.env?.variables ?? [] + + return { + envValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), + fileNames: agentSoul?.config_files?.map(file => file.name) ?? [], + prompt: agentSoul?.prompt, + skillNames: agentSoul?.config_skills?.map(skill => skill.name) ?? [], + } + }, + { timeout: 30_000 }, + ).toEqual({ + envValue: agentBuilderFixedInputs.envPlainValue, + fileNames: expect.arrayContaining([agentBuilderTestMaterials.smallFile]), + prompt: { system_prompt: updatedAgentPrompt }, + skillNames: expect.arrayContaining([agentBuilderPreseededResources.summarySkill]), + }) + }, +) + Then('the Agent v2 Build draft should no longer be active', async function (this: DifyWorld) { const page = this.getPage() diff --git a/e2e/support/agent.ts b/e2e/support/agent.ts index a8a8829a742b41..7fb4b77b7daa40 100644 --- a/e2e/support/agent.ts +++ b/e2e/support/agent.ts @@ -28,6 +28,15 @@ export type AgentComposerConfigFile = { name: string size?: number | null } +export type AgentComposerConfigSkill = { + description?: string | null + file_id?: string | null + file_kind?: string | null + hash?: string | null + mime_type?: string | null + name: string + size?: number | null +} export type AgentComposerEnvVariable = { id?: string | null key?: string | null @@ -38,6 +47,7 @@ export type AgentComposerEnvVariable = { export type AgentSoulConfig = Record & { config_files?: AgentComposerConfigFile[] + config_skills?: AgentComposerConfigSkill[] env?: { secret_refs?: unknown[] variables?: AgentComposerEnvVariable[] @@ -93,6 +103,21 @@ export type AgentDriveSkillUpload = { } } +export type AgentConfigSkillUpload = { + skill: AgentComposerConfigSkill +} + +export type UploadedConsoleFile = { + id: string + mime_type?: string | null + name: string + size?: number | null +} + +export type AgentConfigFileUpload = { + file: AgentComposerConfigFile +} + export type AgentDriveSkill = { description?: string | null name: string @@ -389,6 +414,83 @@ export async function uploadAgentDriveSkill({ } } +export async function uploadAgentConfigFileToDraft({ + agentId, + fileName, + filePath, +}: { + agentId: string + fileName: string + filePath: string +}): Promise { + const ctx = await createApiContext() + try { + const uploadResponse = await ctx.post('/console/api/files/upload', { + multipart: { + file: { + buffer: await readFile(filePath), + mimeType: 'text/plain', + name: fileName, + }, + }, + }) + await expectApiResponseOK(uploadResponse, `Upload Agent v2 config source file ${fileName}`) + const uploadedFile = (await uploadResponse.json()) as UploadedConsoleFile + + const commitResponse = await ctx.post(`/console/api/agent/${agentId}/config/files`, { + data: { + upload_file_id: uploadedFile.id, + }, + }) + await expectApiResponseOK(commitResponse, `Commit Agent v2 config file ${fileName} for ${agentId}`) + const body = (await commitResponse.json()) as AgentConfigFileUpload + const { id: _id, ...file } = body.file as AgentComposerConfigFile & { id?: string } + + return { + ...file, + file_kind: file.file_kind ?? 'upload_file', + } + } + finally { + await ctx.dispose() + } +} + +export async function uploadAgentConfigSkillToDraft({ + agentId, + fileName, + filePath, +}: { + agentId: string + fileName: string + filePath: string +}): Promise { + const ctx = await createApiContext() + try { + const upload = await toSkillArchiveUpload({ fileName, filePath }) + const response = await ctx.post(`/console/api/agent/${agentId}/config/skills/upload`, { + multipart: { + file: { + buffer: upload.buffer, + mimeType: 'application/zip', + name: upload.name, + }, + }, + }) + await expectApiResponseOK(response, `Upload Agent v2 config skill ${fileName} for ${agentId}`) + const body = (await response.json()) as AgentConfigSkillUpload + const { id: _id, ...skill } = body.skill as AgentComposerConfigSkill & { id?: string } + + return { + ...skill, + file_kind: skill.file_kind ?? 'tool_file', + } + } + finally { + await ctx.dispose() + } +} + export async function getAgentDriveSkills(agentId: string): Promise { const ctx = await createApiContext() try { From 93a0ee364dcd94e353a2995381ea241d619a7ad4 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 19:05:10 +0800 Subject: [PATCH 066/185] test(e2e): clean build draft config uploads --- e2e/features/step-definitions/agent-v2/configure.steps.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 92c104652f2984..00723e67577e09 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -275,6 +275,7 @@ Given( throw new Error('Agent v2 build draft config file fixture did not return a file_id.') if (!skill.file_id) throw new Error('Agent v2 build draft Skill fixture did not return a file_id.') + this.createdAgentConfigFiles.push({ agentId, name: configFile.name }) const normalConfig = this.agentBuilderStableChatModel ? createAgentSoulConfigWithModel(normalAgentSoulConfig, this.agentBuilderStableChatModel) From 56075da07d4592712306d25cc20db7b8a9b7326e Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 19:06:49 +0800 Subject: [PATCH 067/185] test(e2e): clean agent config skills --- .../step-definitions/agent-v2/configure.steps.ts | 1 + e2e/features/support/hooks.ts | 4 +++- e2e/features/support/world.ts | 6 ++++++ e2e/support/agent.ts | 11 +++++++++++ 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 00723e67577e09..8a571a89ab9277 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -276,6 +276,7 @@ Given( if (!skill.file_id) throw new Error('Agent v2 build draft Skill fixture did not return a file_id.') this.createdAgentConfigFiles.push({ agentId, name: configFile.name }) + this.createdAgentConfigSkills.push({ agentId, name: skill.name }) const normalConfig = this.agentBuilderStableChatModel ? createAgentSoulConfigWithModel(normalAgentSoulConfig, this.agentBuilderStableChatModel) diff --git a/e2e/features/support/hooks.ts b/e2e/features/support/hooks.ts index 65bb4b8c2a86df..393cbab72c31d6 100644 --- a/e2e/features/support/hooks.ts +++ b/e2e/features/support/hooks.ts @@ -7,7 +7,7 @@ import { fileURLToPath } from 'node:url' import { After, AfterAll, Before, BeforeAll, setDefaultTimeout, Status } from '@cucumber/cucumber' import { chromium } from '@playwright/test' import { AUTH_BOOTSTRAP_TIMEOUT_MS, ensureAuthenticatedState } from '../../fixtures/auth' -import { deleteAgentConfigFile, deleteAgentDriveFile, deleteTestAgent } from '../../support/agent' +import { deleteAgentConfigFile, deleteAgentConfigSkill, deleteAgentDriveFile, deleteTestAgent } from '../../support/agent' import { deleteTestApp } from '../../support/api' import { deleteTestDataset } from '../../support/datasets' import { deleteBuiltinToolCredential } from '../../support/tools' @@ -93,6 +93,8 @@ After(async function (this: DifyWorld, { pickle, result }) { `[e2e] end ${pickle.name} status=${status}${elapsedMs ? ` durationMs=${elapsedMs}` : ''}`, ) + for (const skill of this.createdAgentConfigSkills.toReversed()) + await deleteAgentConfigSkill(skill.agentId, skill.name).catch(() => {}) for (const file of this.createdAgentConfigFiles.toReversed()) await deleteAgentConfigFile(file.agentId, file.name).catch(() => {}) for (const file of this.createdAgentDriveFiles.toReversed()) diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index f94ec887d2410f..c1961f8c2ed1ec 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -14,6 +14,10 @@ export type CreatedAgentConfigFile = { agentId: string name: string } +export type CreatedAgentConfigSkill = { + agentId: string + name: string +} export type CreatedBuiltinToolCredential = { credentialId: string provider: string @@ -51,6 +55,7 @@ export class DifyWorld extends World { createdAgentIds: string[] = [] createdDatasetIds: string[] = [] createdAgentConfigFiles: CreatedAgentConfigFile[] = [] + createdAgentConfigSkills: CreatedAgentConfigSkill[] = [] createdAgentDriveFiles: CreatedAgentDriveFile[] = [] createdBuiltinToolCredentials: CreatedBuiltinToolCredential[] = [] agentBuilderBrokenChatModel: AgentBuilderStableChatModel | undefined @@ -80,6 +85,7 @@ export class DifyWorld extends World { this.createdAgentIds = [] this.createdDatasetIds = [] this.createdAgentConfigFiles = [] + this.createdAgentConfigSkills = [] this.createdAgentDriveFiles = [] this.createdBuiltinToolCredentials = [] this.agentBuilderBrokenChatModel = undefined diff --git a/e2e/support/agent.ts b/e2e/support/agent.ts index 7fb4b77b7daa40..3be77f0dcd514e 100644 --- a/e2e/support/agent.ts +++ b/e2e/support/agent.ts @@ -667,6 +667,17 @@ export async function deleteAgentConfigFile(agentId: string, name: string): Prom } } +export async function deleteAgentConfigSkill(agentId: string, name: string): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.delete(`/console/api/agent/${agentId}/config/skills/${encodeURIComponent(name)}`) + await expectApiResponseOK(response, `Delete Agent v2 config skill ${name} for ${agentId}`) + } + finally { + await ctx.dispose() + } +} + export async function deleteAgentDriveFile(agentId: string, key: string): Promise { const ctx = await createApiContext() try { From a7a2ae85ea4c1d35d785d33ba91a0f60754faadc Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 19:10:19 +0800 Subject: [PATCH 068/185] test(e2e): cover build draft skill dedupe --- e2e/features/agent-v2/build-draft.feature | 17 +++++ .../agent-v2/configure.steps.ts | 75 +++++++++++++++++-- 2 files changed, 84 insertions(+), 8 deletions(-) diff --git a/e2e/features/agent-v2/build-draft.feature b/e2e/features/agent-v2/build-draft.feature index 48e3f93bdcf6d9..095df3f11baca1 100644 --- a/e2e/features/agent-v2/build-draft.feature +++ b/e2e/features/agent-v2/build-draft.feature @@ -71,6 +71,23 @@ Feature: Agent v2 build draft And I should see the supported E2E environment variable in Advanced Settings And the Agent v2 Build draft should no longer be active + Scenario: Applying a Build draft with an existing Skill keeps a single Skill entry + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API + And an Agent v2 Build draft includes the existing e2e-summary-skill Skill + When I open the Agent v2 configure page + Then I should see the Agent v2 Build draft pending changes + And I should see one e2e-summary-skill Skill in the Skills section + And the Agent v2 draft should include one e2e-summary-skill Skill + When I apply the Agent v2 Build draft + Then I should see one e2e-summary-skill Skill in the Skills section + And the Agent v2 draft should include one e2e-summary-skill Skill + And the Agent v2 Build draft should no longer be active + When I refresh the current page + Then I should see one e2e-summary-skill Skill in the Skills section + And the Agent v2 draft should include one e2e-summary-skill Skill + @build-tool-writeback @feature-gated Scenario: Applying a Build draft can add Dify Tools to the Agent configuration Given I am signed in as the default E2E admin diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 8a571a89ab9277..2a9e80205ece24 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -135,6 +135,22 @@ const expectAgentConfigFileSaved = async ( .toContain(fileName) } +const uploadSummaryConfigSkillForBuildDraft = async (world: DifyWorld) => { + const agentId = getCurrentAgentId(world) + const skill = await uploadAgentConfigSkillToDraft({ + agentId, + fileName: agentBuilderTestMaterials.summarySkill, + filePath: getAgentBuilderTestMaterialPath('summarySkill'), + }) + + if (!skill.file_id) + throw new Error('Agent v2 build draft Skill fixture did not return a file_id.') + + world.createdAgentConfigSkills.push({ agentId, name: skill.name }) + + return skill +} + const openAgentAdvancedSettings = async (page: ReturnType) => { const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) const envEditorHeading = advancedSettings.getByRole('heading', { name: 'Env Editor' }) @@ -265,18 +281,11 @@ Given( fileName: agentBuilderTestMaterials.smallFile, filePath: getAgentBuilderTestMaterialPath('smallFile'), }) - const skill = await uploadAgentConfigSkillToDraft({ - agentId, - fileName: agentBuilderTestMaterials.summarySkill, - filePath: getAgentBuilderTestMaterialPath('summarySkill'), - }) + const skill = await uploadSummaryConfigSkillForBuildDraft(this) if (!configFile.file_id) throw new Error('Agent v2 build draft config file fixture did not return a file_id.') - if (!skill.file_id) - throw new Error('Agent v2 build draft Skill fixture did not return a file_id.') this.createdAgentConfigFiles.push({ agentId, name: configFile.name }) - this.createdAgentConfigSkills.push({ agentId, name: skill.name }) const normalConfig = this.agentBuilderStableChatModel ? createAgentSoulConfigWithModel(normalAgentSoulConfig, this.agentBuilderStableChatModel) @@ -307,6 +316,32 @@ Given( }, ) +Given( + 'an Agent v2 Build draft includes the existing e2e-summary-skill Skill', + async function (this: DifyWorld) { + const skill = await uploadSummaryConfigSkillForBuildDraft(this) + const normalConfig = this.agentBuilderStableChatModel + ? createAgentSoulConfigWithModel(normalAgentSoulConfig, this.agentBuilderStableChatModel) + : normalAgentSoulConfig + const updatedConfig = this.agentBuilderStableChatModel + ? createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilderStableChatModel) + : updatedAgentSoulConfig + const configSkills = [{ + ...skill, + file_kind: skill.file_kind ?? 'tool_file', + }] + + await saveAgentComposerDraft(getCurrentAgentId(this), { + ...normalConfig, + config_skills: configSkills, + }) + await saveAgentBuildDraft(getCurrentAgentId(this), { + ...updatedConfig, + config_skills: configSkills, + }) + }, +) + Given('an Agent v2 Build draft uses the updated E2E prompt', async function (this: DifyWorld) { await saveAgentBuildDraft(getCurrentAgentId(this), updatedAgentSoulConfig) }) @@ -654,6 +689,15 @@ Then('I should see the e2e-summary-skill Skill in the Skills section', async fun })).toBeVisible({ timeout: 30_000 }) }) +Then('I should see one e2e-summary-skill Skill in the Skills section', async function (this: DifyWorld) { + const skillsSection = this.getPage().getByRole('region', { name: 'Skills' }) + + await expect(skillsSection.getByRole('button', { + exact: true, + name: agentBuilderPreseededResources.summarySkill, + })).toHaveCount(1) +}) + Then( 'the small Agent v2 file should be saved in the Agent v2 draft', async function (this: DifyWorld) { @@ -1031,6 +1075,21 @@ Then( }, ) +Then( + 'the Agent v2 draft should include one e2e-summary-skill Skill', + async function (this: DifyWorld) { + await expect.poll( + async () => { + const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul + return agentSoul?.config_skills?.filter( + skill => skill.name === agentBuilderPreseededResources.summarySkill, + ).length ?? 0 + }, + { timeout: 30_000 }, + ).toBe(1) + }, +) + Then('the Agent v2 Build draft should no longer be active', async function (this: DifyWorld) { const page = this.getPage() From efa2256bdd77b0d9c097f1d0b5a4956946d447a0 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 19:18:11 +0800 Subject: [PATCH 069/185] test(e2e): cover build draft navigation retention --- e2e/features/agent-v2/build-draft.feature | 15 +++++++++++++++ .../step-definitions/agent-v2/configure.steps.ts | 9 +++++++++ 2 files changed, 24 insertions(+) diff --git a/e2e/features/agent-v2/build-draft.feature b/e2e/features/agent-v2/build-draft.feature index 095df3f11baca1..897d112840d999 100644 --- a/e2e/features/agent-v2/build-draft.feature +++ b/e2e/features/agent-v2/build-draft.feature @@ -88,6 +88,21 @@ Feature: Agent v2 build draft Then I should see one e2e-summary-skill Skill in the Skills section And the Agent v2 draft should include one e2e-summary-skill Skill + Scenario: Pending Build draft remains protected after leaving Configure + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API + And an Agent v2 Build draft uses the updated E2E prompt with the stable E2E model + When I open the Agent v2 configure page + Then I should see the Agent v2 Build draft pending changes + And I should see the updated E2E prompt in the Agent v2 prompt editor + And the normal Agent v2 draft should still use the normal E2E prompt + When I switch to the Agent v2 Access Point section + And I switch to the Agent v2 Configure section + Then I should see the Agent v2 Build draft pending changes + And I should see the updated E2E prompt in the Agent v2 prompt editor + And the normal Agent v2 draft should still use the normal E2E prompt + @build-tool-writeback @feature-gated Scenario: Applying a Build draft can add Dify Tools to the Agent configuration Given I am signed in as the default E2E admin diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 2a9e80205ece24..0d50c54c4f1690 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -363,6 +363,15 @@ When('I open the Agent v2 configure page', async function (this: DifyWorld) { await this.getPage().goto(getAgentConfigurePath(getCurrentAgentId(this))) }) +When('I switch to the Agent v2 Configure section', async function (this: DifyWorld) { + const page = this.getPage() + const agentId = getCurrentAgentId(this) + + await page.getByRole('link', { name: 'Configure' }).click() + await expect(page).toHaveURL(new RegExp(`/roster/agent/${agentId}/configure(?:\\?.*)?$`)) + await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) +}) + When('I generate an Agent v2 Build draft from the fixed instruction', async function (this: DifyWorld) { const page = this.getPage() const agentId = getCurrentAgentId(this) From 8d800e73021866ea93629a0d3b7736d3f4f49605 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 19:22:10 +0800 Subject: [PATCH 070/185] test(e2e): cover build draft discard isolation --- e2e/features/agent-v2/build-draft.feature | 26 +++++++ .../agent-v2/configure.steps.ts | 76 +++++++++++++++++++ 2 files changed, 102 insertions(+) diff --git a/e2e/features/agent-v2/build-draft.feature b/e2e/features/agent-v2/build-draft.feature index 897d112840d999..05f471b1bae053 100644 --- a/e2e/features/agent-v2/build-draft.feature +++ b/e2e/features/agent-v2/build-draft.feature @@ -28,6 +28,32 @@ Feature: Agent v2 build draft Then I should see the normal E2E prompt in the Agent v2 prompt editor And the Agent v2 Build draft should no longer be active + Scenario: Discarding a Build draft does not apply supported configuration changes + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API + And an Agent v2 Build draft adds the supported E2E files, skills, and env + When I open the Agent v2 configure page + Then I should see the Agent v2 Build draft pending changes + And I should see the small Agent v2 file in the Files section + And I should see the e2e-summary-skill Skill in the Skills section + And I should see the supported E2E environment variable in Advanced Settings + And the normal Agent v2 draft should still use the normal E2E prompt + When I discard the Agent v2 Build draft + Then I should see the normal E2E prompt in the Agent v2 prompt editor + And I should not see the small Agent v2 file in the Files section + And I should not see the e2e-summary-skill Skill in the Skills section + And I should not see the supported E2E environment variable in Advanced Settings + And the Agent v2 draft should not include the supported Build draft config + And the Agent v2 Build draft should no longer be active + When I refresh the current page + Then I should see the normal E2E prompt in the Agent v2 prompt editor + And I should not see the small Agent v2 file in the Files section + And I should not see the e2e-summary-skill Skill in the Skills section + And I should not see the supported E2E environment variable in Advanced Settings + And the Agent v2 draft should not include the supported Build draft config + And the Agent v2 Build draft should no longer be active + Scenario: Applying a pending Build draft updates the normal Agent configuration Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 0d50c54c4f1690..0cab4bbe2d2b3f 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -119,6 +119,18 @@ const expectAgentConfigFileVisible = async ( ).toBeVisible({ timeout: 30_000 }) } +const expectAgentConfigFileHidden = async ( + world: DifyWorld, + material: keyof typeof agentBuilderTestMaterials, +) => { + await expect( + world.getPage().getByRole('button', { + exact: true, + name: agentBuilderTestMaterials[material], + }), + ).not.toBeVisible() +} + const expectAgentConfigFileSaved = async ( world: DifyWorld, material: keyof typeof agentBuilderTestMaterials, @@ -190,6 +202,25 @@ const expectAgentEnvVariableVisible = async ( await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() } +const expectAgentEnvVariableHidden = async ( + world: DifyWorld, + key: string, +) => { + const advancedSettings = await openAgentAdvancedSettings(world.getPage()) + + await expect.poll( + async () => { + const text = await advancedSettings.textContent() + const inputValues = await advancedSettings.getByRole('textbox').evaluateAll(inputs => + inputs.map(input => (input as HTMLInputElement).value), + ) + + return inputValues.includes(key) || !!text?.includes(key) + }, + { timeout: 30_000 }, + ).toBe(false) +} + const expectNormalAgentPromptDraft = async (world: DifyWorld) => { await expect.poll( async () => (await getAgentComposerDraft(getCurrentAgentId(world))).agent_soul?.prompt, @@ -685,6 +716,10 @@ Then('I should see the small Agent v2 file in the Files section', async function await expectAgentConfigFileVisible(this, 'smallFile') }) +Then('I should not see the small Agent v2 file in the Files section', async function (this: DifyWorld) { + await expectAgentConfigFileHidden(this, 'smallFile') +}) + Then('I should see the special-name Agent v2 file in the Files section', async function (this: DifyWorld) { await expectAgentConfigFileVisible(this, 'specialFilename') }) @@ -698,6 +733,15 @@ Then('I should see the e2e-summary-skill Skill in the Skills section', async fun })).toBeVisible({ timeout: 30_000 }) }) +Then('I should not see the e2e-summary-skill Skill in the Skills section', async function (this: DifyWorld) { + const skillsSection = this.getPage().getByRole('region', { name: 'Skills' }) + + await expect(skillsSection.getByRole('button', { + exact: true, + name: agentBuilderPreseededResources.summarySkill, + })).not.toBeVisible() +}) + Then('I should see one e2e-summary-skill Skill in the Skills section', async function (this: DifyWorld) { const skillsSection = this.getPage().getByRole('region', { name: 'Skills' }) @@ -995,6 +1039,13 @@ Then( }, ) +Then( + 'I should not see the supported E2E environment variable in Advanced Settings', + async function (this: DifyWorld) { + await expectAgentEnvVariableHidden(this, agentBuilderFixedInputs.envPlainKey) + }, +) + Then( 'I should see the Agent v2 environment variables from the invalid import in Advanced Settings', async function (this: DifyWorld) { @@ -1084,6 +1135,31 @@ Then( }, ) +Then( + 'the Agent v2 draft should not include the supported Build draft config', + async function (this: DifyWorld) { + await expect.poll( + async () => { + const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul + const variables = agentSoul?.env?.variables ?? [] + + return { + envValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), + fileNames: agentSoul?.config_files?.map(file => file.name) ?? [], + prompt: agentSoul?.prompt, + skillNames: agentSoul?.config_skills?.map(skill => skill.name) ?? [], + } + }, + { timeout: 30_000 }, + ).toEqual({ + envValue: undefined, + fileNames: expect.not.arrayContaining([agentBuilderTestMaterials.smallFile]), + prompt: { system_prompt: normalAgentPrompt }, + skillNames: expect.not.arrayContaining([agentBuilderPreseededResources.summarySkill]), + }) + }, +) + Then( 'the Agent v2 draft should include one e2e-summary-skill Skill', async function (this: DifyWorld) { From dcd635d5e04803567b9fcec86fad04cc392256d3 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 19:27:05 +0800 Subject: [PATCH 071/185] test(e2e): align file scenarios with runnable agents --- e2e/features/agent-v2/files.feature | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/e2e/features/agent-v2/files.feature b/e2e/features/agent-v2/files.feature index 8b352f9d86548a..01f1862283683d 100644 --- a/e2e/features/agent-v2/files.feature +++ b/e2e/features/agent-v2/files.feature @@ -2,20 +2,24 @@ Feature: Agent v2 files Scenario: Uploading a small file keeps it in the Agent configuration Given I am signed in as the default E2E admin - And a basic configured Agent v2 test agent has been created via API + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API When I open the Agent v2 configure page And I upload the small Agent v2 file from the Files section Then I should see the small Agent v2 file in the Files section And the small Agent v2 file should be saved in the Agent v2 draft + And the Agent v2 configuration should be saved automatically When I refresh the current page Then I should see the small Agent v2 file in the Files section Scenario: Uploading a special-name file keeps the filename readable Given I am signed in as the default E2E admin - And a basic configured Agent v2 test agent has been created via API + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API When I open the Agent v2 configure page And I upload the special-name Agent v2 file from the Files section Then I should see the special-name Agent v2 file in the Files section And the special-name Agent v2 file should be saved in the Agent v2 draft + And the Agent v2 configuration should be saved automatically When I refresh the current page Then I should see the special-name Agent v2 file in the Files section From 8f5fbb5444b52ea61b2bc9cfdd1749e35bd3b59f Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 19:30:57 +0800 Subject: [PATCH 072/185] test(e2e): align advanced settings with runnable agents --- .../agent-v2/advanced-settings.feature | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/e2e/features/agent-v2/advanced-settings.feature b/e2e/features/agent-v2/advanced-settings.feature index a14aaee97f0636..f8cdd2cb319dab 100644 --- a/e2e/features/agent-v2/advanced-settings.feature +++ b/e2e/features/agent-v2/advanced-settings.feature @@ -2,7 +2,8 @@ Feature: Agent v2 advanced settings Scenario: Advanced Settings exposes supported configuration entries Given I am signed in as the default E2E admin - And a basic configured Agent v2 test agent has been created via API + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API When I open the Agent v2 configure page Then Agent v2 Advanced Settings should describe supported entries while collapsed When I expand Agent v2 Advanced Settings @@ -10,42 +11,50 @@ Feature: Agent v2 advanced settings Scenario: Plain environment variables are saved and restored Given I am signed in as the default E2E admin - And a basic configured Agent v2 test agent has been created via API + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API When I open the Agent v2 configure page And I add the plain Agent v2 environment variable from Advanced Settings Then the plain Agent v2 environment variable should be saved in the Agent v2 draft + And the Agent v2 configuration should be saved automatically When I refresh the current page Then I should see the plain Agent v2 environment variable in Advanced Settings Scenario: Valid environment imports are saved and restored Given I am signed in as the default E2E admin - And a basic configured Agent v2 test agent has been created via API + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API When I open the Agent v2 configure page And I import the valid Agent v2 environment file from Advanced Settings Then the valid Agent v2 environment import should be saved in the Agent v2 draft + And the Agent v2 configuration should be saved automatically When I refresh the current page Then I should see the Agent v2 environment variables from the valid import in Advanced Settings Scenario: Deleted environment variables are removed after refresh Given I am signed in as the default E2E admin - And a basic configured Agent v2 test agent has been created via API + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API When I open the Agent v2 configure page And I add the plain Agent v2 environment variable from Advanced Settings And I add the secondary plain Agent v2 environment variable from Advanced Settings Then the Agent v2 environment variables for deletion should be saved in the Agent v2 draft When I delete the plain Agent v2 environment variable from Advanced Settings Then the plain Agent v2 environment variable should be removed from the Agent v2 draft + And the Agent v2 configuration should be saved automatically When I refresh the current page Then I should not see the deleted Agent v2 environment variable in Advanced Settings Scenario: Invalid environment imports report skipped lines and keep existing variables Given I am signed in as the default E2E admin - And a basic configured Agent v2 test agent has been created via API + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API When I open the Agent v2 configure page And I add the plain Agent v2 environment variable from Advanced Settings Then the plain Agent v2 environment variable should be saved in the Agent v2 draft When I import the invalid Agent v2 environment file from Advanced Settings Then the invalid Agent v2 environment import should report skipped lines And the Agent v2 environment variables from the invalid import should be saved in the Agent v2 draft + And the Agent v2 configuration should be saved automatically When I refresh the current page Then I should see the Agent v2 environment variables from the invalid import in Advanced Settings From 4c393c0d309ef0d9dfc8b8e8b80b3cf779b12f78 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 19:35:46 +0800 Subject: [PATCH 073/185] test(e2e): assert persisted agent model config --- .../agent-v2/configure-persistence.feature | 6 ++- .../agent-v2/configure.steps.ts | 39 +++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/e2e/features/agent-v2/configure-persistence.feature b/e2e/features/agent-v2/configure-persistence.feature index 7b525225bad2fa..b0053eff45a023 100644 --- a/e2e/features/agent-v2/configure-persistence.feature +++ b/e2e/features/agent-v2/configure-persistence.feature @@ -3,10 +3,14 @@ Feature: Agent v2 configure persistence @configure-persistence Scenario: Persisted Agent v2 instructions remain visible after refresh Given I am signed in as the default E2E admin - And an Agent v2 test agent has been created via API + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API When I open the Agent v2 configure page And I fill the Agent v2 prompt editor with the normal E2E prompt Then the normal Agent v2 draft should use the normal E2E prompt + And the Agent v2 draft should use the stable E2E model And the Agent v2 configuration should be saved automatically When I refresh the current page Then I should see the normal E2E prompt in the Agent v2 prompt editor + And I should see the stable E2E model in the Agent v2 model selector + And the Agent v2 draft should use the stable E2E model diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 0cab4bbe2d2b3f..34dc352b0e5087 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -692,6 +692,18 @@ Then( }, ) +Then( + 'I should see the stable E2E model in the Agent v2 model selector', + async function (this: DifyWorld) { + const stableModel = this.agentBuilderStableChatModel + if (!stableModel) + throw new Error('Stable chat model preflight must run before asserting the Agent model.') + + await expect(this.getPage().getByText(stableModel.name, { exact: true })) + .toBeVisible({ timeout: 30_000 }) + }, +) + Then( 'I should see the updated E2E prompt in the Agent v2 prompt editor', async function (this: DifyWorld) { @@ -1110,6 +1122,33 @@ Then( }, ) +Then( + 'the Agent v2 draft should use the stable E2E model', + async function (this: DifyWorld) { + const stableModel = this.agentBuilderStableChatModel + if (!stableModel) + throw new Error('Stable chat model preflight must run before asserting the Agent model.') + + await expect.poll( + async () => { + const model = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul?.model + const modelConfig = typeof model === 'object' && model !== null && !Array.isArray(model) + ? model as Record + : undefined + + return { + model: modelConfig?.model, + provider: modelConfig?.model_provider, + } + }, + { timeout: 30_000 }, + ).toEqual({ + model: stableModel.name, + provider: stableModel.provider, + }) + }, +) + Then( 'the Agent v2 draft should include the supported Build draft config', async function (this: DifyWorld) { From 413a5090263f5ce05a2f61f664d6a3969771d98f Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 19:44:11 +0800 Subject: [PATCH 074/185] test(e2e): open access point overview from roster --- e2e/features/agent-v2/access-point.feature | 3 ++- e2e/features/step-definitions/agent-v2/access-point.steps.ts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/e2e/features/agent-v2/access-point.feature b/e2e/features/agent-v2/access-point.feature index 27d590471fdceb..0605a3d67b483d 100644 --- a/e2e/features/agent-v2/access-point.feature +++ b/e2e/features/agent-v2/access-point.feature @@ -3,7 +3,8 @@ Feature: Agent v2 Access Point Scenario: Access Point shows the available Agent v2 access surfaces Given I am signed in as the default E2E admin And an Agent v2 test agent has been created via API - When I open the Agent v2 Access Point page + When I open the Agent v2 configure page from the Agent Roster + And I switch to the Agent v2 Access Point section Then I should see the Agent v2 Access Point overview @workflow-reference diff --git a/e2e/features/step-definitions/agent-v2/access-point.steps.ts b/e2e/features/step-definitions/agent-v2/access-point.steps.ts index f9bf56bc861868..0d0ad04d6bcab7 100644 --- a/e2e/features/step-definitions/agent-v2/access-point.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point.steps.ts @@ -76,7 +76,7 @@ Then('I should see the Agent v2 Access Point overview', async function (this: Di await expect(accessRegion.getByText('Access URL')).toBeVisible() await expect(accessRegion.getByLabel('Copy access URL')).toBeVisible() await expect(accessRegion.getByLabel('Toggle Web app access')).toBeVisible() - await expect(accessRegion.getByRole('button', { name: 'Launch' })).toBeVisible() + await expect(accessRegion.getByRole('link', { name: 'Launch' })).toBeVisible() await expect(accessRegion.getByRole('button', { name: 'Embedded' })).toBeVisible() await expect(accessRegion.getByRole('button', { name: 'Customize' })).toBeVisible() await expect(accessRegion.getByRole('button', { name: 'Settings' })).toBeVisible() From ddf814118b4d4b5bb3251d049f51330f63f20c75 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 19:50:48 +0800 Subject: [PATCH 075/185] test(e2e): cover web app access actions --- e2e/features/agent-v2/access-point.feature | 20 +++ .../agent-v2/access-point.steps.ts | 122 ++++++++++++++++++ e2e/features/support/world.ts | 6 + 3 files changed, 148 insertions(+) diff --git a/e2e/features/agent-v2/access-point.feature b/e2e/features/agent-v2/access-point.feature index 0605a3d67b483d..c418cfb209d1d8 100644 --- a/e2e/features/agent-v2/access-point.feature +++ b/e2e/features/agent-v2/access-point.feature @@ -7,6 +7,26 @@ Feature: Agent v2 Access Point And I switch to the Agent v2 Access Point section Then I should see the Agent v2 Access Point overview + @web-app-access + Scenario: Web app access actions open their public surfaces + Given I am signed in as the default E2E admin + And the Agent Builder preseeded Agent "E2E Agent Published Web App" is available + And the Agent Builder preseeded Agent "E2E Agent Published Web App" has published Web app access + When I open the preseeded Agent v2 Access Point page for "E2E Agent Published Web App" from the Agent Roster + Then I should see the Agent v2 Web app access URL + And I record the Agent v2 orchestration draft for "E2E Agent Published Web App" + When I copy the Agent v2 Web app access URL + Then the Agent v2 Web app access URL should show it was copied + When I launch the Agent v2 Web app + Then the Agent v2 Web app should open in a new tab + When I open Agent v2 Embedded configuration + Then I should see the Agent v2 Embedded configuration dialog + When I open Agent v2 Web app customization + Then I should see the Agent v2 Web app customization dialog + When I open Agent v2 Web app settings + Then I should see the Agent v2 Web app settings dialog + And the Agent v2 orchestration draft for "E2E Agent Published Web App" should be unchanged + @workflow-reference Scenario: Workflow access shows the referencing workflow Given I am signed in as the default E2E admin diff --git a/e2e/features/step-definitions/agent-v2/access-point.steps.ts b/e2e/features/step-definitions/agent-v2/access-point.steps.ts index 0d0ad04d6bcab7..43a99eb7da1da3 100644 --- a/e2e/features/step-definitions/agent-v2/access-point.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point.steps.ts @@ -3,6 +3,7 @@ import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { getAgentAccessPath, + getAgentComposerDraft, getAgentReferencingWorkflows, setAgentApiAccess, } from '../../../support/agent' @@ -27,6 +28,19 @@ const getPreseededResource = (world: DifyWorld, name: string, kind: 'agent' | 'w return resource } +const getAccessRegion = (world: DifyWorld) => + world.getPage().getByRole('region', { name: 'Access Point' }) + +const getDialog = (world: DifyWorld, name: string | RegExp) => + world.getPage().getByRole('dialog', { name }) + +const closeDialog = async (world: DifyWorld, name: string | RegExp) => { + const dialog = getDialog(world, name) + + await dialog.getByLabel('Close').click() + await expect(dialog).not.toBeVisible() +} + Given( 'Agent v2 Backend service API access has been enabled via API', async function (this: DifyWorld) { @@ -96,6 +110,114 @@ Then('I should see the Agent v2 Access Point overview', async function (this: Di await expect(accessRegion.getByText('No workflow references yet.')).toBeVisible() }) +Then('I should see the Agent v2 Web app access URL', async function (this: DifyWorld) { + const accessRegion = getAccessRegion(this) + + await expect(accessRegion.getByRole('heading', { name: 'Web app' })).toBeVisible() + await expect(accessRegion.getByText('Access URL')).toBeVisible() + await expect(accessRegion.getByLabel('Copy access URL')).toBeEnabled() + await expect(accessRegion.getByRole('link', { name: 'Launch' })).toBeVisible() +}) + +Then( + 'I record the Agent v2 orchestration draft for {string}', + async function (this: DifyWorld, agentName: string) { + const agent = getPreseededResource(this, agentName, 'agent') + const draft = await getAgentComposerDraft(agent.id) + + this.lastAgentComposerDraftSnapshot = JSON.stringify(draft.agent_soul ?? {}) + }, +) + +When('I copy the Agent v2 Web app access URL', async function (this: DifyWorld) { + await getAccessRegion(this).getByLabel('Copy access URL').click() +}) + +Then('the Agent v2 Web app access URL should show it was copied', async function (this: DifyWorld) { + await expect(this.getPage().getByLabel('Copied')).toBeVisible() +}) + +When('I launch the Agent v2 Web app', async function (this: DifyWorld) { + const launchLink = getAccessRegion(this).getByRole('link', { name: 'Launch' }) + const href = await launchLink.getAttribute('href') + if (!href) + throw new Error('Agent v2 Web app Launch link does not expose an href.') + + const [webAppPage] = await Promise.all([ + this.getPage().waitForEvent('popup'), + launchLink.click(), + ]) + + this.lastAgentWebAppURL = href + this.lastAgentWebAppPage = webAppPage +}) + +Then('the Agent v2 Web app should open in a new tab', async function (this: DifyWorld) { + const webAppPage = this.lastAgentWebAppPage + const webAppURL = this.lastAgentWebAppURL + if (!webAppPage || !webAppURL) + throw new Error('No Agent v2 Web app page was opened.') + + await expect(webAppPage).toHaveURL(webAppURL) + await webAppPage.close() + this.lastAgentWebAppPage = undefined + this.lastAgentWebAppURL = undefined +}) + +When('I open Agent v2 Embedded configuration', async function (this: DifyWorld) { + await getAccessRegion(this).getByRole('button', { name: 'Embedded' }).click() +}) + +Then('I should see the Agent v2 Embedded configuration dialog', async function (this: DifyWorld) { + const dialog = getDialog(this, 'Embed on website') + + await expect(dialog).toBeVisible() + await expect(dialog.getByText('Embed on website')).toBeVisible() + await expect(dialog.getByText(/iframe|script/i)).toBeVisible() + await closeDialog(this, 'Embed on website') +}) + +When('I open Agent v2 Web app customization', async function (this: DifyWorld) { + await getAccessRegion(this).getByRole('button', { name: 'Customize' }).click() +}) + +Then('I should see the Agent v2 Web app customization dialog', async function (this: DifyWorld) { + const dialog = getDialog(this, 'Customize AI web app') + + await expect(dialog).toBeVisible() + await expect(dialog.getByText('Customize AI web app')).toBeVisible() + await expect(dialog.getByText(/NEXT_PUBLIC_APP_ID|NEXT_PUBLIC_API_URL/)).toBeVisible() + await closeDialog(this, 'Customize AI web app') +}) + +When('I open Agent v2 Web app settings', async function (this: DifyWorld) { + await getAccessRegion(this).getByRole('button', { name: 'Settings' }).click() +}) + +Then('I should see the Agent v2 Web app settings dialog', async function (this: DifyWorld) { + const dialog = getDialog(this, 'Web App Settings') + + await expect(dialog).toBeVisible() + await expect(dialog.getByText('Web App Settings')).toBeVisible() + await expect(dialog.getByText('web app Name')).toBeVisible() + await expect(dialog.getByText('web app Description')).toBeVisible() + await closeDialog(this, 'Web App Settings') +}) + +Then( + 'the Agent v2 orchestration draft for {string} should be unchanged', + async function (this: DifyWorld, agentName: string) { + const snapshot = this.lastAgentComposerDraftSnapshot + if (!snapshot) + throw new Error('No Agent v2 orchestration draft snapshot was recorded.') + + const agent = getPreseededResource(this, agentName, 'agent') + const draft = await getAgentComposerDraft(agent.id) + + expect(JSON.stringify(draft.agent_soul ?? {})).toBe(snapshot) + }, +) + Then( 'I should see the Agent v2 Workflow access reference for {string}', async function (this: DifyWorld, workflowName: string) { diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index c1961f8c2ed1ec..276c4cdee8e380 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -50,6 +50,9 @@ export class DifyWorld extends World { lastAgentServiceApiBaseURL: string | undefined lastGeneratedAgentApiKey: string | undefined lastAgentApiReferencePage: Page | undefined + lastAgentComposerDraftSnapshot: string | undefined + lastAgentWebAppPage: Page | undefined + lastAgentWebAppURL: string | undefined lastAgentWorkflowReferencePage: Page | undefined createdAppIds: string[] = [] createdAgentIds: string[] = [] @@ -80,6 +83,9 @@ export class DifyWorld extends World { this.lastAgentServiceApiBaseURL = undefined this.lastGeneratedAgentApiKey = undefined this.lastAgentApiReferencePage = undefined + this.lastAgentComposerDraftSnapshot = undefined + this.lastAgentWebAppPage = undefined + this.lastAgentWebAppURL = undefined this.lastAgentWorkflowReferencePage = undefined this.createdAppIds = [] this.createdAgentIds = [] From e5faacca2b9e3c81c7bc04d3294f054b866241c8 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 19:55:32 +0800 Subject: [PATCH 076/185] test(e2e): namespace agent builder world state --- e2e/AGENTS.md | 4 +- .../agent-v2/access-point.steps.ts | 42 ++++++++--------- .../agent-v2/configure.steps.ts | 32 ++++++------- .../agent-v2/preflight.steps.ts | 32 ++++++------- .../agent-v2/workflow-node.steps.ts | 10 ++-- e2e/features/support/world.ts | 46 ++++++++++--------- e2e/support/preflight.ts | 40 ++++++++-------- 7 files changed, 104 insertions(+), 102 deletions(-) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 9f7409d78186a4..db155b7d2f2a4d 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -298,11 +298,11 @@ Use `fixtures/test-materials/` for checked-in files that scenarios upload, previ Use `support/preflight.ts` for scenarios that require optional external resources such as a stable model provider, plugin/tool credential, or knowledge base seed. Prefer an explicit `Given` step that returns a skipped result with a clear blocked-precondition reason over hidden setup in hooks. -Use `the Agent Builder stable chat model is available` before scenarios that must run an Agent with a real model. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` define the single usable model for the E2E environment. The step defaults the type to `llm`, verifies the model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilderStableChatModel`. Any Agent Builder scenario that needs a usable model should explicitly apply this stored model during its own setup instead of hard-coding provider or model names in feature files or hooks. +Use `the Agent Builder stable chat model is available` before scenarios that must run an Agent with a real model. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` define the single usable model for the E2E environment. The step defaults the type to `llm`, verifies the model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilder.preflight.stableModel`. Any Agent Builder scenario that needs a usable model should explicitly apply this stored model during its own setup instead of hard-coding provider or model names in feature files or hooks. Use `the Agent Builder broken chat model is available` before model-recovery scenarios that intentionally start from an invalid model. The step requires `E2E_BROKEN_MODEL_PROVIDER`, defaults `E2E_BROKEN_MODEL_NAME` to `e2e-broken-model`, defaults `E2E_BROKEN_MODEL_TYPE` to `llm`, and only verifies that the model entry exists. The scenario must still assert the user-visible failure and recovery behavior. -Use `the Agent Builder preseeded Agent "{name}" is available`, `the Agent Builder preseeded workflow "{name}" is available`, `the Agent Builder preseeded dataset "{name}" is available`, and `the Agent Builder preseeded tool "{provider} / {tool}" is available` when a scenario depends on a fixed environment resource. These steps verify the resource through Console APIs, store the result in `DifyWorld.agentBuilderPreseededResources`, and return `skipped` with a blocked-precondition attachment when the resource is missing. The tool step checks built-in tool availability only; credential-validity scenarios still need explicit credential-state setup or assertions. +Use `the Agent Builder preseeded Agent "{name}" is available`, `the Agent Builder preseeded workflow "{name}" is available`, `the Agent Builder preseeded dataset "{name}" is available`, and `the Agent Builder preseeded tool "{provider} / {tool}" is available` when a scenario depends on a fixed environment resource. These steps verify the resource through Console APIs, store the result in `DifyWorld.agentBuilder.preflight.preseededResources`, and return `skipped` with a blocked-precondition attachment when the resource is missing. The tool step checks built-in tool availability only; credential-validity scenarios still need explicit credential-state setup or assertions. Use `the Agent Builder preseeded dataset "{name}" is indexed and ready` for knowledge retrieval scenarios that require a completed knowledge base. It verifies that the dataset exists, has documents, all listed documents are available, and every document indexing status is `completed`. Use `the Agent Builder preseeded dataset "{name}" is indexing` for failure-recovery scenarios that require an indexing or queued knowledge base; it verifies at least one document is in `waiting`, `parsing`, `cleaning`, `splitting`, or `indexing`. Fixed-content assertions such as `AGENT_KNOWLEDGE_PASS` belong in the dependent runtime scenario, where the user-visible Agent reply can prove retrieval actually hit the expected content. diff --git a/e2e/features/step-definitions/agent-v2/access-point.steps.ts b/e2e/features/step-definitions/agent-v2/access-point.steps.ts index 43a99eb7da1da3..d20e43f69931a4 100644 --- a/e2e/features/step-definitions/agent-v2/access-point.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point.steps.ts @@ -18,7 +18,7 @@ const getCurrentAgentId = (world: DifyWorld) => { } const getPreseededResource = (world: DifyWorld, name: string, kind: 'agent' | 'workflow') => { - const resource = world.agentBuilderPreseededResources[name] + const resource = world.agentBuilder.preflight.preseededResources[name] if (!resource || resource.kind !== kind) { throw new Error( `Preseeded ${kind} "${name}" is not available. Run the matching preflight step first.`, @@ -46,7 +46,7 @@ Given( async function (this: DifyWorld) { const apiAccess = await setAgentApiAccess(getCurrentAgentId(this), true) - this.lastAgentServiceApiBaseURL = apiAccess.service_api_base_url + this.agentBuilder.accessPoint.serviceApiBaseURL = apiAccess.service_api_base_url }, ) @@ -125,7 +125,7 @@ Then( const agent = getPreseededResource(this, agentName, 'agent') const draft = await getAgentComposerDraft(agent.id) - this.lastAgentComposerDraftSnapshot = JSON.stringify(draft.agent_soul ?? {}) + this.agentBuilder.accessPoint.composerDraftSnapshot = JSON.stringify(draft.agent_soul ?? {}) }, ) @@ -148,20 +148,20 @@ When('I launch the Agent v2 Web app', async function (this: DifyWorld) { launchLink.click(), ]) - this.lastAgentWebAppURL = href - this.lastAgentWebAppPage = webAppPage + this.agentBuilder.accessPoint.webAppURL = href + this.agentBuilder.accessPoint.webAppPage = webAppPage }) Then('the Agent v2 Web app should open in a new tab', async function (this: DifyWorld) { - const webAppPage = this.lastAgentWebAppPage - const webAppURL = this.lastAgentWebAppURL + const webAppPage = this.agentBuilder.accessPoint.webAppPage + const webAppURL = this.agentBuilder.accessPoint.webAppURL if (!webAppPage || !webAppURL) throw new Error('No Agent v2 Web app page was opened.') await expect(webAppPage).toHaveURL(webAppURL) await webAppPage.close() - this.lastAgentWebAppPage = undefined - this.lastAgentWebAppURL = undefined + this.agentBuilder.accessPoint.webAppPage = undefined + this.agentBuilder.accessPoint.webAppURL = undefined }) When('I open Agent v2 Embedded configuration', async function (this: DifyWorld) { @@ -207,7 +207,7 @@ Then('I should see the Agent v2 Web app settings dialog', async function (this: Then( 'the Agent v2 orchestration draft for {string} should be unchanged', async function (this: DifyWorld, agentName: string) { - const snapshot = this.lastAgentComposerDraftSnapshot + const snapshot = this.agentBuilder.accessPoint.composerDraftSnapshot if (!snapshot) throw new Error('No Agent v2 orchestration draft snapshot was recorded.') @@ -265,14 +265,14 @@ When( workflowLink.click(), ]) - this.lastAgentWorkflowReferencePage = workflowPage + this.agentBuilder.accessPoint.workflowReferencePage = workflowPage }, ) Then( 'the Agent v2 Workflow access reference for {string} should open in Studio', async function (this: DifyWorld, workflowName: string) { - const workflowPage = this.lastAgentWorkflowReferencePage + const workflowPage = this.agentBuilder.accessPoint.workflowReferencePage if (!workflowPage) throw new Error('No Agent v2 Workflow access reference page was opened.') @@ -280,21 +280,21 @@ Then( await expect(workflowPage).toHaveURL(new RegExp(`/app/${workflow.id}/workflow(?:\\?.*)?$`)) await workflowPage.close() - this.lastAgentWorkflowReferencePage = undefined + this.agentBuilder.accessPoint.workflowReferencePage = undefined }, ) Then('I should see the Agent v2 Backend service API endpoint', async function (this: DifyWorld) { const page = this.getPage() - if (!this.lastAgentServiceApiBaseURL) + if (!this.agentBuilder.accessPoint.serviceApiBaseURL) throw new Error('No Agent v2 service API endpoint found. Enable Backend service API first.') await expect(page.getByRole('heading', { name: 'Backend service API' })).toBeVisible({ timeout: 30_000, }) await expect(page.getByText('Service API Endpoint')).toBeVisible() - await expect(page.getByText(this.lastAgentServiceApiBaseURL)).toBeVisible() + await expect(page.getByText(this.agentBuilder.accessPoint.serviceApiBaseURL)).toBeVisible() await expect(page.getByLabel('Copy service API endpoint')).toBeEnabled() }) @@ -348,8 +348,8 @@ Then('I should see the newly generated Agent v2 API key once', async function (t await expect(generatedKey).toBeVisible() await expect(generatedKeyDialog.getByLabel('Copy')).toBeVisible() - this.lastGeneratedAgentApiKey = (await generatedKey.textContent())?.trim() - if (!this.lastGeneratedAgentApiKey) + this.agentBuilder.accessPoint.generatedApiKey = (await generatedKey.textContent())?.trim() + if (!this.agentBuilder.accessPoint.generatedApiKey) throw new Error('Generated Agent v2 API key was empty.') }) @@ -364,7 +364,7 @@ When('I close the newly generated Agent v2 API key', async function (this: DifyW Then( 'the Agent v2 API key list should not expose the full generated secret', async function (this: DifyWorld) { - const fullSecret = this.lastGeneratedAgentApiKey + const fullSecret = this.agentBuilder.accessPoint.generatedApiKey if (!fullSecret) throw new Error('No generated Agent v2 API key found.') @@ -395,15 +395,15 @@ When('I open the Agent v2 API Reference', async function (this: DifyWorld) { apiReferenceLink.click(), ]) - this.lastAgentApiReferencePage = apiReferencePage + this.agentBuilder.accessPoint.apiReferencePage = apiReferencePage }) Then('the Agent v2 API Reference should open in a new tab', async function (this: DifyWorld) { - const apiReferencePage = this.lastAgentApiReferencePage + const apiReferencePage = this.agentBuilder.accessPoint.apiReferencePage if (!apiReferencePage) throw new Error('No Agent v2 API Reference page was opened.') await expect(apiReferencePage).toHaveURL(/developing-with-apis/) await apiReferencePage.close() - this.lastAgentApiReferencePage = undefined + this.agentBuilder.accessPoint.apiReferencePage = undefined }) diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 34dc352b0e5087..9b351c738ee56a 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -43,7 +43,7 @@ const getCurrentAgentId = (world: DifyWorld) => { } const getPreseededAgent = (world: DifyWorld, name: string) => { - const resource = world.agentBuilderPreseededResources[name] + const resource = world.agentBuilder.preflight.preseededResources[name] if (!resource || resource.kind !== 'agent') { throw new Error( `Preseeded Agent "${name}" is not available. Run the matching preflight step first.`, @@ -265,13 +265,13 @@ Given( ) Given('a runnable Agent v2 test agent has been created via API', async function (this: DifyWorld) { - if (!this.agentBuilderStableChatModel) + if (!this.agentBuilder.preflight.stableModel) throw new Error('Create a runnable Agent v2 test agent after stable model preflight.') const agent = await createConfiguredTestAgent({ agentSoul: createAgentSoulConfigWithModel( normalAgentSoulConfig, - this.agentBuilderStableChatModel, + this.agentBuilder.preflight.stableModel, ), }) this.createdAgentIds.push(agent.id) @@ -318,11 +318,11 @@ Given( throw new Error('Agent v2 build draft config file fixture did not return a file_id.') this.createdAgentConfigFiles.push({ agentId, name: configFile.name }) - const normalConfig = this.agentBuilderStableChatModel - ? createAgentSoulConfigWithModel(normalAgentSoulConfig, this.agentBuilderStableChatModel) + const normalConfig = this.agentBuilder.preflight.stableModel + ? createAgentSoulConfigWithModel(normalAgentSoulConfig, this.agentBuilder.preflight.stableModel) : normalAgentSoulConfig - const updatedConfig = this.agentBuilderStableChatModel - ? createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilderStableChatModel) + const updatedConfig = this.agentBuilder.preflight.stableModel + ? createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilder.preflight.stableModel) : updatedAgentSoulConfig await saveAgentComposerDraft(agentId, normalConfig) @@ -351,11 +351,11 @@ Given( 'an Agent v2 Build draft includes the existing e2e-summary-skill Skill', async function (this: DifyWorld) { const skill = await uploadSummaryConfigSkillForBuildDraft(this) - const normalConfig = this.agentBuilderStableChatModel - ? createAgentSoulConfigWithModel(normalAgentSoulConfig, this.agentBuilderStableChatModel) + const normalConfig = this.agentBuilder.preflight.stableModel + ? createAgentSoulConfigWithModel(normalAgentSoulConfig, this.agentBuilder.preflight.stableModel) : normalAgentSoulConfig - const updatedConfig = this.agentBuilderStableChatModel - ? createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilderStableChatModel) + const updatedConfig = this.agentBuilder.preflight.stableModel + ? createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilder.preflight.stableModel) : updatedAgentSoulConfig const configSkills = [{ ...skill, @@ -380,12 +380,12 @@ Given('an Agent v2 Build draft uses the updated E2E prompt', async function (thi Given( 'an Agent v2 Build draft uses the updated E2E prompt with the stable E2E model', async function (this: DifyWorld) { - if (!this.agentBuilderStableChatModel) + if (!this.agentBuilder.preflight.stableModel) throw new Error('Create an Agent v2 Build draft with a stable model after stable model preflight.') await saveAgentBuildDraft( getCurrentAgentId(this), - createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilderStableChatModel), + createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilder.preflight.stableModel), ) }, ) @@ -606,7 +606,7 @@ Then('I should see the Agent v2 configure workspace', async function (this: Dify Then('I should see the Agent v2 full-config fixture sections', async function (this: DifyWorld) { const page = this.getPage() - const stableModel = this.agentBuilderStableChatModel + const stableModel = this.agentBuilder.preflight.stableModel if (!stableModel) throw new Error('Stable chat model preflight must run before asserting the full-config Agent.') @@ -695,7 +695,7 @@ Then( Then( 'I should see the stable E2E model in the Agent v2 model selector', async function (this: DifyWorld) { - const stableModel = this.agentBuilderStableChatModel + const stableModel = this.agentBuilder.preflight.stableModel if (!stableModel) throw new Error('Stable chat model preflight must run before asserting the Agent model.') @@ -1125,7 +1125,7 @@ Then( Then( 'the Agent v2 draft should use the stable E2E model', async function (this: DifyWorld) { - const stableModel = this.agentBuilderStableChatModel + const stableModel = this.agentBuilder.preflight.stableModel if (!stableModel) throw new Error('Stable chat model preflight must run before asserting the Agent model.') diff --git a/e2e/features/step-definitions/agent-v2/preflight.steps.ts b/e2e/features/step-definitions/agent-v2/preflight.steps.ts index fbffb1548babb6..acce93c4f2eef0 100644 --- a/e2e/features/step-definitions/agent-v2/preflight.steps.ts +++ b/e2e/features/step-definitions/agent-v2/preflight.steps.ts @@ -24,7 +24,7 @@ Given('the Agent Builder stable chat model is available', async function (this: if (stableModel === 'skipped') return stableModel - this.agentBuilderStableChatModel = stableModel + this.agentBuilder.preflight.stableModel = stableModel }) Given('the Agent Builder broken chat model is available', async function (this: DifyWorld) { @@ -32,7 +32,7 @@ Given('the Agent Builder broken chat model is available', async function (this: if (brokenModel === 'skipped') return brokenModel - this.agentBuilderBrokenChatModel = brokenModel + this.agentBuilder.preflight.brokenModel = brokenModel }) Given( @@ -42,7 +42,7 @@ Given( if (resource === 'skipped') return resource - this.agentBuilderPreseededResources[resourceName] = resource + this.agentBuilder.preflight.preseededResources[resourceName] = resource }, ) @@ -53,7 +53,7 @@ Given( if (resource === 'skipped') return resource - this.agentBuilderPreseededResources[resourceName] = resource + this.agentBuilder.preflight.preseededResources[resourceName] = resource }, ) @@ -64,7 +64,7 @@ Given( if (resource === 'skipped') return resource - this.agentBuilderPreseededResources[resourceName] = resource + this.agentBuilder.preflight.preseededResources[resourceName] = resource }, ) @@ -75,7 +75,7 @@ Given( if (resource === 'skipped') return resource - this.agentBuilderPreseededResources[resourceName] = resource + this.agentBuilder.preflight.preseededResources[resourceName] = resource }, ) @@ -86,7 +86,7 @@ Given( if (resource === 'skipped') return resource - this.agentBuilderPreseededResources[resourceName] = resource + this.agentBuilder.preflight.preseededResources[resourceName] = resource }, ) @@ -97,7 +97,7 @@ Given( if (resource === 'skipped') return resource - this.agentBuilderPreseededResources[resourceName] = resource + this.agentBuilder.preflight.preseededResources[resourceName] = resource }, ) @@ -108,7 +108,7 @@ Given( if (resource === 'skipped') return resource - this.agentBuilderPreseededResources[`${agentName} / ${skillName}`] = resource + this.agentBuilder.preflight.preseededResources[`${agentName} / ${skillName}`] = resource }, ) @@ -119,7 +119,7 @@ Given( if (resource === 'skipped') return resource - this.agentBuilderPreseededResources[`${agentName} / core fixture configuration`] = resource + this.agentBuilder.preflight.preseededResources[`${agentName} / core fixture configuration`] = resource }, ) @@ -130,7 +130,7 @@ Given( if (resource === 'skipped') return resource - this.agentBuilderPreseededResources[`${agentName} / tool state fixture configuration`] + this.agentBuilder.preflight.preseededResources[`${agentName} / tool state fixture configuration`] = resource }, ) @@ -142,7 +142,7 @@ Given( if (resource === 'skipped') return resource - this.agentBuilderPreseededResources[`${agentName} / dual retrieval fixture configuration`] + this.agentBuilder.preflight.preseededResources[`${agentName} / dual retrieval fixture configuration`] = resource }, ) @@ -154,7 +154,7 @@ Given( if (resource === 'skipped') return resource - this.agentBuilderPreseededResources[`${agentName} / file tree fixture`] = resource + this.agentBuilder.preflight.preseededResources[`${agentName} / file tree fixture`] = resource }, ) @@ -165,7 +165,7 @@ Given( if (resource === 'skipped') return resource - this.agentBuilderPreseededResources[`${agentName} / Backend service API key`] = resource + this.agentBuilder.preflight.preseededResources[`${agentName} / Backend service API key`] = resource }, ) @@ -176,7 +176,7 @@ Given( if (resource === 'skipped') return resource - this.agentBuilderPreseededResources[`${agentName} / Web app`] = resource + this.agentBuilder.preflight.preseededResources[`${agentName} / Web app`] = resource }, ) @@ -187,6 +187,6 @@ Given( if (resource === 'skipped') return resource - this.agentBuilderPreseededResources[`${agentName} / ${workflowName}`] = resource + this.agentBuilder.preflight.preseededResources[`${agentName} / ${workflowName}`] = resource }, ) diff --git a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts index 67e5c4e82aa9a8..433d525de2cd2c 100644 --- a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts +++ b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts @@ -44,13 +44,13 @@ const getOutputVariablesFromDraft = async (appId: string) => { Given( 'a workflow app with an Agent v2 node has been created via API', async function (this: DifyWorld) { - if (!this.agentBuilderStableChatModel) + if (!this.agentBuilder.preflight.stableModel) throw new Error('Create an Agent v2 workflow node after stable model preflight.') const agent = await createConfiguredTestAgent({ agentSoul: createAgentSoulConfigWithModel( normalAgentSoulConfig, - this.agentBuilderStableChatModel, + this.agentBuilder.preflight.stableModel, ), }) this.createdAgentIds.push(agent.id) @@ -81,7 +81,7 @@ When( const page = this.getPage() const appId = getCurrentAppId(this) const rows = table.hashes() as AgentV2WorkflowOutputVariable[] - this.agentV2WorkflowOutputVariables = rows + this.agentBuilder.workflow.outputVariables = rows await page.getByRole('button', { name: 'Output Variables' }).click() @@ -111,7 +111,7 @@ Then( 'the Agent v2 workflow node output variables should be saved in the workflow draft', async function (this: DifyWorld) { const appId = getCurrentAppId(this) - const expectedOutputVariables = this.agentV2WorkflowOutputVariables + const expectedOutputVariables = this.agentBuilder.workflow.outputVariables if (expectedOutputVariables.length === 0) throw new Error('No Agent v2 workflow output variables were recorded for this scenario.') @@ -137,7 +137,7 @@ Then( Then('I should see the Agent v2 workflow node output variables', async function (this: DifyWorld) { const page = this.getPage() - const expectedOutputVariables = this.agentV2WorkflowOutputVariables + const expectedOutputVariables = this.agentBuilder.workflow.outputVariables if (expectedOutputVariables.length === 0) throw new Error('No Agent v2 workflow output variables were recorded for this scenario.') diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index 276c4cdee8e380..389e9ce88133b6 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -37,6 +37,28 @@ export type AgentV2WorkflowOutputVariable = { type: string } +export const createAgentBuilderWorldState = () => ({ + preflight: { + brokenModel: undefined as AgentBuilderStableChatModel | undefined, + preseededResources: {} as Record, + stableModel: undefined as AgentBuilderStableChatModel | undefined, + }, + accessPoint: { + apiReferencePage: undefined as Page | undefined, + composerDraftSnapshot: undefined as string | undefined, + generatedApiKey: undefined as string | undefined, + serviceApiBaseURL: undefined as string | undefined, + webAppPage: undefined as Page | undefined, + webAppURL: undefined as string | undefined, + workflowReferencePage: undefined as Page | undefined, + }, + workflow: { + outputVariables: [] as AgentV2WorkflowOutputVariable[], + }, +}) + +export type AgentBuilderWorldState = ReturnType + export class DifyWorld extends World { context: BrowserContext | undefined page: Page | undefined @@ -47,13 +69,6 @@ export class DifyWorld extends World { lastCreatedAppName: string | undefined lastCreatedAgentName: string | undefined lastCreatedAgentRole: string | undefined - lastAgentServiceApiBaseURL: string | undefined - lastGeneratedAgentApiKey: string | undefined - lastAgentApiReferencePage: Page | undefined - lastAgentComposerDraftSnapshot: string | undefined - lastAgentWebAppPage: Page | undefined - lastAgentWebAppURL: string | undefined - lastAgentWorkflowReferencePage: Page | undefined createdAppIds: string[] = [] createdAgentIds: string[] = [] createdDatasetIds: string[] = [] @@ -61,10 +76,7 @@ export class DifyWorld extends World { createdAgentConfigSkills: CreatedAgentConfigSkill[] = [] createdAgentDriveFiles: CreatedAgentDriveFile[] = [] createdBuiltinToolCredentials: CreatedBuiltinToolCredential[] = [] - agentBuilderBrokenChatModel: AgentBuilderStableChatModel | undefined - agentBuilderStableChatModel: AgentBuilderStableChatModel | undefined - agentBuilderPreseededResources: Record = {} - agentV2WorkflowOutputVariables: AgentV2WorkflowOutputVariable[] = [] + agentBuilder: AgentBuilderWorldState = createAgentBuilderWorldState() scenarioCleanups: ScenarioCleanup[] = [] capturedDownloads: Download[] = [] shareURL: string | undefined @@ -80,13 +92,6 @@ export class DifyWorld extends World { this.lastCreatedAppName = undefined this.lastCreatedAgentName = undefined this.lastCreatedAgentRole = undefined - this.lastAgentServiceApiBaseURL = undefined - this.lastGeneratedAgentApiKey = undefined - this.lastAgentApiReferencePage = undefined - this.lastAgentComposerDraftSnapshot = undefined - this.lastAgentWebAppPage = undefined - this.lastAgentWebAppURL = undefined - this.lastAgentWorkflowReferencePage = undefined this.createdAppIds = [] this.createdAgentIds = [] this.createdDatasetIds = [] @@ -94,10 +99,7 @@ export class DifyWorld extends World { this.createdAgentConfigSkills = [] this.createdAgentDriveFiles = [] this.createdBuiltinToolCredentials = [] - this.agentBuilderBrokenChatModel = undefined - this.agentBuilderStableChatModel = undefined - this.agentBuilderPreseededResources = {} - this.agentV2WorkflowOutputVariables = [] + this.agentBuilder = createAgentBuilderWorldState() this.scenarioCleanups = [] this.capturedDownloads = [] this.shareURL = undefined diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index e0924dd5d06810..126b7e449c4887 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -278,7 +278,7 @@ const hasUnauthorizedToolCredentialState = (item: unknown) => { const hasKnowledgeDataset = ( soul: Record, - dataset: NonNullable, + dataset: NonNullable, ) => { const knowledge = asRecord(soul.knowledge) const sets = asArray(knowledge.sets) @@ -295,7 +295,7 @@ const hasKnowledgeDataset = ( const hasKnowledgeSet = ( soul: Record, - dataset: NonNullable, + dataset: NonNullable, { queryMode, queryValue, @@ -351,7 +351,7 @@ const getDatasetIndexingStatuses = async (datasetId: string, resourceName: strin const toDatasetResource = ( resource: NamedResource, -): NonNullable => ({ +): NonNullable => ({ id: resource.id, kind: 'dataset', name: resource.name, @@ -377,7 +377,7 @@ const splitToolDisplayName = (resourceName: string) => { export async function skipMissingPreseededAgent( world: DifyWorld, resourceName: string, -): Promise<'skipped' | NonNullable> { +): Promise<'skipped' | NonNullable> { const query = buildQuery({ limit: '20', name: resourceName, page: '1' }) const resource = await findConsoleResourceByName({ action: `Check preseeded Agent ${resourceName}`, @@ -398,7 +398,7 @@ export async function skipMissingPreseededAgent( export async function skipMissingPreseededWorkflow( world: DifyWorld, resourceName: string, -): Promise<'skipped' | NonNullable> { +): Promise<'skipped' | NonNullable> { const query = buildQuery({ limit: '20', mode: 'workflow', name: resourceName, page: '1' }) const resource = await findConsoleResourceByName({ action: `Check preseeded workflow ${resourceName}`, @@ -419,7 +419,7 @@ export async function skipMissingPreseededWorkflow( export async function skipMissingPreseededDataset( world: DifyWorld, resourceName: string, -): Promise<'skipped' | NonNullable> { +): Promise<'skipped' | NonNullable> { const resource = await getPreseededDataset(resourceName) if (!resource) @@ -431,7 +431,7 @@ export async function skipMissingPreseededDataset( export async function skipMissingReadyPreseededDataset( world: DifyWorld, resourceName: string, -): Promise<'skipped' | NonNullable> { +): Promise<'skipped' | NonNullable> { const resource = await getPreseededDataset(resourceName) if (!resource) @@ -472,7 +472,7 @@ export async function skipMissingReadyPreseededDataset( export async function skipMissingIndexingPreseededDataset( world: DifyWorld, resourceName: string, -): Promise<'skipped' | NonNullable> { +): Promise<'skipped' | NonNullable> { const resource = await getPreseededDataset(resourceName) if (!resource) @@ -499,7 +499,7 @@ export async function skipMissingIndexingPreseededDataset( export async function skipMissingPreseededTool( world: DifyWorld, resourceName: string, -): Promise<'skipped' | NonNullable> { +): Promise<'skipped' | NonNullable> { const parsed = splitToolDisplayName(resourceName) if (!parsed.ok) return skipBlockedPrecondition(world, parsed.reason) @@ -534,7 +534,7 @@ export async function skipMissingPreseededAgentDriveSkill( world: DifyWorld, agentName: string, skillName: string, -): Promise<'skipped' | NonNullable> { +): Promise<'skipped' | NonNullable> { const agent = await skipMissingPreseededAgent(world, agentName) if (agent === 'skipped') return agent @@ -567,7 +567,7 @@ export async function skipMissingPreseededAgentDriveSkill( export async function skipMissingPreseededFullConfigAgentCoreConfiguration( world: DifyWorld, agentName: string, -): Promise<'skipped' | NonNullable> { +): Promise<'skipped' | NonNullable> { const stableModel = await skipMissingAgentBuilderStableChatModel(world) if (stableModel === 'skipped') return stableModel @@ -657,7 +657,7 @@ export async function skipMissingPreseededFullConfigAgentCoreConfiguration( export async function skipMissingPreseededToolStatesAgentConfiguration( world: DifyWorld, agentName: string, -): Promise<'skipped' | NonNullable> { +): Promise<'skipped' | NonNullable> { const agent = await skipMissingPreseededAgent(world, agentName) if (agent === 'skipped') return agent @@ -742,7 +742,7 @@ export async function skipMissingPreseededToolStatesAgentConfiguration( export async function skipMissingPreseededDualRetrievalAgentConfiguration( world: DifyWorld, agentName: string, -): Promise<'skipped' | NonNullable> { +): Promise<'skipped' | NonNullable> { const agent = await skipMissingPreseededAgent(world, agentName) if (agent === 'skipped') return agent @@ -791,7 +791,7 @@ export async function skipMissingPreseededDualRetrievalAgentConfiguration( export async function skipMissingPreseededAgentFileTreeFixture( world: DifyWorld, agentName: string, -): Promise<'skipped' | NonNullable> { +): Promise<'skipped' | NonNullable> { const agent = await skipMissingPreseededAgent(world, agentName) if (agent === 'skipped') return agent @@ -829,7 +829,7 @@ export async function skipMissingPreseededAgentFileTreeFixture( export async function skipMissingPreseededAgentBackendApiKey( world: DifyWorld, agentName: string, -): Promise<'skipped' | NonNullable> { +): Promise<'skipped' | NonNullable> { const agent = await skipMissingPreseededAgent(world, agentName) if (agent === 'skipped') return agent @@ -871,7 +871,7 @@ export async function skipMissingPreseededAgentBackendApiKey( export async function skipMissingPreseededAgentPublishedWebApp( world: DifyWorld, agentName: string, -): Promise<'skipped' | NonNullable> { +): Promise<'skipped' | NonNullable> { const agent = await skipMissingPreseededAgent(world, agentName) if (agent === 'skipped') return agent @@ -915,7 +915,7 @@ export async function skipMissingPreseededAgentWorkflowReference( world: DifyWorld, agentName: string, workflowName: string, -): Promise<'skipped' | NonNullable> { +): Promise<'skipped' | NonNullable> { const agent = await skipMissingPreseededAgent(world, agentName) if (agent === 'skipped') return agent @@ -1027,7 +1027,7 @@ async function skipMissingAgentBuilderModel( }: { requireActive: boolean }, -): Promise<'skipped' | NonNullable> { +): Promise<'skipped' | NonNullable> { if (!config.ok) return skipBlockedPrecondition(world, config.reason) @@ -1073,7 +1073,7 @@ async function skipMissingAgentBuilderModel( export async function skipMissingAgentBuilderStableChatModel( world: DifyWorld, -): Promise<'skipped' | NonNullable> { +): Promise<'skipped' | NonNullable> { return skipMissingAgentBuilderModel(world, readAgentBuilderStableChatModelConfig(), { requireActive: true, }) @@ -1081,7 +1081,7 @@ export async function skipMissingAgentBuilderStableChatModel( export async function skipMissingAgentBuilderBrokenChatModel( world: DifyWorld, -): Promise<'skipped' | NonNullable> { +): Promise<'skipped' | NonNullable> { return skipMissingAgentBuilderModel(world, readAgentBuilderBrokenChatModelConfig(), { requireActive: false, }) From f408fc11136fa4202a0b698864b627d7d8b490cf Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 20:01:07 +0800 Subject: [PATCH 077/185] test(e2e): cover web app access restore --- e2e/features/agent-v2/access-point.feature | 18 +++ .../agent-v2/access-point.steps.ts | 113 ++++++++++++++++-- e2e/support/agent.ts | 9 +- 3 files changed, 129 insertions(+), 11 deletions(-) diff --git a/e2e/features/agent-v2/access-point.feature b/e2e/features/agent-v2/access-point.feature index c418cfb209d1d8..85faef28d31aaa 100644 --- a/e2e/features/agent-v2/access-point.feature +++ b/e2e/features/agent-v2/access-point.feature @@ -27,6 +27,24 @@ Feature: Agent v2 Access Point Then I should see the Agent v2 Web app settings dialog And the Agent v2 orchestration draft for "E2E Agent Published Web App" should be unchanged + @web-app-access + Scenario: Web app access can be disabled and restored + Given I am signed in as the default E2E admin + And the Agent Builder preseeded Agent "E2E Agent Published Web App" is available + And the Agent Builder preseeded Agent "E2E Agent Published Web App" has published Web app access + And Agent v2 Web app access will be restored for "E2E Agent Published Web App" + When I open the preseeded Agent v2 Access Point page for "E2E Agent Published Web App" from the Agent Roster + And I disable Agent v2 Web app access + Then Agent v2 Web app access should be out of service + When I open the disabled Agent v2 Web app URL + Then the disabled Agent v2 Web app should show an unavailable state + When I enable Agent v2 Web app access + Then Agent v2 Web app access should be in service + When I open the restored Agent v2 Web app URL + Then the restored Agent v2 Web app should not show an unavailable state + When I refresh the current page + Then Agent v2 Web app access should be in service + @workflow-reference Scenario: Workflow access shows the referencing workflow Given I am signed in as the default E2E admin diff --git a/e2e/features/step-definitions/agent-v2/access-point.steps.ts b/e2e/features/step-definitions/agent-v2/access-point.steps.ts index d20e43f69931a4..39f66ea282d345 100644 --- a/e2e/features/step-definitions/agent-v2/access-point.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point.steps.ts @@ -6,6 +6,7 @@ import { getAgentComposerDraft, getAgentReferencingWorkflows, setAgentApiAccess, + setAgentSiteAccessAndGetURL, } from '../../../support/agent' import { agentBuilderPreseededResources } from '../../../support/agent-builder-resources' @@ -31,6 +32,9 @@ const getPreseededResource = (world: DifyWorld, name: string, kind: 'agent' | 'w const getAccessRegion = (world: DifyWorld) => world.getPage().getByRole('region', { name: 'Access Point' }) +const getWebAppCard = (world: DifyWorld) => + getAccessRegion(world).locator('article').filter({ hasText: 'Web app' }).first() + const getDialog = (world: DifyWorld, name: string | RegExp) => world.getPage().getByRole('dialog', { name }) @@ -50,6 +54,17 @@ Given( }, ) +Given( + 'Agent v2 Web app access will be restored for {string}', + async function (this: DifyWorld, agentName: string) { + const agent = getPreseededResource(this, agentName, 'agent') + + this.registerCleanup(async () => { + await setAgentSiteAccessAndGetURL(agent.id, true) + }) + }, +) + When('I open the Agent v2 Access Point page', async function (this: DifyWorld) { await this.getPage().goto(getAgentAccessPath(getCurrentAgentId(this))) }) @@ -111,12 +126,12 @@ Then('I should see the Agent v2 Access Point overview', async function (this: Di }) Then('I should see the Agent v2 Web app access URL', async function (this: DifyWorld) { - const accessRegion = getAccessRegion(this) + const webAppCard = getWebAppCard(this) - await expect(accessRegion.getByRole('heading', { name: 'Web app' })).toBeVisible() - await expect(accessRegion.getByText('Access URL')).toBeVisible() - await expect(accessRegion.getByLabel('Copy access URL')).toBeEnabled() - await expect(accessRegion.getByRole('link', { name: 'Launch' })).toBeVisible() + await expect(webAppCard.getByRole('heading', { name: 'Web app' })).toBeVisible() + await expect(webAppCard.getByText('Access URL')).toBeVisible() + await expect(webAppCard.getByLabel('Copy access URL')).toBeEnabled() + await expect(webAppCard.getByRole('link', { name: 'Launch' })).toBeVisible() }) Then( @@ -130,7 +145,7 @@ Then( ) When('I copy the Agent v2 Web app access URL', async function (this: DifyWorld) { - await getAccessRegion(this).getByLabel('Copy access URL').click() + await getWebAppCard(this).getByLabel('Copy access URL').click() }) Then('the Agent v2 Web app access URL should show it was copied', async function (this: DifyWorld) { @@ -138,7 +153,7 @@ Then('the Agent v2 Web app access URL should show it was copied', async function }) When('I launch the Agent v2 Web app', async function (this: DifyWorld) { - const launchLink = getAccessRegion(this).getByRole('link', { name: 'Launch' }) + const launchLink = getWebAppCard(this).getByRole('link', { name: 'Launch' }) const href = await launchLink.getAttribute('href') if (!href) throw new Error('Agent v2 Web app Launch link does not expose an href.') @@ -165,7 +180,7 @@ Then('the Agent v2 Web app should open in a new tab', async function (this: Dify }) When('I open Agent v2 Embedded configuration', async function (this: DifyWorld) { - await getAccessRegion(this).getByRole('button', { name: 'Embedded' }).click() + await getWebAppCard(this).getByRole('button', { name: 'Embedded' }).click() }) Then('I should see the Agent v2 Embedded configuration dialog', async function (this: DifyWorld) { @@ -178,7 +193,7 @@ Then('I should see the Agent v2 Embedded configuration dialog', async function ( }) When('I open Agent v2 Web app customization', async function (this: DifyWorld) { - await getAccessRegion(this).getByRole('button', { name: 'Customize' }).click() + await getWebAppCard(this).getByRole('button', { name: 'Customize' }).click() }) Then('I should see the Agent v2 Web app customization dialog', async function (this: DifyWorld) { @@ -191,7 +206,7 @@ Then('I should see the Agent v2 Web app customization dialog', async function (t }) When('I open Agent v2 Web app settings', async function (this: DifyWorld) { - await getAccessRegion(this).getByRole('button', { name: 'Settings' }).click() + await getWebAppCard(this).getByRole('button', { name: 'Settings' }).click() }) Then('I should see the Agent v2 Web app settings dialog', async function (this: DifyWorld) { @@ -218,6 +233,84 @@ Then( }, ) +When('I disable Agent v2 Web app access', async function (this: DifyWorld) { + const webAppCard = getWebAppCard(this) + const launchLink = webAppCard.getByRole('link', { name: 'Launch' }) + const href = await launchLink.getAttribute('href') + if (!href) + throw new Error('Agent v2 Web app Launch link does not expose an href.') + + this.agentBuilder.accessPoint.webAppURL = href + + await webAppCard.getByLabel('Toggle Web app access').click() +}) + +Then('Agent v2 Web app access should be out of service', async function (this: DifyWorld) { + const webAppCard = getWebAppCard(this) + + await expect(webAppCard.getByText('Out of service')).toBeVisible() + await expect(webAppCard.getByRole('button', { name: 'Launch' })).toBeDisabled() +}) + +When('I open the disabled Agent v2 Web app URL', async function (this: DifyWorld) { + const webAppURL = this.agentBuilder.accessPoint.webAppURL + if (!webAppURL) + throw new Error('No Agent v2 Web app URL was recorded.') + if (!this.context) + throw new Error('Playwright browser context has not been initialized.') + + const webAppPage = await this.context.newPage() + await webAppPage.goto(webAppURL) + + this.agentBuilder.accessPoint.webAppPage = webAppPage +}) + +Then('the disabled Agent v2 Web app should show an unavailable state', async function (this: DifyWorld) { + const webAppPage = this.agentBuilder.accessPoint.webAppPage + if (!webAppPage) + throw new Error('No Agent v2 Web app page was opened.') + + await expect(webAppPage.getByText(/app is unavailable|site is disabled/i)).toBeVisible({ + timeout: 30_000, + }) + await webAppPage.close() + this.agentBuilder.accessPoint.webAppPage = undefined +}) + +When('I enable Agent v2 Web app access', async function (this: DifyWorld) { + await getWebAppCard(this).getByLabel('Toggle Web app access').click() +}) + +Then('Agent v2 Web app access should be in service', async function (this: DifyWorld) { + const webAppCard = getWebAppCard(this) + + await expect(webAppCard.getByText('In service')).toBeVisible() + await expect(webAppCard.getByRole('link', { name: 'Launch' })).toBeVisible() +}) + +When('I open the restored Agent v2 Web app URL', async function (this: DifyWorld) { + const webAppURL = this.agentBuilder.accessPoint.webAppURL + if (!webAppURL) + throw new Error('No Agent v2 Web app URL was recorded.') + if (!this.context) + throw new Error('Playwright browser context has not been initialized.') + + const webAppPage = await this.context.newPage() + await webAppPage.goto(webAppURL) + + this.agentBuilder.accessPoint.webAppPage = webAppPage +}) + +Then('the restored Agent v2 Web app should not show an unavailable state', async function (this: DifyWorld) { + const webAppPage = this.agentBuilder.accessPoint.webAppPage + if (!webAppPage) + throw new Error('No Agent v2 Web app page was opened.') + + await expect(webAppPage.getByText(/app is unavailable|site is disabled/i)).not.toBeVisible() + await webAppPage.close() + this.agentBuilder.accessPoint.webAppPage = undefined +}) + Then( 'I should see the Agent v2 Workflow access reference for {string}', async function (this: DifyWorld, workflowName: string) { diff --git a/e2e/support/agent.ts b/e2e/support/agent.ts index 3be77f0dcd514e..11b57754c08f10 100644 --- a/e2e/support/agent.ts +++ b/e2e/support/agent.ts @@ -589,12 +589,19 @@ export async function publishAgent(agentId: string, versionNote = 'E2E publish') } export async function enableAgentSiteAndGetURL(agentId: string): Promise { + return setAgentSiteAccessAndGetURL(agentId, true) +} + +export async function setAgentSiteAccessAndGetURL( + agentId: string, + enabled: boolean, +): Promise { const agent = await getTestAgent(agentId) const appId = agent.app_id ?? agent.backing_app_id if (!appId) throw new Error(`Agent v2 ${agentId} does not expose a backing app ID.`) - const appDetail = await setAppSiteEnabled(appId, true) + const appDetail = await setAppSiteEnabled(appId, enabled) const token = agent.site?.access_token ?? agent.site?.code ?? appDetail.site.access_token const baseURL = agent.site?.app_base_url ?? appDetail.site.app_base_url From 1d93b7c47b0d3d7f49b863f8d2b72e9a7ba1a2b7 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 20:11:42 +0800 Subject: [PATCH 078/185] test(e2e): cover current flat agent files --- e2e/AGENTS.md | 2 + e2e/features/agent-v2/agent-edit.feature | 8 ++++ .../agent-v2/configure.steps.ts | 26 +++++++++++ .../agent-v2/preflight.steps.ts | 13 ++++++ e2e/support/preflight.ts | 45 ++++++++++++++++++- e2e/support/test-materials.ts | 3 ++ 6 files changed, 96 insertions(+), 1 deletion(-) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index db155b7d2f2a4d..90f741d4f34972 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -316,6 +316,8 @@ Use `the Agent Builder preseeded Agent "{agent}" includes the dual retrieval fix Use `the Agent Builder preseeded Agent "{agent}" includes the file tree fixture files` for file-tree display prerequisites. It verifies the Agent drive contains every file from `agentBuilderFileTreeFixtureFiles` through `/console/api/agent/{agent_id}/drive/files?prefix=files/`. This proves the committed drive file set is ready; keep visual tree expansion, nesting, and preview assertions in dependent user-visible scenarios. +Use `the Agent Builder preseeded Agent "{agent}" includes the current flat file fixture configuration` for the current Agent Edit Files section. Agent config files are still a flat `config_files` list and reject path separators, so this preflight verifies the fixture file basenames are present in the Agent Soul. Treat this as partial coverage for tree-display requirements until the product supports hierarchical config files in the visible Files section. + Use `the Agent Builder preseeded Agent "{agent}" has published Web app access` to verify that a fixed Agent is published, Web app access is enabled, and the Agent detail response includes the site token and base URL needed to open the Web app. Keep Web app launch and runtime assertions in dependent user-visible scenarios. Use `the Agent Builder preseeded Agent "{agent}" is referenced by workflow "{workflow}"` to verify Workflow access prerequisites. It checks both fixed resources exist, then uses `/console/api/agent/{agent_id}/referencing-workflows`, the same Console API used by the Access Point Workflow references table, to verify the workflow references the Agent through at least one published Agent node. diff --git a/e2e/features/agent-v2/agent-edit.feature b/e2e/features/agent-v2/agent-edit.feature index 5f15c76f9c6197..d7e1d05c225eb8 100644 --- a/e2e/features/agent-v2/agent-edit.feature +++ b/e2e/features/agent-v2/agent-edit.feature @@ -14,3 +14,11 @@ Feature: Agent v2 Agent Edit page And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" includes the tool state fixture configuration When I open the preseeded Agent v2 configure page for "E2E New Agent Builder Tool States" from the Agent Roster Then I should see the Agent v2 tool state fixture tools + + Scenario: File fixture entries are visible in the current flat Files list + Given I am signed in as the default E2E admin + And the Agent Builder preseeded Agent "E2E Agent With File Tree" is available + And the Agent Builder preseeded Agent "E2E Agent With File Tree" includes the file tree fixture files + And the Agent Builder preseeded Agent "E2E Agent With File Tree" includes the current flat file fixture configuration + When I open the preseeded Agent v2 configure page for "E2E Agent With File Tree" from the Agent Roster + Then I should see the Agent v2 file fixture entries in the current flat Files list diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 9b351c738ee56a..4db0713527de2f 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -30,6 +30,7 @@ import { import { waitForAgentConfigureAutosaved } from '../../../support/agent-configure' import { skipBlockedPrecondition } from '../../../support/preflight' import { + agentBuilderFileTreeFixtureFileNames, agentBuilderTestMaterials, getAgentBuilderTestMaterialPath, } from '../../../support/test-materials' @@ -682,6 +683,31 @@ Then('I should see the Agent v2 tool state fixture tools', async function (this: ) }) +Then( + 'I should see the Agent v2 file fixture entries in the current flat Files list', + async function (this: DifyWorld) { + const page = this.getPage() + const filesSection = page.getByRole('region', { name: 'Files' }) + const filesList = filesSection.getByLabel('Agent files') + + await expect(filesSection).toBeVisible({ timeout: 30_000 }) + await expect(filesList).toBeVisible() + + for (const fileName of agentBuilderFileTreeFixtureFileNames) { + await expect(filesList.getByRole('button', { + exact: true, + name: fileName, + })).toBeVisible() + } + + await expect(filesList.getByRole('button', { exact: true, name: 'assets' })).toHaveCount(0) + await expect(filesList.getByRole('button', { exact: true, name: 'docs' })).toHaveCount(0) + await expect(filesList.getByRole('button', { exact: true, name: 'public' })).toHaveCount(0) + await expect(filesList.getByRole('button', { exact: true, name: 'src' })).toHaveCount(0) + await expect(filesList.getByRole('button', { exact: true, name: 'web-game' })).toHaveCount(0) + }, +) + Then( 'I should see the normal E2E prompt in the Agent v2 prompt editor', async function (this: DifyWorld) { diff --git a/e2e/features/step-definitions/agent-v2/preflight.steps.ts b/e2e/features/step-definitions/agent-v2/preflight.steps.ts index acce93c4f2eef0..86a499061c2e5c 100644 --- a/e2e/features/step-definitions/agent-v2/preflight.steps.ts +++ b/e2e/features/step-definitions/agent-v2/preflight.steps.ts @@ -8,6 +8,7 @@ import { skipMissingPreseededAgentBackendApiKey, skipMissingPreseededAgentDriveSkill, skipMissingPreseededAgentFileTreeFixture, + skipMissingPreseededAgentFlatFileFixtureConfiguration, skipMissingPreseededAgentPublishedWebApp, skipMissingPreseededAgentWorkflowReference, skipMissingPreseededDataset, @@ -158,6 +159,18 @@ Given( }, ) +Given( + 'the Agent Builder preseeded Agent {string} includes the current flat file fixture configuration', + async function (this: DifyWorld, agentName: string) { + const resource = await skipMissingPreseededAgentFlatFileFixtureConfiguration(this, agentName) + if (resource === 'skipped') + return resource + + this.agentBuilder.preflight.preseededResources[`${agentName} / flat file fixture configuration`] + = resource + }, +) + Given( 'the Agent Builder preseeded Agent {string} has Backend service API access with an API key', async function (this: DifyWorld, agentName: string) { diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index 126b7e449c4887..4fa6f6fa86031d 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -5,7 +5,11 @@ import { agentBuilderPreseededResources, } from './agent-builder-resources' import { createApiContext, expectApiResponseOK } from './api' -import { agentBuilderFileTreeFixtureFiles, agentBuilderTestMaterials } from './test-materials' +import { + agentBuilderFileTreeFixtureFileNames, + agentBuilderFileTreeFixtureFiles, + agentBuilderTestMaterials, +} from './test-materials' const stableChatModelProviderEnv = 'E2E_STABLE_MODEL_PROVIDER' const stableChatModelNameEnv = 'E2E_STABLE_MODEL_NAME' @@ -826,6 +830,45 @@ export async function skipMissingPreseededAgentFileTreeFixture( } } +export async function skipMissingPreseededAgentFlatFileFixtureConfiguration( + world: DifyWorld, + agentName: string, +): Promise<'skipped' | NonNullable> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) + await expectApiResponseOK(response, `Check preseeded Agent flat file fixture ${agentName}`) + const body = (await response.json()) as AgentComposerResponse + const configFiles = Array.isArray(body.agent_soul?.config_files) + ? body.agent_soul.config_files + : [] + const fileNames = configFiles + .map(file => (typeof file === 'object' && file !== null && 'name' in file ? file.name : undefined)) + .filter((name): name is string => typeof name === 'string') + const missingFiles = agentBuilderFileTreeFixtureFileNames.filter(fileName => !fileNames.includes(fileName)) + + if (missingFiles.length > 0) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" is missing current flat Files fixture configuration: ${missingFiles.join(', ')}. Hierarchical Files display remains blocked until Agent config files support tree paths.`, + ) + } + + return { + id: agent.id, + kind: 'agent', + name: agent.name, + } + } + finally { + await ctx.dispose() + } +} + export async function skipMissingPreseededAgentBackendApiKey( world: DifyWorld, agentName: string, diff --git a/e2e/support/test-materials.ts b/e2e/support/test-materials.ts index 49f87a2eadf76b..031d31d51f5d95 100644 --- a/e2e/support/test-materials.ts +++ b/e2e/support/test-materials.ts @@ -42,6 +42,9 @@ export const agentBuilderFileTreeFixtureFiles = [ 'web-game/README.md', ] as const +export const agentBuilderFileTreeFixtureFileNames = agentBuilderFileTreeFixtureFiles + .map(filePath => path.basename(filePath)) + export const getAgentBuilderTestMaterialPath = (material: keyof typeof agentBuilderTestMaterials) => getTestMaterialPath(agentBuilderTestMaterials[material]) From 6ae083964515b839b6f2765ce37a1432ec786755 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 20:15:24 +0800 Subject: [PATCH 079/185] test(e2e): cover dual retrieval settings --- e2e/features/agent-v2/agent-edit.feature | 7 +++ .../agent-v2/configure.steps.ts | 48 +++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/e2e/features/agent-v2/agent-edit.feature b/e2e/features/agent-v2/agent-edit.feature index d7e1d05c225eb8..5dfcbf52d0c150 100644 --- a/e2e/features/agent-v2/agent-edit.feature +++ b/e2e/features/agent-v2/agent-edit.feature @@ -22,3 +22,10 @@ Feature: Agent v2 Agent Edit page And the Agent Builder preseeded Agent "E2E Agent With File Tree" includes the current flat file fixture configuration When I open the preseeded Agent v2 configure page for "E2E Agent With File Tree" from the Agent Roster Then I should see the Agent v2 file fixture entries in the current flat Files list + + Scenario: Dual Knowledge Retrieval settings are visible on the Agent Edit page + Given I am signed in as the default E2E admin + And the Agent Builder preseeded Agent "E2E Agent With Dual Retrieval" is available + And the Agent Builder preseeded Agent "E2E Agent With Dual Retrieval" includes the dual retrieval fixture configuration + When I open the preseeded Agent v2 configure page for "E2E Agent With Dual Retrieval" from the Agent Roster + Then I should see the Agent v2 dual retrieval fixture settings diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 4db0713527de2f..d3f766f64723ee 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -248,6 +248,21 @@ const expectProviderToolActionVisible = async ( return { action, tool } } +const openAgentKnowledgeRetrievalDialog = async (knowledgeSection: Locator, name: string) => { + await knowledgeSection.getByText(name, { exact: true }).hover() + await knowledgeSection.getByRole('button', { + exact: true, + name: `Edit ${name}`, + }).click() + + const dialog = knowledgeSection.page().getByRole('dialog', { + name: 'Knowledge Retrieval · Agent decide', + }) + await expect(dialog).toBeVisible() + + return dialog +} + Given('an Agent v2 test agent has been created via API', async function (this: DifyWorld) { const agent = await createTestAgent() this.createdAgentIds.push(agent.id) @@ -708,6 +723,39 @@ Then( }, ) +Then('I should see the Agent v2 dual retrieval fixture settings', async function (this: DifyWorld) { + const page = this.getPage() + const knowledgeSection = page.getByRole('region', { name: 'Knowledge Retrieval' }) + + await expect(knowledgeSection).toBeVisible({ timeout: 30_000 }) + await expect(knowledgeSection.getByText('Retrieval 1', { exact: true })).toBeVisible() + await expect(knowledgeSection.getByText('Retrieval 2', { exact: true })).toBeVisible() + + const agentDecideDialog = await openAgentKnowledgeRetrievalDialog(knowledgeSection, 'Retrieval 1') + await expect(agentDecideDialog.getByText(agentBuilderPreseededResources.agentKnowledgeBase, { + exact: true, + })).toBeVisible() + await expect(agentDecideDialog.getByRole('radio', { + exact: true, + name: 'Agent decide', + })).toBeChecked() + await agentDecideDialog.getByRole('button', { name: 'Close' }).click() + await expect(agentDecideDialog).not.toBeVisible() + + const customQueryDialog = await openAgentKnowledgeRetrievalDialog(knowledgeSection, 'Retrieval 2') + await expect(customQueryDialog.getByText(agentBuilderPreseededResources.agentKnowledgeBase, { + exact: true, + })).toBeVisible() + await expect(customQueryDialog.getByRole('radio', { + exact: true, + name: 'Custom query', + })).toBeChecked() + await expect(customQueryDialog.getByRole('textbox', { + exact: true, + name: 'Custom query text', + })).toHaveValue(agentBuilderFixedInputs.customKnowledgeQuery) +}) + Then( 'I should see the normal E2E prompt in the Agent v2 prompt editor', async function (this: DifyWorld) { From e17f92468da8100c11ac1e94b46d909d898fbe8a Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 20:19:08 +0800 Subject: [PATCH 080/185] test(e2e): simplify advanced settings precondition --- e2e/features/agent-v2/advanced-settings.feature | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/e2e/features/agent-v2/advanced-settings.feature b/e2e/features/agent-v2/advanced-settings.feature index f8cdd2cb319dab..12db4c28f0c3bc 100644 --- a/e2e/features/agent-v2/advanced-settings.feature +++ b/e2e/features/agent-v2/advanced-settings.feature @@ -2,8 +2,7 @@ Feature: Agent v2 advanced settings Scenario: Advanced Settings exposes supported configuration entries Given I am signed in as the default E2E admin - And the Agent Builder stable chat model is available - And a runnable Agent v2 test agent has been created via API + And a basic configured Agent v2 test agent has been created via API When I open the Agent v2 configure page Then Agent v2 Advanced Settings should describe supported entries while collapsed When I expand Agent v2 Advanced Settings From 7ea2a428939c31eed8eee81857b80596df43ea27 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 20:24:21 +0800 Subject: [PATCH 081/185] test(e2e): simplify env editor preconditions --- e2e/features/agent-v2/advanced-settings.feature | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/e2e/features/agent-v2/advanced-settings.feature b/e2e/features/agent-v2/advanced-settings.feature index 12db4c28f0c3bc..2e7a1edbd2546a 100644 --- a/e2e/features/agent-v2/advanced-settings.feature +++ b/e2e/features/agent-v2/advanced-settings.feature @@ -10,8 +10,7 @@ Feature: Agent v2 advanced settings Scenario: Plain environment variables are saved and restored Given I am signed in as the default E2E admin - And the Agent Builder stable chat model is available - And a runnable Agent v2 test agent has been created via API + And a basic configured Agent v2 test agent has been created via API When I open the Agent v2 configure page And I add the plain Agent v2 environment variable from Advanced Settings Then the plain Agent v2 environment variable should be saved in the Agent v2 draft @@ -21,8 +20,7 @@ Feature: Agent v2 advanced settings Scenario: Valid environment imports are saved and restored Given I am signed in as the default E2E admin - And the Agent Builder stable chat model is available - And a runnable Agent v2 test agent has been created via API + And a basic configured Agent v2 test agent has been created via API When I open the Agent v2 configure page And I import the valid Agent v2 environment file from Advanced Settings Then the valid Agent v2 environment import should be saved in the Agent v2 draft @@ -32,8 +30,7 @@ Feature: Agent v2 advanced settings Scenario: Deleted environment variables are removed after refresh Given I am signed in as the default E2E admin - And the Agent Builder stable chat model is available - And a runnable Agent v2 test agent has been created via API + And a basic configured Agent v2 test agent has been created via API When I open the Agent v2 configure page And I add the plain Agent v2 environment variable from Advanced Settings And I add the secondary plain Agent v2 environment variable from Advanced Settings @@ -46,8 +43,7 @@ Feature: Agent v2 advanced settings Scenario: Invalid environment imports report skipped lines and keep existing variables Given I am signed in as the default E2E admin - And the Agent Builder stable chat model is available - And a runnable Agent v2 test agent has been created via API + And a basic configured Agent v2 test agent has been created via API When I open the Agent v2 configure page And I add the plain Agent v2 environment variable from Advanced Settings Then the plain Agent v2 environment variable should be saved in the Agent v2 draft From 2b8a80c4e2dcf02bd3417e1fbbb11f386fee149e Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 20:37:58 +0800 Subject: [PATCH 082/185] test(e2e): split agent builder glue --- e2e/AGENTS.md | 4 +- .../agent-v2/advanced-settings.steps.ts | 383 ++++++ .../agent-v2/agent-edit.steps.ts | 117 ++ .../agent-v2/build-draft.steps.ts | 291 +++++ .../agent-v2/configure-helpers.ts | 241 ++++ .../agent-v2/configure.steps.ts | 1086 +--------------- .../step-definitions/agent-v2/files.steps.ts | 67 + .../agent-v2/output-variables.steps.ts | 15 + .../agent-v2/publish.steps.ts | 27 + e2e/support/preflight.ts | 1132 +---------------- e2e/support/preflight/access.ts | 166 +++ e2e/support/preflight/agents.ts | 472 +++++++ e2e/support/preflight/common.ts | 128 ++ e2e/support/preflight/datasets.ts | 150 +++ e2e/support/preflight/index.ts | 6 + e2e/support/preflight/models.ts | 158 +++ e2e/support/preflight/tools.ts | 114 ++ 17 files changed, 2344 insertions(+), 2213 deletions(-) create mode 100644 e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts create mode 100644 e2e/features/step-definitions/agent-v2/agent-edit.steps.ts create mode 100644 e2e/features/step-definitions/agent-v2/build-draft.steps.ts create mode 100644 e2e/features/step-definitions/agent-v2/configure-helpers.ts create mode 100644 e2e/features/step-definitions/agent-v2/files.steps.ts create mode 100644 e2e/features/step-definitions/agent-v2/output-variables.steps.ts create mode 100644 e2e/features/step-definitions/agent-v2/publish.steps.ts create mode 100644 e2e/support/preflight/access.ts create mode 100644 e2e/support/preflight/agents.ts create mode 100644 e2e/support/preflight/common.ts create mode 100644 e2e/support/preflight/datasets.ts create mode 100644 e2e/support/preflight/index.ts create mode 100644 e2e/support/preflight/models.ts create mode 100644 e2e/support/preflight/tools.ts diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 90f741d4f34972..1159f47abb90a1 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -296,7 +296,9 @@ Use `support/naming.ts` for generated test resource names. New app, Agent, datas Use `fixtures/test-materials/` for checked-in files that scenarios upload, preview, index, or retrieve. Keep these fixtures small and deterministic, and use `support/test-materials.ts` to resolve their absolute paths. -Use `support/preflight.ts` for scenarios that require optional external resources such as a stable model provider, plugin/tool credential, or knowledge base seed. Prefer an explicit `Given` step that returns a skipped result with a clear blocked-precondition reason over hidden setup in hooks. +Use `support/preflight.ts` for scenarios that require optional external resources such as a stable model provider, plugin/tool credential, or knowledge base seed. Prefer an explicit `Given` step that returns a skipped result with a clear blocked-precondition reason over hidden setup in hooks. Keep `support/preflight.ts` as the public barrel and put resource checks in the matching module under `support/preflight/`: `models.ts`, `agents.ts`, `datasets.ts`, `tools.ts`, or `access.ts`. + +Keep Agent Builder step definitions grouped by user capability, not by DOM component or Cucumber keyword. `configure.steps.ts` owns common configure navigation, refresh, and draft assertions; `build-draft.steps.ts` owns Build mode checkout, apply, discard, and Build draft isolation; `files.steps.ts` owns Files upload and file-list assertions; `advanced-settings.steps.ts` owns Env and advanced settings behavior; `agent-edit.steps.ts` owns saved Agent detail display assertions; `publish.steps.ts` owns publish state; `access-point.steps.ts` owns Access Point behavior; `preflight.steps.ts` should remain the explicit `Given` entrypoint for preflight resources. Use `the Agent Builder stable chat model is available` before scenarios that must run an Agent with a real model. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` define the single usable model for the E2E environment. The step defaults the type to `llm`, verifies the model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilder.preflight.stableModel`. Any Agent Builder scenario that needs a usable model should explicitly apply this stored model during its own setup instead of hard-coding provider or model names in feature files or hooks. diff --git a/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts b/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts new file mode 100644 index 00000000000000..80041582424750 --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts @@ -0,0 +1,383 @@ +import type { DifyWorld } from '../../support/world' +import { Then, When } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { getAgentComposerDraft } from '../../../support/agent' +import { agentBuilderFixedInputs } from '../../../support/agent-builder-resources' +import { skipBlockedPrecondition } from '../../../support/preflight' +import { getAgentBuilderTestMaterialPath } from '../../../support/test-materials' +import { + expectAgentEnvVariableHidden, + expectAgentEnvVariableVisible, + getAgentEnvVariables, + getAgentEnvVariableValue, + getCurrentAgentId, + getEnvVariableKey, + openAgentAdvancedSettings, +} from './configure-helpers' + +When( + 'I add the plain Agent v2 environment variable from Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() + + await advancedSettings + .getByRole('textbox', { name: 'Key' }) + .fill(agentBuilderFixedInputs.envPlainKey) + await advancedSettings + .getByRole('textbox', { name: 'Value' }) + .fill(agentBuilderFixedInputs.envPlainValue) + await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() + }, +) + +When( + 'I import the invalid Agent v2 environment file from Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + const fileChooserPromise = page.waitForEvent('filechooser') + await advancedSettings.getByRole('button', { name: 'Import .env' }).click() + await (await fileChooserPromise).setFiles(getAgentBuilderTestMaterialPath('invalidEnv')) + }, +) + +When( + 'I add the secondary plain Agent v2 environment variable from Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await advancedSettings.getByRole('button', { name: 'Add environment variable' }).click() + await advancedSettings + .getByRole('textbox', { name: 'Key' }) + .last() + .fill(agentBuilderFixedInputs.envModeKey) + await advancedSettings + .getByRole('textbox', { name: 'Value' }) + .last() + .fill(agentBuilderFixedInputs.envModeValue) + await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(2) + }, +) + +When( + 'I delete the plain Agent v2 environment variable from Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await advancedSettings + .getByRole('button', { name: `Delete ${agentBuilderFixedInputs.envPlainKey}` }) + .click() + }, +) + +When( + 'I import the valid Agent v2 environment file from Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() + + const fileChooserPromise = page.waitForEvent('filechooser') + await advancedSettings.getByRole('button', { name: 'Import .env' }).click() + await (await fileChooserPromise).setFiles(getAgentBuilderTestMaterialPath('validEnv')) + }, +) + +When('I expand Agent v2 Advanced Settings', async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() +}) + +Then( + 'Agent v2 Advanced Settings should describe supported entries while collapsed', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await expect(advancedSettings).toBeVisible() + await expect( + advancedSettings.getByText('For power users. Env vars, sandbox & memory.'), + ).toBeVisible() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })) + .not + .toBeVisible() + }, +) + +Then( + 'the plain Agent v2 environment variable should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect + .poll(async () => { + const env = (await getAgentComposerDraft(agentId)).agent_soul?.env + const variable = env?.variables?.find(item => + getEnvVariableKey(item) === agentBuilderFixedInputs.envPlainKey, + ) + + return { + secretCount: env?.secret_refs?.length ?? 0, + value: variable?.value, + } + }, { + timeout: 30_000, + }) + .toEqual({ + secretCount: 0, + value: agentBuilderFixedInputs.envPlainValue, + }) + }, +) + +Then( + 'the Agent v2 environment variables for deletion should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect + .poll(async () => { + const variables = await getAgentEnvVariables(agentId) + + return { + modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey), + plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), + } + }, { + timeout: 30_000, + }) + .toEqual({ + modeValue: agentBuilderFixedInputs.envModeValue, + plainValue: agentBuilderFixedInputs.envPlainValue, + }) + }, +) + +Then( + 'the plain Agent v2 environment variable should be removed from the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect + .poll(async () => { + const variables = await getAgentEnvVariables(agentId) + + return { + modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey), + plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), + } + }, { + timeout: 30_000, + }) + .toEqual({ + modeValue: agentBuilderFixedInputs.envModeValue, + plainValue: undefined, + }) + }, +) + +Then( + 'the valid Agent v2 environment import should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect + .poll(async () => { + const variables = await getAgentEnvVariables(agentId) + + return { + modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey), + plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), + } + }, { + timeout: 30_000, + }) + .toEqual({ + modeValue: agentBuilderFixedInputs.envModeValue, + plainValue: agentBuilderFixedInputs.envPlainValue, + }) + }, +) + +Then( + 'the invalid Agent v2 environment import should report skipped lines', + async function (this: DifyWorld) { + await expect(this.getPage().getByText('2 invalid .env lines were skipped.')).toBeVisible() + }, +) + +Then( + 'the Agent v2 environment variables from the invalid import should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect + .poll(async () => { + const variables = await getAgentEnvVariables(agentId) + + return { + importedValue: getAgentEnvVariableValue( + variables, + agentBuilderFixedInputs.envAfterInvalidImportKey, + ), + plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), + } + }, { + timeout: 30_000, + }) + .toEqual({ + importedValue: agentBuilderFixedInputs.envAfterInvalidImportValue, + plainValue: agentBuilderFixedInputs.envPlainValue, + }) + }, +) + +Then( + 'I should see the supported Agent v2 Advanced Settings entries', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + const envEditor = advancedSettings.getByRole('region', { name: 'Env Editor' }) + + await expect(envEditor).toBeVisible() + await expect(envEditor.getByRole('button', { name: 'Import .env' })).toBeVisible() + await expect(envEditor.getByRole('button', { name: 'Add environment variable' })) + .toBeVisible() + await expect(envEditor.getByText('Key', { exact: true })).toBeVisible() + await expect(envEditor.getByText('Value', { exact: true })).toBeVisible() + await expect(envEditor.getByText('Scope', { exact: true })).toBeVisible() + }, +) + +Then('Agent v2 Content Moderation Settings should be available', async function (this: DifyWorld) { + const advancedSettings = this.getPage().getByRole('region', { name: 'Advanced Settings' }) + const contentModeration = advancedSettings.getByRole('region', { name: 'Content moderation' }) + + try { + await expect(contentModeration).toBeVisible({ timeout: 3_000 }) + } + catch { + return skipBlockedPrecondition( + this, + 'Feature not enabled: Agent v2 Content Moderation Settings is not available in this build.', + ) + } +}) + +Then( + 'I should see the Agent v2 environment variables from the valid import in Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() + await expect.poll( + async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => + inputs.map(input => (input as HTMLInputElement).value), + ), + { timeout: 30_000 }, + ).toEqual(expect.arrayContaining([ + agentBuilderFixedInputs.envPlainKey, + agentBuilderFixedInputs.envPlainValue, + agentBuilderFixedInputs.envModeKey, + agentBuilderFixedInputs.envModeValue, + ])) + await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(2) + }, +) + +Then( + 'I should not see the deleted Agent v2 environment variable in Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() + await expect.poll( + async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => + inputs.map(input => (input as HTMLInputElement).value), + ), + { timeout: 30_000 }, + ).toEqual(expect.arrayContaining([ + agentBuilderFixedInputs.envModeKey, + agentBuilderFixedInputs.envModeValue, + ])) + await expect.poll( + async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => + inputs.map(input => (input as HTMLInputElement).value), + ), + { timeout: 30_000 }, + ).not.toContain(agentBuilderFixedInputs.envPlainKey) + await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(1) + }, +) + +Then( + 'I should see the plain Agent v2 environment variable in Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = await openAgentAdvancedSettings(page) + + await expect(advancedSettings.getByRole('textbox', { name: 'Key' })) + .toHaveValue(agentBuilderFixedInputs.envPlainKey) + await expect(advancedSettings.getByRole('textbox', { name: 'Value' })) + .toHaveValue(agentBuilderFixedInputs.envPlainValue) + await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() + await expect(page.getByRole('button', { name: /^Build$/i })).toBeVisible() + }, +) + +Then( + 'I should see the supported E2E environment variable in Advanced Settings', + async function (this: DifyWorld) { + await expectAgentEnvVariableVisible( + this, + agentBuilderFixedInputs.envPlainKey, + agentBuilderFixedInputs.envPlainValue, + ) + }, +) + +Then( + 'I should not see the supported E2E environment variable in Advanced Settings', + async function (this: DifyWorld) { + await expectAgentEnvVariableHidden(this, agentBuilderFixedInputs.envPlainKey) + }, +) + +Then( + 'I should see the Agent v2 environment variables from the invalid import in Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() + await expect.poll( + async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => + inputs.map(input => (input as HTMLInputElement).value), + ), + { timeout: 30_000 }, + ).toEqual(expect.arrayContaining([ + agentBuilderFixedInputs.envPlainKey, + agentBuilderFixedInputs.envPlainValue, + agentBuilderFixedInputs.envAfterInvalidImportKey, + agentBuilderFixedInputs.envAfterInvalidImportValue, + ])) + await expect(page.getByRole('button', { name: /^Build$/i })).toBeVisible() + }, +) diff --git a/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts b/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts new file mode 100644 index 00000000000000..ff65e1fb28397a --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts @@ -0,0 +1,117 @@ +import type { DifyWorld } from '../../support/world' +import { Then } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { agentBuilderExpectedTokens, agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../../support/agent-builder-resources' +import { agentBuilderTestMaterials } from '../../../support/test-materials' +import { expectProviderToolActionVisible, openAgentKnowledgeRetrievalDialog } from './configure-helpers' + +Then('I should see the Agent v2 full-config fixture sections', async function (this: DifyWorld) { + const page = this.getPage() + const stableModel = this.agentBuilder.preflight.stableModel + if (!stableModel) + throw new Error('Stable chat model preflight must run before asserting the full-config Agent.') + + await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) + await expect(page.getByText(agentBuilderPreseededResources.fullConfigAgent, { exact: true })) + .toBeVisible() + await expect(page.getByText(stableModel.name, { exact: true })).toBeVisible() + + const promptSection = page.getByRole('region', { name: 'Prompt' }) + await expect(promptSection).toBeVisible() + await expect(promptSection).toContainText(agentBuilderExpectedTokens.agentReply) + + const skillsSection = page.getByRole('region', { name: 'Skills' }) + await expect(skillsSection).toBeVisible() + await expect(skillsSection.getByRole('button', { + exact: true, + name: agentBuilderPreseededResources.summarySkill, + })).toBeVisible() + + const filesSection = page.getByRole('region', { name: 'Files' }) + await expect(filesSection).toBeVisible() + await expect(filesSection.getByRole('button', { + exact: true, + name: agentBuilderTestMaterials.smallFile, + })).toBeVisible() + await expect(filesSection.getByRole('button', { + exact: true, + name: agentBuilderTestMaterials.specialFilename, + })).toBeVisible() + + const toolsSection = page.getByRole('region', { name: 'Tools' }) + await expect(toolsSection).toBeVisible() + await expectProviderToolActionVisible( + toolsSection, + agentBuilderPreseededResources.jsonReplaceTool, + ) + + const knowledgeSection = page.getByRole('region', { name: 'Knowledge Retrieval' }) + await expect(knowledgeSection).toBeVisible() + await expect(knowledgeSection.getByText('Retrieval 1', { exact: true })).toBeVisible() + + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + await expect(advancedSettings).toBeVisible() + await expect( + advancedSettings.getByText('For power users. Env vars, sandbox & memory.'), + ).toBeVisible() +}) + +Then('I should see the Agent v2 tool state fixture tools', async function (this: DifyWorld) { + const page = this.getPage() + const toolsSection = page.getByRole('region', { name: 'Tools' }) + + await expect(toolsSection).toBeVisible({ timeout: 30_000 }) + await expect(toolsSection.getByRole('button', { exact: true, name: 'Not authorized' })).toBeVisible() + + const { action: jsonReplaceAction, tool: jsonTool } = await expectProviderToolActionVisible( + toolsSection, + agentBuilderPreseededResources.jsonReplaceTool, + ) + await jsonReplaceAction.hover() + await expect(toolsSection.getByRole('button', { + exact: true, + name: `Edit ${jsonTool.actionName}`, + })).toBeVisible() + await expect(toolsSection.getByRole('button', { + exact: true, + name: `Remove ${jsonTool.actionName}`, + })).toBeVisible() + + await expectProviderToolActionVisible( + toolsSection, + agentBuilderPreseededResources.tavilySearchTool, + ) +}) + +Then('I should see the Agent v2 dual retrieval fixture settings', async function (this: DifyWorld) { + const page = this.getPage() + const knowledgeSection = page.getByRole('region', { name: 'Knowledge Retrieval' }) + + await expect(knowledgeSection).toBeVisible({ timeout: 30_000 }) + await expect(knowledgeSection.getByText('Retrieval 1', { exact: true })).toBeVisible() + await expect(knowledgeSection.getByText('Retrieval 2', { exact: true })).toBeVisible() + + const agentDecideDialog = await openAgentKnowledgeRetrievalDialog(knowledgeSection, 'Retrieval 1') + await expect(agentDecideDialog.getByText(agentBuilderPreseededResources.agentKnowledgeBase, { + exact: true, + })).toBeVisible() + await expect(agentDecideDialog.getByRole('radio', { + exact: true, + name: 'Agent decide', + })).toBeChecked() + await agentDecideDialog.getByRole('button', { name: 'Close' }).click() + await expect(agentDecideDialog).not.toBeVisible() + + const customQueryDialog = await openAgentKnowledgeRetrievalDialog(knowledgeSection, 'Retrieval 2') + await expect(customQueryDialog.getByText(agentBuilderPreseededResources.agentKnowledgeBase, { + exact: true, + })).toBeVisible() + await expect(customQueryDialog.getByRole('radio', { + exact: true, + name: 'Custom query', + })).toBeChecked() + await expect(customQueryDialog.getByRole('textbox', { + exact: true, + name: 'Custom query text', + })).toHaveValue(agentBuilderFixedInputs.customKnowledgeQuery) +}) diff --git a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts new file mode 100644 index 00000000000000..728f9916b208f5 --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts @@ -0,0 +1,291 @@ +import type { DifyWorld } from '../../support/world' +import { readFile } from 'node:fs/promises' +import { Given, Then, When } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { + createAgentSoulConfigWithModel, + getAgentComposerDraft, + normalAgentPrompt, + normalAgentSoulConfig, + saveAgentBuildDraft, + saveAgentComposerDraft, + updatedAgentPrompt, + updatedAgentSoulConfig, + uploadAgentConfigFileToDraft, +} from '../../../support/agent' +import { agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../../support/agent-builder-resources' +import { skipBlockedPrecondition } from '../../../support/preflight' +import { agentBuilderTestMaterials, getAgentBuilderTestMaterialPath } from '../../../support/test-materials' +import { + getAgentEnvVariableValue, + getCurrentAgentId, + uploadSummaryConfigSkillForBuildDraft, +} from './configure-helpers' + +Given( + 'an Agent v2 Build draft adds the supported E2E files, skills, and env', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + const configFile = await uploadAgentConfigFileToDraft({ + agentId, + fileName: agentBuilderTestMaterials.smallFile, + filePath: getAgentBuilderTestMaterialPath('smallFile'), + }) + const skill = await uploadSummaryConfigSkillForBuildDraft(this) + + if (!configFile.file_id) + throw new Error('Agent v2 build draft config file fixture did not return a file_id.') + this.createdAgentConfigFiles.push({ agentId, name: configFile.name }) + + const normalConfig = this.agentBuilder.preflight.stableModel + ? createAgentSoulConfigWithModel(normalAgentSoulConfig, this.agentBuilder.preflight.stableModel) + : normalAgentSoulConfig + const updatedConfig = this.agentBuilder.preflight.stableModel + ? createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilder.preflight.stableModel) + : updatedAgentSoulConfig + + await saveAgentComposerDraft(agentId, normalConfig) + await saveAgentBuildDraft(agentId, { + ...updatedConfig, + config_files: [configFile], + config_skills: [{ + ...skill, + file_kind: skill.file_kind ?? 'tool_file', + }], + env: { + secret_refs: [], + variables: [{ + id: agentBuilderFixedInputs.envPlainKey, + key: agentBuilderFixedInputs.envPlainKey, + name: agentBuilderFixedInputs.envPlainKey, + value: agentBuilderFixedInputs.envPlainValue, + variable: agentBuilderFixedInputs.envPlainKey, + }], + }, + }) + }, +) + +Given( + 'an Agent v2 Build draft includes the existing e2e-summary-skill Skill', + async function (this: DifyWorld) { + const skill = await uploadSummaryConfigSkillForBuildDraft(this) + const normalConfig = this.agentBuilder.preflight.stableModel + ? createAgentSoulConfigWithModel(normalAgentSoulConfig, this.agentBuilder.preflight.stableModel) + : normalAgentSoulConfig + const updatedConfig = this.agentBuilder.preflight.stableModel + ? createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilder.preflight.stableModel) + : updatedAgentSoulConfig + const configSkills = [{ + ...skill, + file_kind: skill.file_kind ?? 'tool_file', + }] + + await saveAgentComposerDraft(getCurrentAgentId(this), { + ...normalConfig, + config_skills: configSkills, + }) + await saveAgentBuildDraft(getCurrentAgentId(this), { + ...updatedConfig, + config_skills: configSkills, + }) + }, +) + +Given('an Agent v2 Build draft uses the updated E2E prompt', async function (this: DifyWorld) { + await saveAgentBuildDraft(getCurrentAgentId(this), updatedAgentSoulConfig) +}) + +Given( + 'an Agent v2 Build draft uses the updated E2E prompt with the stable E2E model', + async function (this: DifyWorld) { + if (!this.agentBuilder.preflight.stableModel) + throw new Error('Create an Agent v2 Build draft with a stable model after stable model preflight.') + + await saveAgentBuildDraft( + getCurrentAgentId(this), + createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilder.preflight.stableModel), + ) + }, +) + +When('I generate an Agent v2 Build draft from the fixed instruction', async function (this: DifyWorld) { + const page = this.getPage() + const agentId = getCurrentAgentId(this) + const instruction = (await readFile(getAgentBuilderTestMaterialPath('buildInstruction'), 'utf8')).trim() + + await page.getByRole('button', { name: 'Build' }).click() + await page.getByPlaceholder('Describe what your agent should do').fill(instruction) + + const checkoutResponsePromise = page.waitForResponse(response => ( + response.request().method() === 'POST' + && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/build-draft/checkout`) + )) + const chatResponsePromise = page.waitForResponse(response => ( + response.request().method() === 'POST' + && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/chat-messages`) + )) + + await page.getByRole('button', { name: 'Start build' }).click() + expect((await checkoutResponsePromise).ok()).toBe(true) + expect((await chatResponsePromise).ok()).toBe(true) + await expect(page.getByText('Build draft')).toBeVisible({ timeout: 120_000 }) + await expect(page.getByRole('button', { name: 'Apply' })).toBeEnabled({ timeout: 120_000 }) + await expect(page.getByRole('button', { name: 'Discard' })).toBeEnabled() +}) + +When('I discard the Agent v2 Build draft', async function (this: DifyWorld) { + await this.getPage().getByRole('button', { name: 'Discard' }).click() +}) + +When('I apply the Agent v2 Build draft', async function (this: DifyWorld) { + const page = this.getPage() + const agentId = getCurrentAgentId(this) + const applyButton = page.getByRole('button', { name: 'Apply' }) + + await expect(applyButton).toBeEnabled({ timeout: 30_000 }) + const finalizeResponsePromise = page.waitForResponse(response => ( + response.request().method() === 'POST' + && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/build-chat/finalize`) + ), { timeout: 120_000 }) + const applyResponsePromise = page.waitForResponse(response => ( + response.request().method() === 'POST' + && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/build-draft/apply`) + ), { timeout: 120_000 }) + + await applyButton.click() + expect((await finalizeResponsePromise).ok()).toBe(true) + expect((await applyResponsePromise).ok()).toBe(true) + await expect(page.getByText('Action succeeded')).toBeVisible() +}) + +Then('Agent v2 Build chat Dify Tool writeback should be available', async function (this: DifyWorld) { + const toolsSection = this.getPage().getByRole('region', { name: 'Tools' }) + + await expect(toolsSection).toBeVisible({ timeout: 30_000 }) + + return skipBlockedPrecondition( + this, + 'Build draft Dify Tool writeback is not available: Build draft currently supports files, skills, and env only.', + ) +}) + +Then('I should see the Agent v2 Build draft pending changes', async function (this: DifyWorld) { + const page = this.getPage() + + await expect(page.getByText('Build draft')).toBeVisible({ timeout: 30_000 }) + await expect(page.getByRole('button', { name: 'Apply' })).toBeEnabled() + await expect(page.getByRole('button', { name: 'Discard' })).toBeEnabled() +}) + +Then('I should see the Agent v2 Build mode confirmation state', async function (this: DifyWorld) { + const page = this.getPage() + + await expect(page.getByText('Build mode', { exact: true })).toBeVisible() + await expect( + page.getByText('You\'re in build mode. Shape this setup through the chat on the right, then Apply.'), + ).toBeVisible() +}) + +Then('I should see the e2e-summary-skill Skill in the Skills section', async function (this: DifyWorld) { + const skillsSection = this.getPage().getByRole('region', { name: 'Skills' }) + + await expect(skillsSection).toBeVisible({ timeout: 30_000 }) + await expect(skillsSection.getByRole('button', { + exact: true, + name: agentBuilderPreseededResources.summarySkill, + })).toBeVisible() +}) + +Then('I should not see the e2e-summary-skill Skill in the Skills section', async function (this: DifyWorld) { + const skillsSection = this.getPage().getByRole('region', { name: 'Skills' }) + + await expect(skillsSection).toBeVisible({ timeout: 30_000 }) + await expect(skillsSection.getByRole('button', { + exact: true, + name: agentBuilderPreseededResources.summarySkill, + })).toHaveCount(0) +}) + +Then('I should see one e2e-summary-skill Skill in the Skills section', async function (this: DifyWorld) { + const skillsSection = this.getPage().getByRole('region', { name: 'Skills' }) + + await expect(skillsSection).toBeVisible({ timeout: 30_000 }) + await expect(skillsSection.getByRole('button', { + exact: true, + name: agentBuilderPreseededResources.summarySkill, + })).toHaveCount(1) +}) + +Then( + 'the Agent v2 draft should include the supported Build draft config', + async function (this: DifyWorld) { + await expect.poll( + async () => { + const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul + const variables = agentSoul?.env?.variables ?? [] + + return { + envValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), + fileNames: agentSoul?.config_files?.map(file => file.name) ?? [], + prompt: agentSoul?.prompt, + skillNames: agentSoul?.config_skills?.map(skill => skill.name) ?? [], + } + }, + { timeout: 30_000 }, + ).toEqual({ + envValue: agentBuilderFixedInputs.envPlainValue, + fileNames: expect.arrayContaining([agentBuilderTestMaterials.smallFile]), + prompt: { system_prompt: updatedAgentPrompt }, + skillNames: expect.arrayContaining([agentBuilderPreseededResources.summarySkill]), + }) + }, +) + +Then( + 'the Agent v2 draft should not include the supported Build draft config', + async function (this: DifyWorld) { + await expect.poll( + async () => { + const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul + const variables = agentSoul?.env?.variables ?? [] + + return { + envValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), + fileNames: agentSoul?.config_files?.map(file => file.name) ?? [], + prompt: agentSoul?.prompt, + skillNames: agentSoul?.config_skills?.map(skill => skill.name) ?? [], + } + }, + { timeout: 30_000 }, + ).toEqual({ + envValue: undefined, + fileNames: expect.not.arrayContaining([agentBuilderTestMaterials.smallFile]), + prompt: { system_prompt: normalAgentPrompt }, + skillNames: expect.not.arrayContaining([agentBuilderPreseededResources.summarySkill]), + }) + }, +) + +Then( + 'the Agent v2 draft should include one e2e-summary-skill Skill', + async function (this: DifyWorld) { + await expect.poll( + async () => { + const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul + return agentSoul?.config_skills?.filter( + skill => skill.name === agentBuilderPreseededResources.summarySkill, + ).length ?? 0 + }, + { timeout: 30_000 }, + ).toBe(1) + }, +) + +Then('the Agent v2 Build draft should no longer be active', async function (this: DifyWorld) { + const page = this.getPage() + + await expect(page.getByText('Build draft')).not.toBeVisible() + await expect(page.getByRole('button', { name: 'Apply' })).not.toBeVisible() + await expect(page.getByRole('button', { name: 'Discard' })).not.toBeVisible() +}) diff --git a/e2e/features/step-definitions/agent-v2/configure-helpers.ts b/e2e/features/step-definitions/agent-v2/configure-helpers.ts new file mode 100644 index 00000000000000..19d5fcbc478e91 --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/configure-helpers.ts @@ -0,0 +1,241 @@ +import type { Locator } from '@playwright/test' +import type { AgentComposerEnvVariable } from '../../../support/agent' +import type { DifyWorld } from '../../support/world' +import { expect } from '@playwright/test' +import { + getAgentComposerDraft, + normalAgentPrompt, + uploadAgentConfigSkillToDraft, +} from '../../../support/agent' +import { + agentBuilderTestMaterials, + getAgentBuilderTestMaterialPath, +} from '../../../support/test-materials' + +export const getCurrentAgentId = (world: DifyWorld) => { + const agentId = world.createdAgentIds.at(-1) + if (!agentId) + throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') + + return agentId +} + +export const getPreseededAgent = (world: DifyWorld, name: string) => { + const resource = world.agentBuilder.preflight.preseededResources[name] + if (!resource || resource.kind !== 'agent') { + throw new Error( + `Preseeded Agent "${name}" is not available. Run the matching preflight step first.`, + ) + } + + return resource +} + +const getPreseededToolDisplayParts = (displayName: string) => { + const [providerName, actionName] = displayName.split(' / ') + if (!providerName || !actionName) + throw new Error(`Preseeded tool display name must use "Provider / Action": ${displayName}`) + + return { actionName, providerName } +} + +export const getEnvVariableKey = (variable: AgentComposerEnvVariable) => + variable.key ?? variable.name ?? variable.variable + +export const getAgentEnvVariableValue = ( + variables: AgentComposerEnvVariable[], + key: string, +) => variables.find(variable => getEnvVariableKey(variable) === key)?.value + +export const getAgentEnvVariables = async (agentId: string) => + (await getAgentComposerDraft(agentId)).agent_soul?.env?.variables ?? [] + +export const uploadAgentConfigFile = async ( + world: DifyWorld, + material: keyof typeof agentBuilderTestMaterials, +) => { + const page = world.getPage() + const agentId = getCurrentAgentId(world) + const fileName = agentBuilderTestMaterials[material] + const filePath = getAgentBuilderTestMaterialPath(material) + + await page.getByRole('button', { name: 'Add file' }).click() + const dialog = page.getByRole('dialog', { name: 'Upload file' }) + await expect(dialog).toBeVisible() + + const fileChooserPromise = page.waitForEvent('filechooser') + await dialog.getByRole('button', { name: 'browse' }).click() + await (await fileChooserPromise).setFiles(filePath) + await expect(dialog.getByText(fileName)).toBeVisible() + + const commitResponsePromise = page.waitForResponse(response => ( + response.request().method() === 'POST' + && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/config/files`) + )) + await dialog.getByRole('button', { name: 'Upload' }).click() + const commitResponse = await commitResponsePromise + expect(commitResponse.status()).toBe(201) + const committed = await commitResponse.json() as { file?: { name?: string } } + await expect(dialog).not.toBeVisible({ timeout: 30_000 }) + + const committedName = committed.file?.name + if (!committedName) + throw new Error('Agent config file upload response did not include a file name.') + + world.createdAgentConfigFiles.push({ agentId, name: committedName }) +} + +export const expectAgentConfigFileVisible = async ( + world: DifyWorld, + material: keyof typeof agentBuilderTestMaterials, +) => { + await expect( + world.getPage().getByRole('button', { + exact: true, + name: agentBuilderTestMaterials[material], + }), + ).toBeVisible({ timeout: 30_000 }) +} + +export const expectAgentConfigFileHidden = async ( + world: DifyWorld, + material: keyof typeof agentBuilderTestMaterials, +) => { + await expect( + world.getPage().getByRole('button', { + exact: true, + name: agentBuilderTestMaterials[material], + }), + ).not.toBeVisible() +} + +export const expectAgentConfigFileSaved = async ( + world: DifyWorld, + material: keyof typeof agentBuilderTestMaterials, +) => { + const agentId = getCurrentAgentId(world) + const fileName = agentBuilderTestMaterials[material] + + await expect + .poll(async () => ( + await getAgentComposerDraft(agentId) + ).agent_soul?.config_files?.map(file => file.name) ?? [], { + timeout: 30_000, + }) + .toContain(fileName) +} + +export const uploadSummaryConfigSkillForBuildDraft = async (world: DifyWorld) => { + const agentId = getCurrentAgentId(world) + const skill = await uploadAgentConfigSkillToDraft({ + agentId, + fileName: agentBuilderTestMaterials.summarySkill, + filePath: getAgentBuilderTestMaterialPath('summarySkill'), + }) + + if (!skill.file_id) + throw new Error('Agent v2 build draft Skill fixture did not return a file_id.') + + world.createdAgentConfigSkills.push({ agentId, name: skill.name }) + + return skill +} + +export const openAgentAdvancedSettings = async (page: ReturnType) => { + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + const envEditorHeading = advancedSettings.getByRole('heading', { name: 'Env Editor' }) + + if (!await envEditorHeading.isVisible().catch(() => false)) + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + + await expect(envEditorHeading).toBeVisible() + + return advancedSettings +} + +export const expectAgentEnvVariableVisible = async ( + world: DifyWorld, + key: string, + value: string, +) => { + const advancedSettings = await openAgentAdvancedSettings(world.getPage()) + + await expect.poll( + async () => { + const text = await advancedSettings.textContent() + const inputValues = await advancedSettings.getByRole('textbox').evaluateAll(inputs => + inputs.map(input => (input as HTMLInputElement).value), + ) + + return { + hasKey: inputValues.includes(key) || !!text?.includes(key), + hasValue: inputValues.includes(value) || !!text?.includes(value), + } + }, + { timeout: 30_000 }, + ).toEqual({ + hasKey: true, + hasValue: true, + }) + await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() +} + +export const expectAgentEnvVariableHidden = async ( + world: DifyWorld, + key: string, +) => { + const advancedSettings = await openAgentAdvancedSettings(world.getPage()) + + await expect.poll( + async () => { + const text = await advancedSettings.textContent() + const inputValues = await advancedSettings.getByRole('textbox').evaluateAll(inputs => + inputs.map(input => (input as HTMLInputElement).value), + ) + + return inputValues.includes(key) || !!text?.includes(key) + }, + { timeout: 30_000 }, + ).toBe(false) +} + +export const expectNormalAgentPromptDraft = async (world: DifyWorld) => { + await expect.poll( + async () => (await getAgentComposerDraft(getCurrentAgentId(world))).agent_soul?.prompt, + { timeout: 30_000 }, + ).toEqual({ system_prompt: normalAgentPrompt }) +} + +export const expectProviderToolActionVisible = async ( + toolsSection: Locator, + displayName: string, +) => { + const tool = getPreseededToolDisplayParts(displayName) + const provider = toolsSection.getByRole('button', { + exact: true, + name: tool.providerName, + }) + await expect(provider).toBeVisible() + + const action = toolsSection.getByText(tool.actionName, { exact: true }) + if (!await action.isVisible()) + await provider.click() + await expect(action).toBeVisible() + + return { action, tool } +} + +export const openAgentKnowledgeRetrievalDialog = async (knowledgeSection: Locator, name: string) => { + await knowledgeSection.getByText(name, { exact: true }).hover() + await knowledgeSection.getByRole('button', { + exact: true, + name: `Edit ${name}`, + }).click() + + const dialog = knowledgeSection.page().getByRole('dialog', { + name: 'Knowledge Retrieval · Agent decide', + }) + await expect(dialog).toBeVisible() + + return dialog +} diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index d3f766f64723ee..b89e7461879596 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -1,7 +1,4 @@ -import type { Locator } from '@playwright/test' -import type { AgentComposerEnvVariable } from '../../../support/agent' import type { DifyWorld } from '../../support/world' -import { readFile } from 'node:fs/promises' import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { @@ -11,257 +8,18 @@ import { getAgentComposerDraft, getAgentConfigurePath, getAgentDriveSkills, - getTestAgent, normalAgentPrompt, normalAgentSoulConfig, - saveAgentBuildDraft, saveAgentComposerDraft, updatedAgentPrompt, - updatedAgentSoulConfig, - uploadAgentConfigFileToDraft, - uploadAgentConfigSkillToDraft, uploadAgentDriveSkill, } from '../../../support/agent' +import { agentBuilderTestMaterials, getAgentBuilderTestMaterialPath } from '../../../support/test-materials' import { - agentBuilderExpectedTokens, - agentBuilderFixedInputs, - agentBuilderPreseededResources, -} from '../../../support/agent-builder-resources' -import { waitForAgentConfigureAutosaved } from '../../../support/agent-configure' -import { skipBlockedPrecondition } from '../../../support/preflight' -import { - agentBuilderFileTreeFixtureFileNames, - agentBuilderTestMaterials, - getAgentBuilderTestMaterialPath, -} from '../../../support/test-materials' - -const getCurrentAgentId = (world: DifyWorld) => { - const agentId = world.createdAgentIds.at(-1) - if (!agentId) - throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') - - return agentId -} - -const getPreseededAgent = (world: DifyWorld, name: string) => { - const resource = world.agentBuilder.preflight.preseededResources[name] - if (!resource || resource.kind !== 'agent') { - throw new Error( - `Preseeded Agent "${name}" is not available. Run the matching preflight step first.`, - ) - } - - return resource -} - -const getPreseededToolDisplayParts = (displayName: string) => { - const [providerName, actionName] = displayName.split(' / ') - if (!providerName || !actionName) - throw new Error(`Preseeded tool display name must use "Provider / Action": ${displayName}`) - - return { actionName, providerName } -} - -const getEnvVariableKey = (variable: AgentComposerEnvVariable) => - variable.key ?? variable.name ?? variable.variable - -const getAgentEnvVariableValue = ( - variables: AgentComposerEnvVariable[], - key: string, -) => variables.find(variable => getEnvVariableKey(variable) === key)?.value - -const getAgentEnvVariables = async (agentId: string) => - (await getAgentComposerDraft(agentId)).agent_soul?.env?.variables ?? [] - -const uploadAgentConfigFile = async ( - world: DifyWorld, - material: keyof typeof agentBuilderTestMaterials, -) => { - const page = world.getPage() - const agentId = getCurrentAgentId(world) - const fileName = agentBuilderTestMaterials[material] - const filePath = getAgentBuilderTestMaterialPath(material) - - await page.getByRole('button', { name: 'Add file' }).click() - const dialog = page.getByRole('dialog', { name: 'Upload file' }) - await expect(dialog).toBeVisible() - - const fileChooserPromise = page.waitForEvent('filechooser') - await dialog.getByRole('button', { name: 'browse' }).click() - await (await fileChooserPromise).setFiles(filePath) - await expect(dialog.getByText(fileName)).toBeVisible() - - const commitResponsePromise = page.waitForResponse(response => ( - response.request().method() === 'POST' - && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/config/files`) - )) - await dialog.getByRole('button', { name: 'Upload' }).click() - const commitResponse = await commitResponsePromise - expect(commitResponse.status()).toBe(201) - const committed = await commitResponse.json() as { file?: { name?: string } } - await expect(dialog).not.toBeVisible({ timeout: 30_000 }) - - const committedName = committed.file?.name - if (!committedName) - throw new Error('Agent config file upload response did not include a file name.') - - world.createdAgentConfigFiles.push({ agentId, name: committedName }) -} - -const expectAgentConfigFileVisible = async ( - world: DifyWorld, - material: keyof typeof agentBuilderTestMaterials, -) => { - await expect( - world.getPage().getByRole('button', { - exact: true, - name: agentBuilderTestMaterials[material], - }), - ).toBeVisible({ timeout: 30_000 }) -} - -const expectAgentConfigFileHidden = async ( - world: DifyWorld, - material: keyof typeof agentBuilderTestMaterials, -) => { - await expect( - world.getPage().getByRole('button', { - exact: true, - name: agentBuilderTestMaterials[material], - }), - ).not.toBeVisible() -} - -const expectAgentConfigFileSaved = async ( - world: DifyWorld, - material: keyof typeof agentBuilderTestMaterials, -) => { - const agentId = getCurrentAgentId(world) - const fileName = agentBuilderTestMaterials[material] - - await expect - .poll(async () => ( - await getAgentComposerDraft(agentId) - ).agent_soul?.config_files?.map(file => file.name) ?? [], { - timeout: 30_000, - }) - .toContain(fileName) -} - -const uploadSummaryConfigSkillForBuildDraft = async (world: DifyWorld) => { - const agentId = getCurrentAgentId(world) - const skill = await uploadAgentConfigSkillToDraft({ - agentId, - fileName: agentBuilderTestMaterials.summarySkill, - filePath: getAgentBuilderTestMaterialPath('summarySkill'), - }) - - if (!skill.file_id) - throw new Error('Agent v2 build draft Skill fixture did not return a file_id.') - - world.createdAgentConfigSkills.push({ agentId, name: skill.name }) - - return skill -} - -const openAgentAdvancedSettings = async (page: ReturnType) => { - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - const envEditorHeading = advancedSettings.getByRole('heading', { name: 'Env Editor' }) - - if (!await envEditorHeading.isVisible().catch(() => false)) - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() - - await expect(envEditorHeading).toBeVisible() - - return advancedSettings -} - -const expectAgentEnvVariableVisible = async ( - world: DifyWorld, - key: string, - value: string, -) => { - const advancedSettings = await openAgentAdvancedSettings(world.getPage()) - - await expect.poll( - async () => { - const text = await advancedSettings.textContent() - const inputValues = await advancedSettings.getByRole('textbox').evaluateAll(inputs => - inputs.map(input => (input as HTMLInputElement).value), - ) - - return { - hasKey: inputValues.includes(key) || !!text?.includes(key), - hasValue: inputValues.includes(value) || !!text?.includes(value), - } - }, - { timeout: 30_000 }, - ).toEqual({ - hasKey: true, - hasValue: true, - }) - await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() -} - -const expectAgentEnvVariableHidden = async ( - world: DifyWorld, - key: string, -) => { - const advancedSettings = await openAgentAdvancedSettings(world.getPage()) - - await expect.poll( - async () => { - const text = await advancedSettings.textContent() - const inputValues = await advancedSettings.getByRole('textbox').evaluateAll(inputs => - inputs.map(input => (input as HTMLInputElement).value), - ) - - return inputValues.includes(key) || !!text?.includes(key) - }, - { timeout: 30_000 }, - ).toBe(false) -} - -const expectNormalAgentPromptDraft = async (world: DifyWorld) => { - await expect.poll( - async () => (await getAgentComposerDraft(getCurrentAgentId(world))).agent_soul?.prompt, - { timeout: 30_000 }, - ).toEqual({ system_prompt: normalAgentPrompt }) -} - -const expectProviderToolActionVisible = async ( - toolsSection: Locator, - displayName: string, -) => { - const tool = getPreseededToolDisplayParts(displayName) - const provider = toolsSection.getByRole('button', { - exact: true, - name: tool.providerName, - }) - await expect(provider).toBeVisible() - - const action = toolsSection.getByText(tool.actionName, { exact: true }) - if (!await action.isVisible()) - await provider.click() - await expect(action).toBeVisible() - - return { action, tool } -} - -const openAgentKnowledgeRetrievalDialog = async (knowledgeSection: Locator, name: string) => { - await knowledgeSection.getByText(name, { exact: true }).hover() - await knowledgeSection.getByRole('button', { - exact: true, - name: `Edit ${name}`, - }).click() - - const dialog = knowledgeSection.page().getByRole('dialog', { - name: 'Knowledge Retrieval · Agent decide', - }) - await expect(dialog).toBeVisible() - - return dialog -} + expectNormalAgentPromptDraft, + getCurrentAgentId, + getPreseededAgent, +} from './configure-helpers' Given('an Agent v2 test agent has been created via API', async function (this: DifyWorld) { const agent = await createTestAgent() @@ -319,93 +77,6 @@ Then('the Agent v2 test agent should include drive skill {string}', async functi expect(skills.map(skill => skill.name)).toContain(skillName) }) -Given( - 'an Agent v2 Build draft adds the supported E2E files, skills, and env', - async function (this: DifyWorld) { - const agentId = getCurrentAgentId(this) - const configFile = await uploadAgentConfigFileToDraft({ - agentId, - fileName: agentBuilderTestMaterials.smallFile, - filePath: getAgentBuilderTestMaterialPath('smallFile'), - }) - const skill = await uploadSummaryConfigSkillForBuildDraft(this) - - if (!configFile.file_id) - throw new Error('Agent v2 build draft config file fixture did not return a file_id.') - this.createdAgentConfigFiles.push({ agentId, name: configFile.name }) - - const normalConfig = this.agentBuilder.preflight.stableModel - ? createAgentSoulConfigWithModel(normalAgentSoulConfig, this.agentBuilder.preflight.stableModel) - : normalAgentSoulConfig - const updatedConfig = this.agentBuilder.preflight.stableModel - ? createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilder.preflight.stableModel) - : updatedAgentSoulConfig - - await saveAgentComposerDraft(agentId, normalConfig) - await saveAgentBuildDraft(agentId, { - ...updatedConfig, - config_files: [configFile], - config_skills: [{ - ...skill, - file_kind: skill.file_kind ?? 'tool_file', - }], - env: { - secret_refs: [], - variables: [{ - id: agentBuilderFixedInputs.envPlainKey, - key: agentBuilderFixedInputs.envPlainKey, - name: agentBuilderFixedInputs.envPlainKey, - value: agentBuilderFixedInputs.envPlainValue, - variable: agentBuilderFixedInputs.envPlainKey, - }], - }, - }) - }, -) - -Given( - 'an Agent v2 Build draft includes the existing e2e-summary-skill Skill', - async function (this: DifyWorld) { - const skill = await uploadSummaryConfigSkillForBuildDraft(this) - const normalConfig = this.agentBuilder.preflight.stableModel - ? createAgentSoulConfigWithModel(normalAgentSoulConfig, this.agentBuilder.preflight.stableModel) - : normalAgentSoulConfig - const updatedConfig = this.agentBuilder.preflight.stableModel - ? createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilder.preflight.stableModel) - : updatedAgentSoulConfig - const configSkills = [{ - ...skill, - file_kind: skill.file_kind ?? 'tool_file', - }] - - await saveAgentComposerDraft(getCurrentAgentId(this), { - ...normalConfig, - config_skills: configSkills, - }) - await saveAgentBuildDraft(getCurrentAgentId(this), { - ...updatedConfig, - config_skills: configSkills, - }) - }, -) - -Given('an Agent v2 Build draft uses the updated E2E prompt', async function (this: DifyWorld) { - await saveAgentBuildDraft(getCurrentAgentId(this), updatedAgentSoulConfig) -}) - -Given( - 'an Agent v2 Build draft uses the updated E2E prompt with the stable E2E model', - async function (this: DifyWorld) { - if (!this.agentBuilder.preflight.stableModel) - throw new Error('Create an Agent v2 Build draft with a stable model after stable model preflight.') - - await saveAgentBuildDraft( - getCurrentAgentId(this), - createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilder.preflight.stableModel), - ) - }, -) - When('I open the Agent v2 configure page', async function (this: DifyWorld) { await this.getPage().goto(getAgentConfigurePath(getCurrentAgentId(this))) }) @@ -419,31 +90,6 @@ When('I switch to the Agent v2 Configure section', async function (this: DifyWor await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) }) -When('I generate an Agent v2 Build draft from the fixed instruction', async function (this: DifyWorld) { - const page = this.getPage() - const agentId = getCurrentAgentId(this) - const instruction = (await readFile(getAgentBuilderTestMaterialPath('buildInstruction'), 'utf8')).trim() - - await page.getByRole('button', { name: 'Build' }).click() - await page.getByPlaceholder('Describe what your agent should do').fill(instruction) - - const checkoutResponsePromise = page.waitForResponse(response => ( - response.request().method() === 'POST' - && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/build-draft/checkout`) - )) - const chatResponsePromise = page.waitForResponse(response => ( - response.request().method() === 'POST' - && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/chat-messages`) - )) - - await page.getByRole('button', { name: 'Start build' }).click() - expect((await checkoutResponsePromise).ok()).toBe(true) - expect((await chatResponsePromise).ok()).toBe(true) - await expect(page.getByText('Build draft')).toBeVisible({ timeout: 120_000 }) - await expect(page.getByRole('button', { name: 'Apply' })).toBeEnabled({ timeout: 120_000 }) - await expect(page.getByRole('button', { name: 'Discard' })).toBeEnabled() -}) - When('I open the Agent v2 configure page from the Agent Roster', async function (this: DifyWorld) { const page = this.getPage() const agentId = getCurrentAgentId(this) @@ -478,132 +124,6 @@ When('I fill the Agent v2 prompt editor with the normal E2E prompt', async funct await promptSection.getByRole('textbox', { name: 'Prompt' }).fill(normalAgentPrompt) }) -When('I discard the Agent v2 Build draft', async function (this: DifyWorld) { - await this.getPage().getByRole('button', { name: 'Discard' }).click() -}) - -When('I apply the Agent v2 Build draft', async function (this: DifyWorld) { - const page = this.getPage() - const agentId = getCurrentAgentId(this) - const applyButton = page.getByRole('button', { name: 'Apply' }) - - await expect(applyButton).toBeEnabled({ timeout: 30_000 }) - const finalizeResponsePromise = page.waitForResponse(response => ( - response.request().method() === 'POST' - && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/build-chat/finalize`) - ), { timeout: 120_000 }) - const applyResponsePromise = page.waitForResponse(response => ( - response.request().method() === 'POST' - && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/build-draft/apply`) - ), { timeout: 120_000 }) - - await applyButton.click() - expect((await finalizeResponsePromise).ok()).toBe(true) - expect((await applyResponsePromise).ok()).toBe(true) - await expect(page.getByText('Action succeeded')).toBeVisible() -}) - -When('I publish the Agent v2 draft', async function (this: DifyWorld) { - const page = this.getPage() - const publishButton = page.getByRole('button', { name: /^Publish(?: update)?$/ }) - - await expect(publishButton).toBeEnabled({ timeout: 30_000 }) - await publishButton.click() -}) - -When('I upload the small Agent v2 file from the Files section', async function (this: DifyWorld) { - await uploadAgentConfigFile(this, 'smallFile') -}) - -When('I upload the special-name Agent v2 file from the Files section', async function (this: DifyWorld) { - await uploadAgentConfigFile(this, 'specialFilename') -}) - -When( - 'I add the plain Agent v2 environment variable from Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() - await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() - - await advancedSettings - .getByRole('textbox', { name: 'Key' }) - .fill(agentBuilderFixedInputs.envPlainKey) - await advancedSettings - .getByRole('textbox', { name: 'Value' }) - .fill(agentBuilderFixedInputs.envPlainValue) - await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() - }, -) - -When( - 'I import the invalid Agent v2 environment file from Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - const fileChooserPromise = page.waitForEvent('filechooser') - await advancedSettings.getByRole('button', { name: 'Import .env' }).click() - await (await fileChooserPromise).setFiles(getAgentBuilderTestMaterialPath('invalidEnv')) - }, -) - -When( - 'I add the secondary plain Agent v2 environment variable from Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await advancedSettings.getByRole('button', { name: 'Add environment variable' }).click() - await advancedSettings - .getByRole('textbox', { name: 'Key' }) - .last() - .fill(agentBuilderFixedInputs.envModeKey) - await advancedSettings - .getByRole('textbox', { name: 'Value' }) - .last() - .fill(agentBuilderFixedInputs.envModeValue) - await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(2) - }, -) - -When( - 'I delete the plain Agent v2 environment variable from Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await advancedSettings - .getByRole('button', { name: `Delete ${agentBuilderFixedInputs.envPlainKey}` }) - .click() - }, -) - -When( - 'I import the valid Agent v2 environment file from Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() - await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() - - const fileChooserPromise = page.waitForEvent('filechooser') - await advancedSettings.getByRole('button', { name: 'Import .env' }).click() - await (await fileChooserPromise).setFiles(getAgentBuilderTestMaterialPath('validEnv')) - }, -) - -When('I expand Agent v2 Advanced Settings', async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() - await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() -}) - Then('I should be on the Agent v2 configure page', async function (this: DifyWorld) { const agentId = getCurrentAgentId(this) @@ -620,142 +140,6 @@ Then('I should see the Agent v2 configure workspace', async function (this: Dify await expect(page.getByText(this.lastCreatedAgentName!)).toBeVisible() }) -Then('I should see the Agent v2 full-config fixture sections', async function (this: DifyWorld) { - const page = this.getPage() - const stableModel = this.agentBuilder.preflight.stableModel - if (!stableModel) - throw new Error('Stable chat model preflight must run before asserting the full-config Agent.') - - await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) - await expect(page.getByText(agentBuilderPreseededResources.fullConfigAgent, { exact: true })) - .toBeVisible() - await expect(page.getByText(stableModel.name, { exact: true })).toBeVisible() - - const promptSection = page.getByRole('region', { name: 'Prompt' }) - await expect(promptSection).toBeVisible() - await expect(promptSection).toContainText(agentBuilderExpectedTokens.agentReply) - - const skillsSection = page.getByRole('region', { name: 'Skills' }) - await expect(skillsSection).toBeVisible() - await expect(skillsSection.getByRole('button', { - exact: true, - name: agentBuilderPreseededResources.summarySkill, - })).toBeVisible() - - const filesSection = page.getByRole('region', { name: 'Files' }) - await expect(filesSection).toBeVisible() - await expect(filesSection.getByRole('button', { - exact: true, - name: agentBuilderTestMaterials.smallFile, - })).toBeVisible() - await expect(filesSection.getByRole('button', { - exact: true, - name: agentBuilderTestMaterials.specialFilename, - })).toBeVisible() - - const toolsSection = page.getByRole('region', { name: 'Tools' }) - await expect(toolsSection).toBeVisible() - await expectProviderToolActionVisible( - toolsSection, - agentBuilderPreseededResources.jsonReplaceTool, - ) - - const knowledgeSection = page.getByRole('region', { name: 'Knowledge Retrieval' }) - await expect(knowledgeSection).toBeVisible() - await expect(knowledgeSection.getByText('Retrieval 1', { exact: true })).toBeVisible() - - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - await expect(advancedSettings).toBeVisible() - await expect( - advancedSettings.getByText('For power users. Env vars, sandbox & memory.'), - ).toBeVisible() -}) - -Then('I should see the Agent v2 tool state fixture tools', async function (this: DifyWorld) { - const page = this.getPage() - const toolsSection = page.getByRole('region', { name: 'Tools' }) - - await expect(toolsSection).toBeVisible({ timeout: 30_000 }) - await expect(toolsSection.getByRole('button', { exact: true, name: 'Not authorized' })).toBeVisible() - - const { action: jsonReplaceAction, tool: jsonTool } = await expectProviderToolActionVisible( - toolsSection, - agentBuilderPreseededResources.jsonReplaceTool, - ) - await jsonReplaceAction.hover() - await expect(toolsSection.getByRole('button', { - exact: true, - name: `Edit ${jsonTool.actionName}`, - })).toBeVisible() - await expect(toolsSection.getByRole('button', { - exact: true, - name: `Remove ${jsonTool.actionName}`, - })).toBeVisible() - - await expectProviderToolActionVisible( - toolsSection, - agentBuilderPreseededResources.tavilySearchTool, - ) -}) - -Then( - 'I should see the Agent v2 file fixture entries in the current flat Files list', - async function (this: DifyWorld) { - const page = this.getPage() - const filesSection = page.getByRole('region', { name: 'Files' }) - const filesList = filesSection.getByLabel('Agent files') - - await expect(filesSection).toBeVisible({ timeout: 30_000 }) - await expect(filesList).toBeVisible() - - for (const fileName of agentBuilderFileTreeFixtureFileNames) { - await expect(filesList.getByRole('button', { - exact: true, - name: fileName, - })).toBeVisible() - } - - await expect(filesList.getByRole('button', { exact: true, name: 'assets' })).toHaveCount(0) - await expect(filesList.getByRole('button', { exact: true, name: 'docs' })).toHaveCount(0) - await expect(filesList.getByRole('button', { exact: true, name: 'public' })).toHaveCount(0) - await expect(filesList.getByRole('button', { exact: true, name: 'src' })).toHaveCount(0) - await expect(filesList.getByRole('button', { exact: true, name: 'web-game' })).toHaveCount(0) - }, -) - -Then('I should see the Agent v2 dual retrieval fixture settings', async function (this: DifyWorld) { - const page = this.getPage() - const knowledgeSection = page.getByRole('region', { name: 'Knowledge Retrieval' }) - - await expect(knowledgeSection).toBeVisible({ timeout: 30_000 }) - await expect(knowledgeSection.getByText('Retrieval 1', { exact: true })).toBeVisible() - await expect(knowledgeSection.getByText('Retrieval 2', { exact: true })).toBeVisible() - - const agentDecideDialog = await openAgentKnowledgeRetrievalDialog(knowledgeSection, 'Retrieval 1') - await expect(agentDecideDialog.getByText(agentBuilderPreseededResources.agentKnowledgeBase, { - exact: true, - })).toBeVisible() - await expect(agentDecideDialog.getByRole('radio', { - exact: true, - name: 'Agent decide', - })).toBeChecked() - await agentDecideDialog.getByRole('button', { name: 'Close' }).click() - await expect(agentDecideDialog).not.toBeVisible() - - const customQueryDialog = await openAgentKnowledgeRetrievalDialog(knowledgeSection, 'Retrieval 2') - await expect(customQueryDialog.getByText(agentBuilderPreseededResources.agentKnowledgeBase, { - exact: true, - })).toBeVisible() - await expect(customQueryDialog.getByRole('radio', { - exact: true, - name: 'Custom query', - })).toBeChecked() - await expect(customQueryDialog.getByRole('textbox', { - exact: true, - name: 'Custom query text', - })).toHaveValue(agentBuilderFixedInputs.customKnowledgeQuery) -}) - Then( 'I should see the normal E2E prompt in the Agent v2 prompt editor', async function (this: DifyWorld) { @@ -798,380 +182,6 @@ Then( }, ) -Then('I should see the small Agent v2 file in the Files section', async function (this: DifyWorld) { - await expectAgentConfigFileVisible(this, 'smallFile') -}) - -Then('I should not see the small Agent v2 file in the Files section', async function (this: DifyWorld) { - await expectAgentConfigFileHidden(this, 'smallFile') -}) - -Then('I should see the special-name Agent v2 file in the Files section', async function (this: DifyWorld) { - await expectAgentConfigFileVisible(this, 'specialFilename') -}) - -Then('I should see the e2e-summary-skill Skill in the Skills section', async function (this: DifyWorld) { - const skillsSection = this.getPage().getByRole('region', { name: 'Skills' }) - - await expect(skillsSection.getByRole('button', { - exact: true, - name: agentBuilderPreseededResources.summarySkill, - })).toBeVisible({ timeout: 30_000 }) -}) - -Then('I should not see the e2e-summary-skill Skill in the Skills section', async function (this: DifyWorld) { - const skillsSection = this.getPage().getByRole('region', { name: 'Skills' }) - - await expect(skillsSection.getByRole('button', { - exact: true, - name: agentBuilderPreseededResources.summarySkill, - })).not.toBeVisible() -}) - -Then('I should see one e2e-summary-skill Skill in the Skills section', async function (this: DifyWorld) { - const skillsSection = this.getPage().getByRole('region', { name: 'Skills' }) - - await expect(skillsSection.getByRole('button', { - exact: true, - name: agentBuilderPreseededResources.summarySkill, - })).toHaveCount(1) -}) - -Then( - 'the small Agent v2 file should be saved in the Agent v2 draft', - async function (this: DifyWorld) { - await expectAgentConfigFileSaved(this, 'smallFile') - }, -) - -Then( - 'the special-name Agent v2 file should be saved in the Agent v2 draft', - async function (this: DifyWorld) { - await expectAgentConfigFileSaved(this, 'specialFilename') - }, -) - -Then( - 'Agent v2 Advanced Settings should describe supported entries while collapsed', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await expect(advancedSettings).toBeVisible() - await expect( - advancedSettings.getByText('For power users. Env vars, sandbox & memory.'), - ).toBeVisible() - await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })) - .not - .toBeVisible() - }, -) - -Then( - 'the plain Agent v2 environment variable should be saved in the Agent v2 draft', - async function (this: DifyWorld) { - const agentId = getCurrentAgentId(this) - - await expect - .poll(async () => { - const env = (await getAgentComposerDraft(agentId)).agent_soul?.env - const variable = env?.variables?.find(item => - getEnvVariableKey(item) === agentBuilderFixedInputs.envPlainKey, - ) - - return { - secretCount: env?.secret_refs?.length ?? 0, - value: variable?.value, - } - }, { - timeout: 30_000, - }) - .toEqual({ - secretCount: 0, - value: agentBuilderFixedInputs.envPlainValue, - }) - }, -) - -Then( - 'the Agent v2 environment variables for deletion should be saved in the Agent v2 draft', - async function (this: DifyWorld) { - const agentId = getCurrentAgentId(this) - - await expect - .poll(async () => { - const variables = await getAgentEnvVariables(agentId) - - return { - modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey), - plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), - } - }, { - timeout: 30_000, - }) - .toEqual({ - modeValue: agentBuilderFixedInputs.envModeValue, - plainValue: agentBuilderFixedInputs.envPlainValue, - }) - }, -) - -Then( - 'the plain Agent v2 environment variable should be removed from the Agent v2 draft', - async function (this: DifyWorld) { - const agentId = getCurrentAgentId(this) - - await expect - .poll(async () => { - const variables = await getAgentEnvVariables(agentId) - - return { - modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey), - plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), - } - }, { - timeout: 30_000, - }) - .toEqual({ - modeValue: agentBuilderFixedInputs.envModeValue, - plainValue: undefined, - }) - }, -) - -Then( - 'the valid Agent v2 environment import should be saved in the Agent v2 draft', - async function (this: DifyWorld) { - const agentId = getCurrentAgentId(this) - - await expect - .poll(async () => { - const variables = await getAgentEnvVariables(agentId) - - return { - modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey), - plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), - } - }, { - timeout: 30_000, - }) - .toEqual({ - modeValue: agentBuilderFixedInputs.envModeValue, - plainValue: agentBuilderFixedInputs.envPlainValue, - }) - }, -) - -Then( - 'the invalid Agent v2 environment import should report skipped lines', - async function (this: DifyWorld) { - await expect(this.getPage().getByText('2 invalid .env lines were skipped.')).toBeVisible() - }, -) - -Then( - 'the Agent v2 environment variables from the invalid import should be saved in the Agent v2 draft', - async function (this: DifyWorld) { - const agentId = getCurrentAgentId(this) - - await expect - .poll(async () => { - const variables = await getAgentEnvVariables(agentId) - - return { - importedValue: getAgentEnvVariableValue( - variables, - agentBuilderFixedInputs.envAfterInvalidImportKey, - ), - plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), - } - }, { - timeout: 30_000, - }) - .toEqual({ - importedValue: agentBuilderFixedInputs.envAfterInvalidImportValue, - plainValue: agentBuilderFixedInputs.envPlainValue, - }) - }, -) - -Then( - 'I should see the supported Agent v2 Advanced Settings entries', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - const envEditor = advancedSettings.getByRole('region', { name: 'Env Editor' }) - - await expect(envEditor).toBeVisible() - await expect(envEditor.getByRole('button', { name: 'Import .env' })).toBeVisible() - await expect(envEditor.getByRole('button', { name: 'Add environment variable' })) - .toBeVisible() - await expect(envEditor.getByText('Key', { exact: true })).toBeVisible() - await expect(envEditor.getByText('Value', { exact: true })).toBeVisible() - await expect(envEditor.getByText('Scope', { exact: true })).toBeVisible() - }, -) - -Then('Agent v2 Content Moderation Settings should be available', async function (this: DifyWorld) { - const advancedSettings = this.getPage().getByRole('region', { name: 'Advanced Settings' }) - const contentModeration = advancedSettings.getByRole('region', { name: 'Content moderation' }) - - try { - await expect(contentModeration).toBeVisible({ timeout: 3_000 }) - } - catch { - return skipBlockedPrecondition( - this, - 'Feature not enabled: Agent v2 Content Moderation Settings is not available in this build.', - ) - } -}) - -Then('Agent v2 Build chat Dify Tool writeback should be available', async function (this: DifyWorld) { - const toolsSection = this.getPage().getByRole('region', { name: 'Tools' }) - - await expect(toolsSection).toBeVisible({ timeout: 30_000 }) - - return skipBlockedPrecondition( - this, - 'Build draft Dify Tool writeback is not available: Build draft currently supports files, skills, and env only.', - ) -}) - -Then('Agent v2 standalone Output Variables should be available', async function (this: DifyWorld) { - const page = this.getPage() - - await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) - - return skipBlockedPrecondition( - this, - 'Standalone Agent Output Variables are not available: output variables currently belong to Workflow Agent v2 nodes.', - ) -}) - -Then( - 'I should see the Agent v2 environment variables from the valid import in Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() - await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() - await expect.poll( - async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => - inputs.map(input => (input as HTMLInputElement).value), - ), - { timeout: 30_000 }, - ).toEqual(expect.arrayContaining([ - agentBuilderFixedInputs.envPlainKey, - agentBuilderFixedInputs.envPlainValue, - agentBuilderFixedInputs.envModeKey, - agentBuilderFixedInputs.envModeValue, - ])) - await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(2) - }, -) - -Then( - 'I should not see the deleted Agent v2 environment variable in Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() - await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() - await expect.poll( - async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => - inputs.map(input => (input as HTMLInputElement).value), - ), - { timeout: 30_000 }, - ).toEqual(expect.arrayContaining([ - agentBuilderFixedInputs.envModeKey, - agentBuilderFixedInputs.envModeValue, - ])) - await expect.poll( - async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => - inputs.map(input => (input as HTMLInputElement).value), - ), - { timeout: 30_000 }, - ).not.toContain(agentBuilderFixedInputs.envPlainKey) - await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(1) - }, -) - -Then( - 'I should see the plain Agent v2 environment variable in Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = await openAgentAdvancedSettings(page) - - await expect(advancedSettings.getByRole('textbox', { name: 'Key' })) - .toHaveValue(agentBuilderFixedInputs.envPlainKey) - await expect(advancedSettings.getByRole('textbox', { name: 'Value' })) - .toHaveValue(agentBuilderFixedInputs.envPlainValue) - await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() - await expect(page.getByRole('button', { name: /^Build$/i })).toBeVisible() - }, -) - -Then( - 'I should see the supported E2E environment variable in Advanced Settings', - async function (this: DifyWorld) { - await expectAgentEnvVariableVisible( - this, - agentBuilderFixedInputs.envPlainKey, - agentBuilderFixedInputs.envPlainValue, - ) - }, -) - -Then( - 'I should not see the supported E2E environment variable in Advanced Settings', - async function (this: DifyWorld) { - await expectAgentEnvVariableHidden(this, agentBuilderFixedInputs.envPlainKey) - }, -) - -Then( - 'I should see the Agent v2 environment variables from the invalid import in Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() - await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() - await expect.poll( - async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => - inputs.map(input => (input as HTMLInputElement).value), - ), - { timeout: 30_000 }, - ).toEqual(expect.arrayContaining([ - agentBuilderFixedInputs.envPlainKey, - agentBuilderFixedInputs.envPlainValue, - agentBuilderFixedInputs.envAfterInvalidImportKey, - agentBuilderFixedInputs.envAfterInvalidImportValue, - ])) - await expect(page.getByRole('button', { name: /^Build$/i })).toBeVisible() - }, -) - -Then('I should see the Agent v2 Build draft pending changes', async function (this: DifyWorld) { - const page = this.getPage() - - await expect(page.getByText('Build draft')).toBeVisible({ timeout: 30_000 }) - await expect(page.getByRole('button', { name: 'Apply' })).toBeEnabled() - await expect(page.getByRole('button', { name: 'Discard' })).toBeEnabled() -}) - -Then('I should see the Agent v2 Build mode confirmation state', async function (this: DifyWorld) { - const page = this.getPage() - - await expect(page.getByText('Build mode', { exact: true })).toBeVisible() - await expect( - page.getByText('You\'re in build mode. Shape this setup through the chat on the right, then Apply.'), - ).toBeVisible() -}) - Then( 'the normal Agent v2 draft should still use the normal E2E prompt', async function (this: DifyWorld) { @@ -1222,89 +232,3 @@ Then( }) }, ) - -Then( - 'the Agent v2 draft should include the supported Build draft config', - async function (this: DifyWorld) { - await expect.poll( - async () => { - const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul - const variables = agentSoul?.env?.variables ?? [] - - return { - envValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), - fileNames: agentSoul?.config_files?.map(file => file.name) ?? [], - prompt: agentSoul?.prompt, - skillNames: agentSoul?.config_skills?.map(skill => skill.name) ?? [], - } - }, - { timeout: 30_000 }, - ).toEqual({ - envValue: agentBuilderFixedInputs.envPlainValue, - fileNames: expect.arrayContaining([agentBuilderTestMaterials.smallFile]), - prompt: { system_prompt: updatedAgentPrompt }, - skillNames: expect.arrayContaining([agentBuilderPreseededResources.summarySkill]), - }) - }, -) - -Then( - 'the Agent v2 draft should not include the supported Build draft config', - async function (this: DifyWorld) { - await expect.poll( - async () => { - const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul - const variables = agentSoul?.env?.variables ?? [] - - return { - envValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), - fileNames: agentSoul?.config_files?.map(file => file.name) ?? [], - prompt: agentSoul?.prompt, - skillNames: agentSoul?.config_skills?.map(skill => skill.name) ?? [], - } - }, - { timeout: 30_000 }, - ).toEqual({ - envValue: undefined, - fileNames: expect.not.arrayContaining([agentBuilderTestMaterials.smallFile]), - prompt: { system_prompt: normalAgentPrompt }, - skillNames: expect.not.arrayContaining([agentBuilderPreseededResources.summarySkill]), - }) - }, -) - -Then( - 'the Agent v2 draft should include one e2e-summary-skill Skill', - async function (this: DifyWorld) { - await expect.poll( - async () => { - const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul - return agentSoul?.config_skills?.filter( - skill => skill.name === agentBuilderPreseededResources.summarySkill, - ).length ?? 0 - }, - { timeout: 30_000 }, - ).toBe(1) - }, -) - -Then('the Agent v2 Build draft should no longer be active', async function (this: DifyWorld) { - const page = this.getPage() - - await expect(page.getByText('Build draft')).not.toBeVisible() - await expect(page.getByRole('button', { name: 'Apply' })).not.toBeVisible() - await expect(page.getByRole('button', { name: 'Discard' })).not.toBeVisible() -}) - -Then('the Agent v2 configuration should be saved automatically', async function (this: DifyWorld) { - await waitForAgentConfigureAutosaved(this.getPage()) -}) - -Then('the Agent v2 draft should be published and up to date', async function (this: DifyWorld) { - const page = this.getPage() - const agentId = getCurrentAgentId(this) - - await expect(page.getByRole('button', { name: 'Published' })).toBeVisible({ timeout: 30_000 }) - await expect(page.getByText('Up to date')).toBeVisible() - await expect.poll(async () => (await getTestAgent(agentId)).active_config_is_published).toBe(true) -}) diff --git a/e2e/features/step-definitions/agent-v2/files.steps.ts b/e2e/features/step-definitions/agent-v2/files.steps.ts new file mode 100644 index 00000000000000..8c77b59d4f3b8c --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/files.steps.ts @@ -0,0 +1,67 @@ +import type { DifyWorld } from '../../support/world' +import { Then, When } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { agentBuilderFileTreeFixtureFileNames } from '../../../support/test-materials' +import { + expectAgentConfigFileHidden, + expectAgentConfigFileSaved, + expectAgentConfigFileVisible, + uploadAgentConfigFile, +} from './configure-helpers' + +When('I upload the small Agent v2 file from the Files section', async function (this: DifyWorld) { + await uploadAgentConfigFile(this, 'smallFile') +}) + +When('I upload the special-name Agent v2 file from the Files section', async function (this: DifyWorld) { + await uploadAgentConfigFile(this, 'specialFilename') +}) + +Then( + 'I should see the Agent v2 file fixture entries in the current flat Files list', + async function (this: DifyWorld) { + const page = this.getPage() + const filesSection = page.getByRole('region', { name: 'Files' }) + const filesList = filesSection.getByLabel('Agent files') + + await expect(filesSection).toBeVisible({ timeout: 30_000 }) + await expect(filesList).toBeVisible() + + for (const fileName of agentBuilderFileTreeFixtureFileNames) { + await expect(filesList.getByRole('button', { + exact: true, + name: fileName, + })).toBeVisible() + } + + await expect(filesList.getByRole('button', { exact: true, name: 'assets' })).toHaveCount(0) + await expect(filesList.getByRole('button', { exact: true, name: 'docs' })).toHaveCount(0) + await expect(filesList.getByRole('button', { exact: true, name: 'public' })).toHaveCount(0) + await expect(filesList.getByRole('button', { exact: true, name: 'src' })).toHaveCount(0) + await expect(filesList.getByRole('button', { exact: true, name: 'web-game' })).toHaveCount(0) + }, +) +Then('I should see the small Agent v2 file in the Files section', async function (this: DifyWorld) { + await expectAgentConfigFileVisible(this, 'smallFile') +}) + +Then('I should not see the small Agent v2 file in the Files section', async function (this: DifyWorld) { + await expectAgentConfigFileHidden(this, 'smallFile') +}) + +Then('I should see the special-name Agent v2 file in the Files section', async function (this: DifyWorld) { + await expectAgentConfigFileVisible(this, 'specialFilename') +}) +Then( + 'the small Agent v2 file should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + await expectAgentConfigFileSaved(this, 'smallFile') + }, +) + +Then( + 'the special-name Agent v2 file should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + await expectAgentConfigFileSaved(this, 'specialFilename') + }, +) diff --git a/e2e/features/step-definitions/agent-v2/output-variables.steps.ts b/e2e/features/step-definitions/agent-v2/output-variables.steps.ts new file mode 100644 index 00000000000000..b091ff46edfff7 --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/output-variables.steps.ts @@ -0,0 +1,15 @@ +import type { DifyWorld } from '../../support/world' +import { Then } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { skipBlockedPrecondition } from '../../../support/preflight' + +Then('Agent v2 standalone Output Variables should be available', async function (this: DifyWorld) { + const page = this.getPage() + + await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) + + return skipBlockedPrecondition( + this, + 'Standalone Agent Output Variables are not available: output variables currently belong to Workflow Agent v2 nodes.', + ) +}) diff --git a/e2e/features/step-definitions/agent-v2/publish.steps.ts b/e2e/features/step-definitions/agent-v2/publish.steps.ts new file mode 100644 index 00000000000000..13699f6578ab04 --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/publish.steps.ts @@ -0,0 +1,27 @@ +import type { DifyWorld } from '../../support/world' +import { Then, When } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { getTestAgent } from '../../../support/agent' +import { waitForAgentConfigureAutosaved } from '../../../support/agent-configure' +import { getCurrentAgentId } from './configure-helpers' + +When('I publish the Agent v2 draft', async function (this: DifyWorld) { + const page = this.getPage() + const publishButton = page.getByRole('button', { name: /^Publish(?: update)?$/ }) + + await expect(publishButton).toBeEnabled({ timeout: 30_000 }) + await publishButton.click() +}) + +Then('the Agent v2 configuration should be saved automatically', async function (this: DifyWorld) { + await waitForAgentConfigureAutosaved(this.getPage()) +}) + +Then('the Agent v2 draft should be published and up to date', async function (this: DifyWorld) { + const page = this.getPage() + const agentId = getCurrentAgentId(this) + + await expect(page.getByRole('button', { name: 'Published' })).toBeVisible({ timeout: 30_000 }) + await expect(page.getByText('Up to date')).toBeVisible() + await expect.poll(async () => (await getTestAgent(agentId)).active_config_is_published).toBe(true) +}) diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index 4fa6f6fa86031d..fa1640a966b1bd 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -1,1131 +1 @@ -import type { DifyWorld } from '../features/support/world' -import { - agentBuilderExpectedTokens, - agentBuilderFixedInputs, - agentBuilderPreseededResources, -} from './agent-builder-resources' -import { createApiContext, expectApiResponseOK } from './api' -import { - agentBuilderFileTreeFixtureFileNames, - agentBuilderFileTreeFixtureFiles, - agentBuilderTestMaterials, -} from './test-materials' - -const stableChatModelProviderEnv = 'E2E_STABLE_MODEL_PROVIDER' -const stableChatModelNameEnv = 'E2E_STABLE_MODEL_NAME' -const stableChatModelTypeEnv = 'E2E_STABLE_MODEL_TYPE' -const brokenChatModelProviderEnv = 'E2E_BROKEN_MODEL_PROVIDER' -const brokenChatModelNameEnv = 'E2E_BROKEN_MODEL_NAME' -const brokenChatModelTypeEnv = 'E2E_BROKEN_MODEL_TYPE' -const activeModelStatus = 'active' -const defaultStableChatModelType = 'llm' -const defaultBrokenChatModelName = agentBuilderPreseededResources.brokenModel - -export type E2EResourcePrecondition - = | { - ok: true - value: string - } - | { - ok: false - reason: string - } - -export const readRequiredEnvResource = ( - envName: string, - description: string, -): E2EResourcePrecondition => { - const value = process.env[envName]?.trim() - if (value) - return { ok: true, value } - - return { - ok: false, - reason: `${description} requires ${envName}.`, - } -} - -export function skipBlockedPrecondition(world: DifyWorld, reason: string): 'skipped' { - const message = `Blocked precondition: ${reason}` - console.warn(`[e2e] ${message}`) - world.attach(message, 'text/plain') - return 'skipped' -} - -export function skipMissingEnvResource( - world: DifyWorld, - envName: string, - description: string, -): 'skipped' | string { - const resource = readRequiredEnvResource(envName, description) - if (resource.ok) - return resource.value - - return skipBlockedPrecondition(world, resource.reason) -} - -export const requiredAgentBuilderPreseededResources = Object.values(agentBuilderPreseededResources) - -export function skipMissingAgentBuilderPreseed( - world: DifyWorld, - resourceName: string, - envName: string, -): 'skipped' | string { - return skipMissingEnvResource( - world, - envName, - `Preseeded Agent Builder resource "${resourceName}"`, - ) -} - -type ModelTypeListResponse = { - data: Array<{ - provider: string - models: Array<{ - label?: { - en_US?: string - zh_Hans?: string - } - model: string - status?: string - }> - status?: string - }> -} - -type NamedResource = { - id: string - name: string -} - -type DatasetResource = NamedResource & { - document_count: number - total_available_documents: number -} - -type NamedResourceListResponse = { - data: T[] -} - -type DocumentIndexingStatus - = | 'cleaning' - | 'completed' - | 'indexing' - | 'parsing' - | 'splitting' - | 'waiting' - -type DatasetIndexingStatusResponse = { - data: Array<{ - id: string - indexing_status?: string - }> -} - -const completedDocumentIndexingStatus: DocumentIndexingStatus = 'completed' -const activeDocumentIndexingStatuses = new Set([ - 'cleaning', - 'indexing', - 'parsing', - 'splitting', - 'waiting', -]) - -type LocalizedLabel = { - en_US?: string - zh_Hans?: string -} - -type BuiltinToolProvider = { - label?: LocalizedLabel - name: string - tools: Array<{ - label?: LocalizedLabel - name: string - }> -} - -type AgentDriveSkillListResponse = { - items: Array<{ - name: string - path: string - }> -} - -type AgentDriveFileListResponse = { - items?: Array<{ - key: string - }> -} - -type AgentComposerResponse = { - agent_soul?: Record -} - -type AgentApiAccessResponse = { - api_key_count: number - enabled: boolean -} - -type AgentApiKeyListResponse = { - data: Array<{ - id: string - }> -} - -type AgentReferencingWorkflowsResponse = { - data: Array<{ - app_id: string - app_name: string - node_ids?: string[] - }> -} - -type PreseededAgentDetailResponse = { - active_config_is_published?: boolean - enable_site?: boolean - site?: { - access_token?: string | null - app_base_url?: string | null - code?: string | null - } | null -} - -const findConsoleResourceByName = async ({ - action, - path, - resourceName, -}: { - action: string - path: string - resourceName: string -}) => { - const ctx = await createApiContext() - try { - const response = await ctx.get(path) - await expectApiResponseOK(response, action) - const body = (await response.json()) as NamedResourceListResponse - - return body.data.find(item => item.name === resourceName) - } - finally { - await ctx.dispose() - } -} - -const buildQuery = (params: Record) => new URLSearchParams(params).toString() - -const matchesNameOrLabel = (value: string, name: string, label?: LocalizedLabel) => - value === name || value === label?.en_US || value === label?.zh_Hans - -const isRecord = (value: unknown): value is Record => - typeof value === 'object' && value !== null && !Array.isArray(value) - -const asRecord = (value: unknown): Record => (isRecord(value) ? value : {}) - -const asArray = (value: unknown): unknown[] => (Array.isArray(value) ? value : []) - -const asString = (value: unknown) => (typeof value === 'string' ? value : '') - -const hasNamedOrKeyedEntry = (items: unknown[], expectedName: string) => - items.some((item) => { - const record = asRecord(item) - const values = [record.name, record.drive_key, record.reference, record.file_id, record.id].map( - asString, - ) - - return values.some(value => value === expectedName || value.endsWith(`/${expectedName}`)) - }) - -const findToolEntry = ( - items: unknown[], - { - providerDisplayName, - providerName, - toolDisplayName, - toolName, - }: { - providerDisplayName: string - providerName: string - toolDisplayName: string - toolName: string - }, -) => - items.find((item) => { - const record = asRecord(item) - const providerValues = [record.provider_id, record.provider, record.plugin_id, record.name].map( - asString, - ) - const toolValues = [record.tool_name, record.name].map(asString) - - return ( - providerValues.some(value => value === providerName || value === providerDisplayName) - && toolValues.some(value => value === toolName || value === toolDisplayName) - ) - }) - -const hasToolEntry = ( - items: unknown[], - tool: { - providerDisplayName: string - providerName: string - toolDisplayName: string - toolName: string - }, -) => Boolean(findToolEntry(items, tool)) - -const hasUnauthorizedToolCredentialState = (item: unknown) => { - const record = asRecord(item) - - return asString(record.credential_type) === 'unauthorized' -} - -const hasKnowledgeDataset = ( - soul: Record, - dataset: NonNullable, -) => { - const knowledge = asRecord(soul.knowledge) - const sets = asArray(knowledge.sets) - - return sets.some((set) => { - const datasets = asArray(asRecord(set).datasets) - - return datasets.some((item) => { - const record = asRecord(item) - return record.id === dataset.id || record.name === dataset.name - }) - }) -} - -const hasKnowledgeSet = ( - soul: Record, - dataset: NonNullable, - { - queryMode, - queryValue, - }: { - queryMode: 'generated_query' | 'user_query' - queryValue?: string - }, -) => { - const knowledge = asRecord(soul.knowledge) - const sets = asArray(knowledge.sets) - - return sets.some((set) => { - const record = asRecord(set) - const query = asRecord(record.query) - const datasets = asArray(record.datasets) - const hasExpectedDataset = datasets.some((item) => { - const datasetRecord = asRecord(item) - return datasetRecord.id === dataset.id || datasetRecord.name === dataset.name - }) - - if (!hasExpectedDataset || query.mode !== queryMode) - return false - if (queryValue === undefined) - return true - - return asString(query.value).trim() === queryValue - }) -} - -const getPreseededDataset = async (resourceName: string) => { - const query = buildQuery({ keyword: resourceName, limit: '20', page: '1' }) - - return findConsoleResourceByName({ - action: `Check preseeded dataset ${resourceName}`, - path: `/console/api/datasets?${query}`, - resourceName, - }) -} - -const getDatasetIndexingStatuses = async (datasetId: string, resourceName: string) => { - const ctx = await createApiContext() - try { - const response = await ctx.get(`/console/api/datasets/${datasetId}/indexing-status`) - await expectApiResponseOK(response, `Check preseeded dataset indexing status ${resourceName}`) - const body = (await response.json()) as DatasetIndexingStatusResponse - - return body.data - } - finally { - await ctx.dispose() - } -} - -const toDatasetResource = ( - resource: NamedResource, -): NonNullable => ({ - id: resource.id, - kind: 'dataset', - name: resource.name, -}) - -const splitToolDisplayName = (resourceName: string) => { - const [providerName, toolName] = resourceName.split('/').map(item => item.trim()) - - if (!providerName || !toolName) { - return { - ok: false as const, - reason: `Preseeded tool "${resourceName}" must use "Provider / Tool" format.`, - } - } - - return { - ok: true as const, - providerName, - toolName, - } -} - -export async function skipMissingPreseededAgent( - world: DifyWorld, - resourceName: string, -): Promise<'skipped' | NonNullable> { - const query = buildQuery({ limit: '20', name: resourceName, page: '1' }) - const resource = await findConsoleResourceByName({ - action: `Check preseeded Agent ${resourceName}`, - path: `/console/api/agent?${query}`, - resourceName, - }) - - if (!resource) - return skipBlockedPrecondition(world, `Preseeded Agent "${resourceName}" was not found.`) - - return { - id: resource.id, - kind: 'agent', - name: resource.name, - } -} - -export async function skipMissingPreseededWorkflow( - world: DifyWorld, - resourceName: string, -): Promise<'skipped' | NonNullable> { - const query = buildQuery({ limit: '20', mode: 'workflow', name: resourceName, page: '1' }) - const resource = await findConsoleResourceByName({ - action: `Check preseeded workflow ${resourceName}`, - path: `/console/api/apps?${query}`, - resourceName, - }) - - if (!resource) - return skipBlockedPrecondition(world, `Preseeded workflow "${resourceName}" was not found.`) - - return { - id: resource.id, - kind: 'workflow', - name: resource.name, - } -} - -export async function skipMissingPreseededDataset( - world: DifyWorld, - resourceName: string, -): Promise<'skipped' | NonNullable> { - const resource = await getPreseededDataset(resourceName) - - if (!resource) - return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`) - - return toDatasetResource(resource) -} - -export async function skipMissingReadyPreseededDataset( - world: DifyWorld, - resourceName: string, -): Promise<'skipped' | NonNullable> { - const resource = await getPreseededDataset(resourceName) - - if (!resource) - return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`) - - if (resource.document_count < 1) { - return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" has no documents.`) - } - - if (resource.total_available_documents !== resource.document_count) { - return skipBlockedPrecondition( - world, - `Preseeded dataset "${resourceName}" has ${resource.total_available_documents}/${resource.document_count} available documents.`, - ) - } - - const statuses = await getDatasetIndexingStatuses(resource.id, resourceName) - if (statuses.length < 1) { - return skipBlockedPrecondition( - world, - `Preseeded dataset "${resourceName}" has no document indexing status.`, - ) - } - - const incompleteStatus = statuses.find( - item => item.indexing_status !== completedDocumentIndexingStatus, - ) - if (incompleteStatus) { - return skipBlockedPrecondition( - world, - `Preseeded dataset "${resourceName}" includes document ${incompleteStatus.id} with indexing status "${incompleteStatus.indexing_status ?? 'missing'}".`, - ) - } - - return toDatasetResource(resource) -} - -export async function skipMissingIndexingPreseededDataset( - world: DifyWorld, - resourceName: string, -): Promise<'skipped' | NonNullable> { - const resource = await getPreseededDataset(resourceName) - - if (!resource) - return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`) - - const statuses = await getDatasetIndexingStatuses(resource.id, resourceName) - const indexingStatus = statuses.find(item => - activeDocumentIndexingStatuses.has(item.indexing_status ?? ''), - ) - - if (!indexingStatus) { - const actualStatuses - = statuses.map(item => item.indexing_status ?? 'missing').join(', ') || 'none' - - return skipBlockedPrecondition( - world, - `Preseeded dataset "${resourceName}" is not indexing or queued; document statuses: ${actualStatuses}.`, - ) - } - - return toDatasetResource(resource) -} - -export async function skipMissingPreseededTool( - world: DifyWorld, - resourceName: string, -): Promise<'skipped' | NonNullable> { - const parsed = splitToolDisplayName(resourceName) - if (!parsed.ok) - return skipBlockedPrecondition(world, parsed.reason) - - const ctx = await createApiContext() - try { - const response = await ctx.get('/console/api/workspaces/current/tools/builtin') - await expectApiResponseOK(response, `Check preseeded tool ${resourceName}`) - const providers = (await response.json()) as BuiltinToolProvider[] - const provider = providers.find(item => - matchesNameOrLabel(parsed.providerName, item.name, item.label), - ) - const tool = provider?.tools.find(item => - matchesNameOrLabel(parsed.toolName, item.name, item.label), - ) - - if (!provider || !tool) - return skipBlockedPrecondition(world, `Preseeded tool "${resourceName}" was not found.`) - - return { - id: `${provider.name}/${tool.name}`, - kind: 'tool', - name: resourceName, - } - } - finally { - await ctx.dispose() - } -} - -export async function skipMissingPreseededAgentDriveSkill( - world: DifyWorld, - agentName: string, - skillName: string, -): Promise<'skipped' | NonNullable> { - const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent - - const ctx = await createApiContext() - try { - const response = await ctx.get(`/console/api/agent/${agent.id}/drive/skills`) - await expectApiResponseOK(response, `Check preseeded Agent skill ${skillName}`) - const body = (await response.json()) as AgentDriveSkillListResponse - const skill = body.items.find(item => item.name === skillName) - - if (!skill) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" does not include drive skill "${skillName}".`, - ) - } - - return { - id: skill.path, - kind: 'skill', - name: skill.name, - } - } - finally { - await ctx.dispose() - } -} - -export async function skipMissingPreseededFullConfigAgentCoreConfiguration( - world: DifyWorld, - agentName: string, -): Promise<'skipped' | NonNullable> { - const stableModel = await skipMissingAgentBuilderStableChatModel(world) - if (stableModel === 'skipped') - return stableModel - - const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent - - const summarySkill = await skipMissingPreseededAgentDriveSkill( - world, - agentName, - agentBuilderPreseededResources.summarySkill, - ) - if (summarySkill === 'skipped') - return summarySkill - - const jsonTool = await skipMissingPreseededTool( - world, - agentBuilderPreseededResources.jsonReplaceTool, - ) - if (jsonTool === 'skipped') - return jsonTool - - const knowledgeBase = await skipMissingReadyPreseededDataset( - world, - agentBuilderPreseededResources.agentKnowledgeBase, - ) - if (knowledgeBase === 'skipped') - return knowledgeBase - - const ctx = await createApiContext() - try { - const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) - await expectApiResponseOK(response, `Check preseeded Agent core configuration ${agentName}`) - const body = (await response.json()) as AgentComposerResponse - const soul = body.agent_soul ?? {} - const missing: string[] = [] - - const model = asRecord(soul.model) - if (model.model_provider !== stableModel.provider || model.model !== stableModel.name) - missing.push(`${agentBuilderPreseededResources.stableChatModel} model config`) - - const prompt = asString(asRecord(soul.prompt).system_prompt) - if (!prompt.includes(agentBuilderExpectedTokens.agentReply)) - missing.push(`Prompt token ${agentBuilderExpectedTokens.agentReply}`) - - const files = asArray(asRecord(soul.files).files) - for (const fileName of [ - agentBuilderTestMaterials.smallFile, - agentBuilderTestMaterials.specialFilename, - ]) { - if (!hasNamedOrKeyedEntry(files, fileName)) - missing.push(`file ${fileName}`) - } - - const [providerName = '', toolName = ''] = jsonTool.id.split('/') - const parsedTool = splitToolDisplayName(agentBuilderPreseededResources.jsonReplaceTool) - if ( - parsedTool.ok - && !hasToolEntry(asArray(asRecord(soul.tools).dify_tools), { - providerDisplayName: parsedTool.providerName, - providerName, - toolDisplayName: parsedTool.toolName, - toolName, - }) - ) { - missing.push(agentBuilderPreseededResources.jsonReplaceTool) - } - - if (!hasKnowledgeDataset(soul, knowledgeBase)) - missing.push(agentBuilderPreseededResources.agentKnowledgeBase) - - if (missing.length > 0) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" is missing core fixture configuration: ${missing.join(', ')}.`, - ) - } - - return agent - } - finally { - await ctx.dispose() - } -} - -export async function skipMissingPreseededToolStatesAgentConfiguration( - world: DifyWorld, - agentName: string, -): Promise<'skipped' | NonNullable> { - const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent - - const summarySkill = await skipMissingPreseededAgentDriveSkill( - world, - agentName, - agentBuilderPreseededResources.summarySkill, - ) - if (summarySkill === 'skipped') - return summarySkill - - const jsonTool = await skipMissingPreseededTool( - world, - agentBuilderPreseededResources.jsonReplaceTool, - ) - if (jsonTool === 'skipped') - return jsonTool - - const tavilyTool = await skipMissingPreseededTool( - world, - agentBuilderPreseededResources.tavilySearchTool, - ) - if (tavilyTool === 'skipped') - return tavilyTool - - const ctx = await createApiContext() - try { - const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) - await expectApiResponseOK(response, `Check preseeded Agent tool states ${agentName}`) - const body = (await response.json()) as AgentComposerResponse - const soul = body.agent_soul ?? {} - const toolItems = asArray(asRecord(soul.tools).dify_tools) - const missing: string[] = [] - - const [jsonProviderName = '', jsonToolName = ''] = jsonTool.id.split('/') - const parsedJsonTool = splitToolDisplayName(agentBuilderPreseededResources.jsonReplaceTool) - if ( - parsedJsonTool.ok - && !findToolEntry(toolItems, { - providerDisplayName: parsedJsonTool.providerName, - providerName: jsonProviderName, - toolDisplayName: parsedJsonTool.toolName, - toolName: jsonToolName, - }) - ) { - missing.push(agentBuilderPreseededResources.jsonReplaceTool) - } - - const [tavilyProviderName = '', tavilyToolName = ''] = tavilyTool.id.split('/') - const parsedTavilyTool = splitToolDisplayName(agentBuilderPreseededResources.tavilySearchTool) - const tavilyEntry = parsedTavilyTool.ok - ? findToolEntry(toolItems, { - providerDisplayName: parsedTavilyTool.providerName, - providerName: tavilyProviderName, - toolDisplayName: parsedTavilyTool.toolName, - toolName: tavilyToolName, - }) - : undefined - - if (!tavilyEntry) { - missing.push(agentBuilderPreseededResources.tavilySearchTool) - } - else if (!hasUnauthorizedToolCredentialState(tavilyEntry)) { - missing.push(`${agentBuilderPreseededResources.tavilySearchTool} unauthorized credential state`) - } - - if (missing.length > 0) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" is missing tool state fixture configuration: ${missing.join(', ')}.`, - ) - } - - return agent - } - finally { - await ctx.dispose() - } -} - -export async function skipMissingPreseededDualRetrievalAgentConfiguration( - world: DifyWorld, - agentName: string, -): Promise<'skipped' | NonNullable> { - const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent - - const knowledgeBase = await skipMissingReadyPreseededDataset( - world, - agentBuilderPreseededResources.agentKnowledgeBase, - ) - if (knowledgeBase === 'skipped') - return knowledgeBase - - const ctx = await createApiContext() - try { - const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) - await expectApiResponseOK(response, `Check preseeded Agent dual retrieval ${agentName}`) - const body = (await response.json()) as AgentComposerResponse - const soul = body.agent_soul ?? {} - const missing: string[] = [] - - if (!hasKnowledgeSet(soul, knowledgeBase, { queryMode: 'generated_query' })) - missing.push('Agent decide Knowledge Retrieval') - - if ( - !hasKnowledgeSet(soul, knowledgeBase, { - queryMode: 'user_query', - queryValue: agentBuilderFixedInputs.customKnowledgeQuery, - }) - ) { - missing.push('Custom query Knowledge Retrieval') - } - - if (missing.length > 0) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" is missing dual retrieval fixture configuration: ${missing.join(', ')}.`, - ) - } - - return agent - } - finally { - await ctx.dispose() - } -} - -export async function skipMissingPreseededAgentFileTreeFixture( - world: DifyWorld, - agentName: string, -): Promise<'skipped' | NonNullable> { - const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent - - const ctx = await createApiContext() - try { - const query = buildQuery({ prefix: 'files/' }) - const response = await ctx.get(`/console/api/agent/${agent.id}/drive/files?${query}`) - await expectApiResponseOK(response, `Check preseeded Agent file tree ${agentName}`) - const body = (await response.json()) as AgentDriveFileListResponse - const keys = (body.items ?? []).map(item => item.key) - const missingFiles = agentBuilderFileTreeFixtureFiles.filter( - filePath => - !keys.some(key => key === `files/${filePath}` || key.endsWith(`/${filePath}`)), - ) - - if (missingFiles.length > 0) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" is missing file tree fixture files: ${missingFiles.join(', ')}.`, - ) - } - - return { - id: agent.id, - kind: 'agent', - name: agent.name, - } - } - finally { - await ctx.dispose() - } -} - -export async function skipMissingPreseededAgentFlatFileFixtureConfiguration( - world: DifyWorld, - agentName: string, -): Promise<'skipped' | NonNullable> { - const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent - - const ctx = await createApiContext() - try { - const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) - await expectApiResponseOK(response, `Check preseeded Agent flat file fixture ${agentName}`) - const body = (await response.json()) as AgentComposerResponse - const configFiles = Array.isArray(body.agent_soul?.config_files) - ? body.agent_soul.config_files - : [] - const fileNames = configFiles - .map(file => (typeof file === 'object' && file !== null && 'name' in file ? file.name : undefined)) - .filter((name): name is string => typeof name === 'string') - const missingFiles = agentBuilderFileTreeFixtureFileNames.filter(fileName => !fileNames.includes(fileName)) - - if (missingFiles.length > 0) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" is missing current flat Files fixture configuration: ${missingFiles.join(', ')}. Hierarchical Files display remains blocked until Agent config files support tree paths.`, - ) - } - - return { - id: agent.id, - kind: 'agent', - name: agent.name, - } - } - finally { - await ctx.dispose() - } -} - -export async function skipMissingPreseededAgentBackendApiKey( - world: DifyWorld, - agentName: string, -): Promise<'skipped' | NonNullable> { - const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent - - const ctx = await createApiContext() - try { - const accessResponse = await ctx.get(`/console/api/agent/${agent.id}/api-access`) - await expectApiResponseOK(accessResponse, `Check preseeded Agent API access ${agentName}`) - const access = (await accessResponse.json()) as AgentApiAccessResponse - if (!access.enabled || access.api_key_count < 1) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" does not have Backend service API enabled with an API key.`, - ) - } - - const keyResponse = await ctx.get(`/console/api/agent/${agent.id}/api-keys`) - await expectApiResponseOK(keyResponse, `Check preseeded Agent API key ${agentName}`) - const keys = (await keyResponse.json()) as AgentApiKeyListResponse - const key = keys.data.at(0) - if (!key) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" Backend service API key list is empty.`, - ) - } - - return { - id: key.id, - kind: 'api-key', - name: `${agentName} Backend service API key`, - } - } - finally { - await ctx.dispose() - } -} - -export async function skipMissingPreseededAgentPublishedWebApp( - world: DifyWorld, - agentName: string, -): Promise<'skipped' | NonNullable> { - const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent - - const ctx = await createApiContext() - try { - const response = await ctx.get(`/console/api/agent/${agent.id}`) - await expectApiResponseOK(response, `Check preseeded Agent published Web app ${agentName}`) - const detail = (await response.json()) as PreseededAgentDetailResponse - if (detail.active_config_is_published !== true) { - return skipBlockedPrecondition(world, `Preseeded Agent "${agentName}" is not published.`) - } - - if (detail.enable_site !== true) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" Web app is not enabled.`, - ) - } - - const siteToken = detail.site?.access_token ?? detail.site?.code - if (!siteToken || !detail.site?.app_base_url) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" Web app URL is not available.`, - ) - } - - return { - id: agent.id, - kind: 'agent', - name: agent.name, - } - } - finally { - await ctx.dispose() - } -} - -export async function skipMissingPreseededAgentWorkflowReference( - world: DifyWorld, - agentName: string, - workflowName: string, -): Promise<'skipped' | NonNullable> { - const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent - - const workflow = await skipMissingPreseededWorkflow(world, workflowName) - if (workflow === 'skipped') - return workflow - - const ctx = await createApiContext() - try { - const response = await ctx.get(`/console/api/agent/${agent.id}/referencing-workflows`) - await expectApiResponseOK(response, `Check preseeded Agent workflow reference ${agentName}`) - const references = (await response.json()) as AgentReferencingWorkflowsResponse - const reference = references.data.find( - item => item.app_id === workflow.id || item.app_name === workflow.name, - ) - - if (!reference) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" is not referenced by workflow "${workflowName}".`, - ) - } - - if (!reference.node_ids || reference.node_ids.length < 1) { - return skipBlockedPrecondition( - world, - `Preseeded workflow "${workflowName}" does not expose Agent reference nodes for "${agentName}".`, - ) - } - - return { - id: workflow.id, - kind: 'workflow', - name: workflow.name, - } - } - finally { - await ctx.dispose() - } -} - -type ModelPreflightConfig - = | { - ok: true - provider: string - resourceName: string - type: string - value: string - } - | { - ok: false - reason: string - } - -export function readAgentBuilderStableChatModelConfig(): ModelPreflightConfig { - const provider = process.env[stableChatModelProviderEnv]?.trim() - const name = process.env[stableChatModelNameEnv]?.trim() - const type = process.env[stableChatModelTypeEnv]?.trim() || defaultStableChatModelType - - const missing: string[] = [] - if (!provider) - missing.push(stableChatModelProviderEnv) - if (!name) - missing.push(stableChatModelNameEnv) - - if (!provider || !name) { - return { - ok: false, - reason: `${agentBuilderPreseededResources.stableChatModel} requires ${missing.join(', ')}.`, - } - } - - return { - ok: true, - provider, - resourceName: agentBuilderPreseededResources.stableChatModel, - type, - value: name, - } -} - -export function readAgentBuilderBrokenChatModelConfig(): ModelPreflightConfig { - const provider = process.env[brokenChatModelProviderEnv]?.trim() - const name = process.env[brokenChatModelNameEnv]?.trim() || defaultBrokenChatModelName - const type = process.env[brokenChatModelTypeEnv]?.trim() || defaultStableChatModelType - - if (!provider) { - return { - ok: false, - reason: `${agentBuilderPreseededResources.brokenModelProvider} requires ${brokenChatModelProviderEnv}.`, - } - } - - return { - ok: true, - provider, - resourceName: agentBuilderPreseededResources.brokenModelProvider, - type, - value: name, - } -} - -async function skipMissingAgentBuilderModel( - world: DifyWorld, - config: ModelPreflightConfig, - { - requireActive, - }: { - requireActive: boolean - }, -): Promise<'skipped' | NonNullable> { - if (!config.ok) - return skipBlockedPrecondition(world, config.reason) - - const ctx = await createApiContext() - try { - const response = await ctx.get( - `/console/api/workspaces/current/models/model-types/${config.type}`, - ) - await expectApiResponseOK(response, `Check ${config.resourceName}`) - const body = (await response.json()) as ModelTypeListResponse - const provider = body.data.find(item => item.provider === config.provider) - const model = provider?.models.find( - item => - item.model === config.value - || item.label?.en_US === config.value - || item.label?.zh_Hans === config.value, - ) - - if (!provider || !model) { - return skipBlockedPrecondition( - world, - `${config.resourceName} was not found as ${config.provider}/${config.value} (${config.type}).`, - ) - } - - if (requireActive && model.status !== activeModelStatus) { - return skipBlockedPrecondition( - world, - `${config.resourceName} is ${model.status ?? 'missing status'} instead of ${activeModelStatus}.`, - ) - } - - return { - name: model.model, - provider: provider.provider, - type: config.type, - } - } - finally { - await ctx.dispose() - } -} - -export async function skipMissingAgentBuilderStableChatModel( - world: DifyWorld, -): Promise<'skipped' | NonNullable> { - return skipMissingAgentBuilderModel(world, readAgentBuilderStableChatModelConfig(), { - requireActive: true, - }) -} - -export async function skipMissingAgentBuilderBrokenChatModel( - world: DifyWorld, -): Promise<'skipped' | NonNullable> { - return skipMissingAgentBuilderModel(world, readAgentBuilderBrokenChatModelConfig(), { - requireActive: false, - }) -} +export * from './preflight/index' diff --git a/e2e/support/preflight/access.ts b/e2e/support/preflight/access.ts new file mode 100644 index 00000000000000..e04be75ec799ae --- /dev/null +++ b/e2e/support/preflight/access.ts @@ -0,0 +1,166 @@ +import type { DifyWorld } from '../../features/support/world' +import type { PreseededResource } from './common' +import { createApiContext, expectApiResponseOK } from '../api' +import { skipMissingPreseededAgent, skipMissingPreseededWorkflow } from './agents' +import { skipBlockedPrecondition } from './common' + +type AgentApiAccessResponse = { + api_key_count: number + enabled: boolean +} + +type AgentApiKeyListResponse = { + data: Array<{ + id: string + }> +} + +type AgentReferencingWorkflowsResponse = { + data: Array<{ + app_id: string + app_name: string + node_ids?: string[] + }> +} + +type PreseededAgentDetailResponse = { + active_config_is_published?: boolean + enable_site?: boolean + site?: { + access_token?: string | null + app_base_url?: string | null + code?: string | null + } | null +} + +export async function skipMissingPreseededAgentBackendApiKey( + world: DifyWorld, + agentName: string, +): Promise<'skipped' | PreseededResource> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const ctx = await createApiContext() + try { + const accessResponse = await ctx.get(`/console/api/agent/${agent.id}/api-access`) + await expectApiResponseOK(accessResponse, `Check preseeded Agent API access ${agentName}`) + const access = (await accessResponse.json()) as AgentApiAccessResponse + if (!access.enabled || access.api_key_count < 1) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" does not have Backend service API enabled with an API key.`, + ) + } + + const keyResponse = await ctx.get(`/console/api/agent/${agent.id}/api-keys`) + await expectApiResponseOK(keyResponse, `Check preseeded Agent API key ${agentName}`) + const keys = (await keyResponse.json()) as AgentApiKeyListResponse + const key = keys.data.at(0) + if (!key) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" Backend service API key list is empty.`, + ) + } + + return { + id: key.id, + kind: 'api-key', + name: `${agentName} Backend service API key`, + } + } + finally { + await ctx.dispose() + } +} + +export async function skipMissingPreseededAgentPublishedWebApp( + world: DifyWorld, + agentName: string, +): Promise<'skipped' | PreseededResource> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agent.id}`) + await expectApiResponseOK(response, `Check preseeded Agent published Web app ${agentName}`) + const detail = (await response.json()) as PreseededAgentDetailResponse + if (detail.active_config_is_published !== true) { + return skipBlockedPrecondition(world, `Preseeded Agent "${agentName}" is not published.`) + } + + if (detail.enable_site !== true) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" Web app is not enabled.`, + ) + } + + const siteToken = detail.site?.access_token ?? detail.site?.code + if (!siteToken || !detail.site?.app_base_url) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" Web app URL is not available.`, + ) + } + + return { + id: agent.id, + kind: 'agent', + name: agent.name, + } + } + finally { + await ctx.dispose() + } +} + +export async function skipMissingPreseededAgentWorkflowReference( + world: DifyWorld, + agentName: string, + workflowName: string, +): Promise<'skipped' | PreseededResource> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const workflow = await skipMissingPreseededWorkflow(world, workflowName) + if (workflow === 'skipped') + return workflow + + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agent.id}/referencing-workflows`) + await expectApiResponseOK(response, `Check preseeded Agent workflow reference ${agentName}`) + const references = (await response.json()) as AgentReferencingWorkflowsResponse + const reference = references.data.find( + item => item.app_id === workflow.id || item.app_name === workflow.name, + ) + + if (!reference) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" is not referenced by workflow "${workflowName}".`, + ) + } + + if (!reference.node_ids || reference.node_ids.length < 1) { + return skipBlockedPrecondition( + world, + `Preseeded workflow "${workflowName}" does not expose Agent reference nodes for "${agentName}".`, + ) + } + + return { + id: workflow.id, + kind: 'workflow', + name: workflow.name, + } + } + finally { + await ctx.dispose() + } +} diff --git a/e2e/support/preflight/agents.ts b/e2e/support/preflight/agents.ts new file mode 100644 index 00000000000000..90bdde7a12e62d --- /dev/null +++ b/e2e/support/preflight/agents.ts @@ -0,0 +1,472 @@ +import type { DifyWorld } from '../../features/support/world' +import type { AgentComposerResponse, PreseededResource } from './common' +import { + agentBuilderExpectedTokens, + agentBuilderFixedInputs, + agentBuilderPreseededResources, +} from '../agent-builder-resources' +import { createApiContext, expectApiResponseOK } from '../api' +import { + agentBuilderFileTreeFixtureFileNames, + agentBuilderFileTreeFixtureFiles, + agentBuilderTestMaterials, +} from '../test-materials' +import { + + asArray, + asRecord, + asString, + buildQuery, + findConsoleResourceByName, + hasNamedOrKeyedEntry, + + skipBlockedPrecondition, +} from './common' +import { skipMissingReadyPreseededDataset } from './datasets' +import { skipMissingAgentBuilderStableChatModel } from './models' +import { + findToolEntry, + hasToolEntry, + hasUnauthorizedToolCredentialState, + skipMissingPreseededTool, + splitToolDisplayName, +} from './tools' + +type AgentDriveSkillListResponse = { + items: Array<{ + name: string + path: string + }> +} + +type AgentDriveFileListResponse = { + items?: Array<{ + key: string + }> +} + +const hasKnowledgeDataset = ( + soul: Record, + dataset: PreseededResource, +) => { + const knowledge = asRecord(soul.knowledge) + const sets = asArray(knowledge.sets) + + return sets.some((set) => { + const datasets = asArray(asRecord(set).datasets) + + return datasets.some((item) => { + const record = asRecord(item) + return record.id === dataset.id || record.name === dataset.name + }) + }) +} + +const hasKnowledgeSet = ( + soul: Record, + dataset: PreseededResource, + { + queryMode, + queryValue, + }: { + queryMode: 'generated_query' | 'user_query' + queryValue?: string + }, +) => { + const knowledge = asRecord(soul.knowledge) + const sets = asArray(knowledge.sets) + + return sets.some((set) => { + const record = asRecord(set) + const query = asRecord(record.query) + const datasets = asArray(record.datasets) + const hasExpectedDataset = datasets.some((item) => { + const datasetRecord = asRecord(item) + return datasetRecord.id === dataset.id || datasetRecord.name === dataset.name + }) + + if (!hasExpectedDataset || query.mode !== queryMode) + return false + if (queryValue === undefined) + return true + + return asString(query.value).trim() === queryValue + }) +} + +export async function skipMissingPreseededAgent( + world: DifyWorld, + resourceName: string, +): Promise<'skipped' | PreseededResource> { + const query = buildQuery({ limit: '20', name: resourceName, page: '1' }) + const resource = await findConsoleResourceByName({ + action: `Check preseeded Agent ${resourceName}`, + path: `/console/api/agent?${query}`, + resourceName, + }) + + if (!resource) + return skipBlockedPrecondition(world, `Preseeded Agent "${resourceName}" was not found.`) + + return { + id: resource.id, + kind: 'agent', + name: resource.name, + } +} + +export async function skipMissingPreseededWorkflow( + world: DifyWorld, + resourceName: string, +): Promise<'skipped' | PreseededResource> { + const query = buildQuery({ limit: '20', mode: 'workflow', name: resourceName, page: '1' }) + const resource = await findConsoleResourceByName({ + action: `Check preseeded workflow ${resourceName}`, + path: `/console/api/apps?${query}`, + resourceName, + }) + + if (!resource) + return skipBlockedPrecondition(world, `Preseeded workflow "${resourceName}" was not found.`) + + return { + id: resource.id, + kind: 'workflow', + name: resource.name, + } +} + +export async function skipMissingPreseededAgentDriveSkill( + world: DifyWorld, + agentName: string, + skillName: string, +): Promise<'skipped' | PreseededResource> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agent.id}/drive/skills`) + await expectApiResponseOK(response, `Check preseeded Agent skill ${skillName}`) + const body = (await response.json()) as AgentDriveSkillListResponse + const skill = body.items.find(item => item.name === skillName) + + if (!skill) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" does not include drive skill "${skillName}".`, + ) + } + + return { + id: skill.path, + kind: 'skill', + name: skill.name, + } + } + finally { + await ctx.dispose() + } +} + +export async function skipMissingPreseededFullConfigAgentCoreConfiguration( + world: DifyWorld, + agentName: string, +): Promise<'skipped' | PreseededResource> { + const stableModel = await skipMissingAgentBuilderStableChatModel(world) + if (stableModel === 'skipped') + return stableModel + + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const summarySkill = await skipMissingPreseededAgentDriveSkill( + world, + agentName, + agentBuilderPreseededResources.summarySkill, + ) + if (summarySkill === 'skipped') + return summarySkill + + const jsonTool = await skipMissingPreseededTool( + world, + agentBuilderPreseededResources.jsonReplaceTool, + ) + if (jsonTool === 'skipped') + return jsonTool + + const knowledgeBase = await skipMissingReadyPreseededDataset( + world, + agentBuilderPreseededResources.agentKnowledgeBase, + ) + if (knowledgeBase === 'skipped') + return knowledgeBase + + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) + await expectApiResponseOK(response, `Check preseeded Agent core configuration ${agentName}`) + const body = (await response.json()) as AgentComposerResponse + const soul = body.agent_soul ?? {} + const missing: string[] = [] + + const model = asRecord(soul.model) + if (model.model_provider !== stableModel.provider || model.model !== stableModel.name) + missing.push(`${agentBuilderPreseededResources.stableChatModel} model config`) + + const prompt = asString(asRecord(soul.prompt).system_prompt) + if (!prompt.includes(agentBuilderExpectedTokens.agentReply)) + missing.push(`Prompt token ${agentBuilderExpectedTokens.agentReply}`) + + const files = asArray(asRecord(soul.files).files) + for (const fileName of [ + agentBuilderTestMaterials.smallFile, + agentBuilderTestMaterials.specialFilename, + ]) { + if (!hasNamedOrKeyedEntry(files, fileName)) + missing.push(`file ${fileName}`) + } + + const [providerName = '', toolName = ''] = jsonTool.id.split('/') + const parsedTool = splitToolDisplayName(agentBuilderPreseededResources.jsonReplaceTool) + if ( + parsedTool.ok + && !hasToolEntry(asArray(asRecord(soul.tools).dify_tools), { + providerDisplayName: parsedTool.providerName, + providerName, + toolDisplayName: parsedTool.toolName, + toolName, + }) + ) { + missing.push(agentBuilderPreseededResources.jsonReplaceTool) + } + + if (!hasKnowledgeDataset(soul, knowledgeBase)) + missing.push(agentBuilderPreseededResources.agentKnowledgeBase) + + if (missing.length > 0) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" is missing core fixture configuration: ${missing.join(', ')}.`, + ) + } + + return agent + } + finally { + await ctx.dispose() + } +} + +export async function skipMissingPreseededToolStatesAgentConfiguration( + world: DifyWorld, + agentName: string, +): Promise<'skipped' | PreseededResource> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const summarySkill = await skipMissingPreseededAgentDriveSkill( + world, + agentName, + agentBuilderPreseededResources.summarySkill, + ) + if (summarySkill === 'skipped') + return summarySkill + + const jsonTool = await skipMissingPreseededTool( + world, + agentBuilderPreseededResources.jsonReplaceTool, + ) + if (jsonTool === 'skipped') + return jsonTool + + const tavilyTool = await skipMissingPreseededTool( + world, + agentBuilderPreseededResources.tavilySearchTool, + ) + if (tavilyTool === 'skipped') + return tavilyTool + + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) + await expectApiResponseOK(response, `Check preseeded Agent tool states ${agentName}`) + const body = (await response.json()) as AgentComposerResponse + const soul = body.agent_soul ?? {} + const toolItems = asArray(asRecord(soul.tools).dify_tools) + const missing: string[] = [] + + const [jsonProviderName = '', jsonToolName = ''] = jsonTool.id.split('/') + const parsedJsonTool = splitToolDisplayName(agentBuilderPreseededResources.jsonReplaceTool) + if ( + parsedJsonTool.ok + && !findToolEntry(toolItems, { + providerDisplayName: parsedJsonTool.providerName, + providerName: jsonProviderName, + toolDisplayName: parsedJsonTool.toolName, + toolName: jsonToolName, + }) + ) { + missing.push(agentBuilderPreseededResources.jsonReplaceTool) + } + + const [tavilyProviderName = '', tavilyToolName = ''] = tavilyTool.id.split('/') + const parsedTavilyTool = splitToolDisplayName(agentBuilderPreseededResources.tavilySearchTool) + const tavilyEntry = parsedTavilyTool.ok + ? findToolEntry(toolItems, { + providerDisplayName: parsedTavilyTool.providerName, + providerName: tavilyProviderName, + toolDisplayName: parsedTavilyTool.toolName, + toolName: tavilyToolName, + }) + : undefined + + if (!tavilyEntry) { + missing.push(agentBuilderPreseededResources.tavilySearchTool) + } + else if (!hasUnauthorizedToolCredentialState(tavilyEntry)) { + missing.push(`${agentBuilderPreseededResources.tavilySearchTool} unauthorized credential state`) + } + + if (missing.length > 0) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" is missing tool state fixture configuration: ${missing.join(', ')}.`, + ) + } + + return agent + } + finally { + await ctx.dispose() + } +} + +export async function skipMissingPreseededDualRetrievalAgentConfiguration( + world: DifyWorld, + agentName: string, +): Promise<'skipped' | PreseededResource> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const knowledgeBase = await skipMissingReadyPreseededDataset( + world, + agentBuilderPreseededResources.agentKnowledgeBase, + ) + if (knowledgeBase === 'skipped') + return knowledgeBase + + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) + await expectApiResponseOK(response, `Check preseeded Agent dual retrieval ${agentName}`) + const body = (await response.json()) as AgentComposerResponse + const soul = body.agent_soul ?? {} + const missing: string[] = [] + + if (!hasKnowledgeSet(soul, knowledgeBase, { queryMode: 'generated_query' })) + missing.push('Agent decide Knowledge Retrieval') + + if ( + !hasKnowledgeSet(soul, knowledgeBase, { + queryMode: 'user_query', + queryValue: agentBuilderFixedInputs.customKnowledgeQuery, + }) + ) { + missing.push('Custom query Knowledge Retrieval') + } + + if (missing.length > 0) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" is missing dual retrieval fixture configuration: ${missing.join(', ')}.`, + ) + } + + return agent + } + finally { + await ctx.dispose() + } +} + +export async function skipMissingPreseededAgentFileTreeFixture( + world: DifyWorld, + agentName: string, +): Promise<'skipped' | PreseededResource> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const ctx = await createApiContext() + try { + const query = buildQuery({ prefix: 'files/' }) + const response = await ctx.get(`/console/api/agent/${agent.id}/drive/files?${query}`) + await expectApiResponseOK(response, `Check preseeded Agent file tree ${agentName}`) + const body = (await response.json()) as AgentDriveFileListResponse + const keys = (body.items ?? []).map(item => item.key) + const missingFiles = agentBuilderFileTreeFixtureFiles.filter( + filePath => + !keys.some(key => key === `files/${filePath}` || key.endsWith(`/${filePath}`)), + ) + + if (missingFiles.length > 0) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" is missing file tree fixture files: ${missingFiles.join(', ')}.`, + ) + } + + return { + id: agent.id, + kind: 'agent', + name: agent.name, + } + } + finally { + await ctx.dispose() + } +} + +export async function skipMissingPreseededAgentFlatFileFixtureConfiguration( + world: DifyWorld, + agentName: string, +): Promise<'skipped' | PreseededResource> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) + await expectApiResponseOK(response, `Check preseeded Agent flat file fixture ${agentName}`) + const body = (await response.json()) as AgentComposerResponse + const configFiles = Array.isArray(body.agent_soul?.config_files) + ? body.agent_soul.config_files + : [] + const fileNames = configFiles + .map(file => (typeof file === 'object' && file !== null && 'name' in file ? file.name : undefined)) + .filter((name): name is string => typeof name === 'string') + const missingFiles = agentBuilderFileTreeFixtureFileNames.filter(fileName => !fileNames.includes(fileName)) + + if (missingFiles.length > 0) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" is missing current flat Files fixture configuration: ${missingFiles.join(', ')}. Hierarchical Files display remains blocked until Agent config files support tree paths.`, + ) + } + + return { + id: agent.id, + kind: 'agent', + name: agent.name, + } + } + finally { + await ctx.dispose() + } +} diff --git a/e2e/support/preflight/common.ts b/e2e/support/preflight/common.ts new file mode 100644 index 00000000000000..90b491338b7f55 --- /dev/null +++ b/e2e/support/preflight/common.ts @@ -0,0 +1,128 @@ +import type { DifyWorld } from '../../features/support/world' +import { agentBuilderPreseededResources } from '../agent-builder-resources' +import { createApiContext, expectApiResponseOK } from '../api' + +export type PreseededResource = NonNullable< + DifyWorld['agentBuilder']['preflight']['preseededResources'][string] +> + +export type E2EResourcePrecondition + = | { + ok: true + value: string + } + | { + ok: false + reason: string + } + +export type NamedResource = { + id: string + name: string +} + +export type NamedResourceListResponse = { + data: T[] +} + +export type LocalizedLabel = { + en_US?: string + zh_Hans?: string +} + +export type AgentComposerResponse = { + agent_soul?: Record +} + +export const readRequiredEnvResource = ( + envName: string, + description: string, +): E2EResourcePrecondition => { + const value = process.env[envName]?.trim() + if (value) + return { ok: true, value } + + return { + ok: false, + reason: `${description} requires ${envName}.`, + } +} + +export function skipBlockedPrecondition(world: DifyWorld, reason: string): 'skipped' { + const message = `Blocked precondition: ${reason}` + console.warn(`[e2e] ${message}`) + world.attach(message, 'text/plain') + return 'skipped' +} + +export function skipMissingEnvResource( + world: DifyWorld, + envName: string, + description: string, +): 'skipped' | string { + const resource = readRequiredEnvResource(envName, description) + if (resource.ok) + return resource.value + + return skipBlockedPrecondition(world, resource.reason) +} + +export const requiredAgentBuilderPreseededResources = Object.values(agentBuilderPreseededResources) + +export function skipMissingAgentBuilderPreseed( + world: DifyWorld, + resourceName: string, + envName: string, +): 'skipped' | string { + return skipMissingEnvResource( + world, + envName, + `Preseeded Agent Builder resource "${resourceName}"`, + ) +} + +export const findConsoleResourceByName = async ({ + action, + path, + resourceName, +}: { + action: string + path: string + resourceName: string +}) => { + const ctx = await createApiContext() + try { + const response = await ctx.get(path) + await expectApiResponseOK(response, action) + const body = (await response.json()) as NamedResourceListResponse + + return body.data.find(item => item.name === resourceName) + } + finally { + await ctx.dispose() + } +} + +export const buildQuery = (params: Record) => new URLSearchParams(params).toString() + +export const matchesNameOrLabel = (value: string, name: string, label?: LocalizedLabel) => + value === name || value === label?.en_US || value === label?.zh_Hans + +export const isRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value) + +export const asRecord = (value: unknown): Record => (isRecord(value) ? value : {}) + +export const asArray = (value: unknown): unknown[] => (Array.isArray(value) ? value : []) + +export const asString = (value: unknown) => (typeof value === 'string' ? value : '') + +export const hasNamedOrKeyedEntry = (items: unknown[], expectedName: string) => + items.some((item) => { + const record = asRecord(item) + const values = [record.name, record.drive_key, record.reference, record.file_id, record.id].map( + asString, + ) + + return values.some(value => value === expectedName || value.endsWith(`/${expectedName}`)) + }) diff --git a/e2e/support/preflight/datasets.ts b/e2e/support/preflight/datasets.ts new file mode 100644 index 00000000000000..81c2bf5208eb10 --- /dev/null +++ b/e2e/support/preflight/datasets.ts @@ -0,0 +1,150 @@ +import type { DifyWorld } from '../../features/support/world' +import type { NamedResource, PreseededResource } from './common' +import { createApiContext, expectApiResponseOK } from '../api' +import { + buildQuery, + findConsoleResourceByName, + + skipBlockedPrecondition, +} from './common' + +type DatasetResource = NamedResource & { + document_count: number + total_available_documents: number +} + +type DocumentIndexingStatus + = | 'cleaning' + | 'completed' + | 'indexing' + | 'parsing' + | 'splitting' + | 'waiting' + +type DatasetIndexingStatusResponse = { + data: Array<{ + id: string + indexing_status?: string + }> +} + +const completedDocumentIndexingStatus: DocumentIndexingStatus = 'completed' +const activeDocumentIndexingStatuses = new Set([ + 'cleaning', + 'indexing', + 'parsing', + 'splitting', + 'waiting', +]) + +export const getPreseededDataset = async (resourceName: string) => { + const query = buildQuery({ keyword: resourceName, limit: '20', page: '1' }) + + return findConsoleResourceByName({ + action: `Check preseeded dataset ${resourceName}`, + path: `/console/api/datasets?${query}`, + resourceName, + }) +} + +const getDatasetIndexingStatuses = async (datasetId: string, resourceName: string) => { + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/datasets/${datasetId}/indexing-status`) + await expectApiResponseOK(response, `Check preseeded dataset indexing status ${resourceName}`) + const body = (await response.json()) as DatasetIndexingStatusResponse + + return body.data + } + finally { + await ctx.dispose() + } +} + +export const toDatasetResource = ( + resource: NamedResource, +): PreseededResource => ({ + id: resource.id, + kind: 'dataset', + name: resource.name, +}) + +export async function skipMissingPreseededDataset( + world: DifyWorld, + resourceName: string, +): Promise<'skipped' | PreseededResource> { + const resource = await getPreseededDataset(resourceName) + + if (!resource) + return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`) + + return toDatasetResource(resource) +} + +export async function skipMissingReadyPreseededDataset( + world: DifyWorld, + resourceName: string, +): Promise<'skipped' | PreseededResource> { + const resource = await getPreseededDataset(resourceName) + + if (!resource) + return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`) + + if (resource.document_count < 1) { + return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" has no documents.`) + } + + if (resource.total_available_documents !== resource.document_count) { + return skipBlockedPrecondition( + world, + `Preseeded dataset "${resourceName}" has ${resource.total_available_documents}/${resource.document_count} available documents.`, + ) + } + + const statuses = await getDatasetIndexingStatuses(resource.id, resourceName) + if (statuses.length < 1) { + return skipBlockedPrecondition( + world, + `Preseeded dataset "${resourceName}" has no document indexing status.`, + ) + } + + const incompleteStatus = statuses.find( + item => item.indexing_status !== completedDocumentIndexingStatus, + ) + if (incompleteStatus) { + return skipBlockedPrecondition( + world, + `Preseeded dataset "${resourceName}" includes document ${incompleteStatus.id} with indexing status "${incompleteStatus.indexing_status ?? 'missing'}".`, + ) + } + + return toDatasetResource(resource) +} + +export async function skipMissingIndexingPreseededDataset( + world: DifyWorld, + resourceName: string, +): Promise<'skipped' | PreseededResource> { + const resource = await getPreseededDataset(resourceName) + + if (!resource) + return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`) + + const statuses = await getDatasetIndexingStatuses(resource.id, resourceName) + const indexingStatus = statuses.find(item => + activeDocumentIndexingStatuses.has(item.indexing_status ?? ''), + ) + + if (!indexingStatus) { + const actualStatuses + = statuses.map(item => item.indexing_status ?? 'missing').join(', ') || 'none' + + return skipBlockedPrecondition( + world, + `Preseeded dataset "${resourceName}" is not indexing or queued; document statuses: ${actualStatuses}.`, + ) + } + + return toDatasetResource(resource) +} diff --git a/e2e/support/preflight/index.ts b/e2e/support/preflight/index.ts new file mode 100644 index 00000000000000..4c3ada4720af65 --- /dev/null +++ b/e2e/support/preflight/index.ts @@ -0,0 +1,6 @@ +export * from './access' +export * from './agents' +export * from './common' +export * from './datasets' +export * from './models' +export * from './tools' diff --git a/e2e/support/preflight/models.ts b/e2e/support/preflight/models.ts new file mode 100644 index 00000000000000..05e544c3ab20f8 --- /dev/null +++ b/e2e/support/preflight/models.ts @@ -0,0 +1,158 @@ +import type { DifyWorld } from '../../features/support/world' +import { agentBuilderPreseededResources } from '../agent-builder-resources' +import { createApiContext, expectApiResponseOK } from '../api' +import { skipBlockedPrecondition } from './common' + +const stableChatModelProviderEnv = 'E2E_STABLE_MODEL_PROVIDER' +const stableChatModelNameEnv = 'E2E_STABLE_MODEL_NAME' +const stableChatModelTypeEnv = 'E2E_STABLE_MODEL_TYPE' +const brokenChatModelProviderEnv = 'E2E_BROKEN_MODEL_PROVIDER' +const brokenChatModelNameEnv = 'E2E_BROKEN_MODEL_NAME' +const brokenChatModelTypeEnv = 'E2E_BROKEN_MODEL_TYPE' +const activeModelStatus = 'active' +const defaultStableChatModelType = 'llm' +const defaultBrokenChatModelName = agentBuilderPreseededResources.brokenModel + +type ModelTypeListResponse = { + data: Array<{ + provider: string + models: Array<{ + label?: { + en_US?: string + zh_Hans?: string + } + model: string + status?: string + }> + status?: string + }> +} + +type ModelPreflightConfig + = | { + ok: true + provider: string + resourceName: string + type: string + value: string + } + | { + ok: false + reason: string + } + +export function readAgentBuilderStableChatModelConfig(): ModelPreflightConfig { + const provider = process.env[stableChatModelProviderEnv]?.trim() + const name = process.env[stableChatModelNameEnv]?.trim() + const type = process.env[stableChatModelTypeEnv]?.trim() || defaultStableChatModelType + + const missing: string[] = [] + if (!provider) + missing.push(stableChatModelProviderEnv) + if (!name) + missing.push(stableChatModelNameEnv) + + if (!provider || !name) { + return { + ok: false, + reason: `${agentBuilderPreseededResources.stableChatModel} requires ${missing.join(', ')}.`, + } + } + + return { + ok: true, + provider, + resourceName: agentBuilderPreseededResources.stableChatModel, + type, + value: name, + } +} + +export function readAgentBuilderBrokenChatModelConfig(): ModelPreflightConfig { + const provider = process.env[brokenChatModelProviderEnv]?.trim() + const name = process.env[brokenChatModelNameEnv]?.trim() || defaultBrokenChatModelName + const type = process.env[brokenChatModelTypeEnv]?.trim() || defaultStableChatModelType + + if (!provider) { + return { + ok: false, + reason: `${agentBuilderPreseededResources.brokenModelProvider} requires ${brokenChatModelProviderEnv}.`, + } + } + + return { + ok: true, + provider, + resourceName: agentBuilderPreseededResources.brokenModelProvider, + type, + value: name, + } +} + +async function skipMissingAgentBuilderModel( + world: DifyWorld, + config: ModelPreflightConfig, + { + requireActive, + }: { + requireActive: boolean + }, +): Promise<'skipped' | NonNullable> { + if (!config.ok) + return skipBlockedPrecondition(world, config.reason) + + const ctx = await createApiContext() + try { + const response = await ctx.get( + `/console/api/workspaces/current/models/model-types/${config.type}`, + ) + await expectApiResponseOK(response, `Check ${config.resourceName}`) + const body = (await response.json()) as ModelTypeListResponse + const provider = body.data.find(item => item.provider === config.provider) + const model = provider?.models.find( + item => + item.model === config.value + || item.label?.en_US === config.value + || item.label?.zh_Hans === config.value, + ) + + if (!provider || !model) { + return skipBlockedPrecondition( + world, + `${config.resourceName} was not found as ${config.provider}/${config.value} (${config.type}).`, + ) + } + + if (requireActive && model.status !== activeModelStatus) { + return skipBlockedPrecondition( + world, + `${config.resourceName} is ${model.status ?? 'missing status'} instead of ${activeModelStatus}.`, + ) + } + + return { + name: model.model, + provider: provider.provider, + type: config.type, + } + } + finally { + await ctx.dispose() + } +} + +export async function skipMissingAgentBuilderStableChatModel( + world: DifyWorld, +): Promise<'skipped' | NonNullable> { + return skipMissingAgentBuilderModel(world, readAgentBuilderStableChatModelConfig(), { + requireActive: true, + }) +} + +export async function skipMissingAgentBuilderBrokenChatModel( + world: DifyWorld, +): Promise<'skipped' | NonNullable> { + return skipMissingAgentBuilderModel(world, readAgentBuilderBrokenChatModelConfig(), { + requireActive: false, + }) +} diff --git a/e2e/support/preflight/tools.ts b/e2e/support/preflight/tools.ts new file mode 100644 index 00000000000000..b188c9fe24e072 --- /dev/null +++ b/e2e/support/preflight/tools.ts @@ -0,0 +1,114 @@ +import type { DifyWorld } from '../../features/support/world' +import type { LocalizedLabel, PreseededResource } from './common' +import { createApiContext, expectApiResponseOK } from '../api' +import { + asRecord, + asString, + + matchesNameOrLabel, + + skipBlockedPrecondition, +} from './common' + +type BuiltinToolProvider = { + label?: LocalizedLabel + name: string + tools: Array<{ + label?: LocalizedLabel + name: string + }> +} + +export const splitToolDisplayName = (resourceName: string) => { + const [providerName, toolName] = resourceName.split('/').map(item => item.trim()) + + if (!providerName || !toolName) { + return { + ok: false as const, + reason: `Preseeded tool "${resourceName}" must use "Provider / Tool" format.`, + } + } + + return { + ok: true as const, + providerName, + toolName, + } +} + +export const findToolEntry = ( + items: unknown[], + { + providerDisplayName, + providerName, + toolDisplayName, + toolName, + }: { + providerDisplayName: string + providerName: string + toolDisplayName: string + toolName: string + }, +) => + items.find((item) => { + const record = asRecord(item) + const providerValues = [record.provider_id, record.provider, record.plugin_id, record.name].map( + asString, + ) + const toolValues = [record.tool_name, record.name].map(asString) + + return ( + providerValues.some(value => value === providerName || value === providerDisplayName) + && toolValues.some(value => value === toolName || value === toolDisplayName) + ) + }) + +export const hasToolEntry = ( + items: unknown[], + tool: { + providerDisplayName: string + providerName: string + toolDisplayName: string + toolName: string + }, +) => Boolean(findToolEntry(items, tool)) + +export const hasUnauthorizedToolCredentialState = (item: unknown) => { + const record = asRecord(item) + + return asString(record.credential_type) === 'unauthorized' +} + +export async function skipMissingPreseededTool( + world: DifyWorld, + resourceName: string, +): Promise<'skipped' | PreseededResource> { + const parsed = splitToolDisplayName(resourceName) + if (!parsed.ok) + return skipBlockedPrecondition(world, parsed.reason) + + const ctx = await createApiContext() + try { + const response = await ctx.get('/console/api/workspaces/current/tools/builtin') + await expectApiResponseOK(response, `Check preseeded tool ${resourceName}`) + const providers = (await response.json()) as BuiltinToolProvider[] + const provider = providers.find(item => + matchesNameOrLabel(parsed.providerName, item.name, item.label), + ) + const tool = provider?.tools.find(item => + matchesNameOrLabel(parsed.toolName, item.name, item.label), + ) + + if (!provider || !tool) + return skipBlockedPrecondition(world, `Preseeded tool "${resourceName}" was not found.`) + + return { + id: `${provider.name}/${tool.name}`, + kind: 'tool', + name: resourceName, + } + } + finally { + await ctx.dispose() + } +} From 3a5dff0b48ee8e698b7a57420e0aca89b2be888b Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 20:44:24 +0800 Subject: [PATCH 083/185] docs(e2e): scope agent v2 guidance --- e2e/AGENTS.md | 48 +--------- e2e/features/agent-v2/AGENTS.md | 151 ++++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 44 deletions(-) create mode 100644 e2e/features/agent-v2/AGENTS.md diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 1159f47abb90a1..b9a24c6004d0e6 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -296,37 +296,13 @@ Use `support/naming.ts` for generated test resource names. New app, Agent, datas Use `fixtures/test-materials/` for checked-in files that scenarios upload, preview, index, or retrieve. Keep these fixtures small and deterministic, and use `support/test-materials.ts` to resolve their absolute paths. -Use `support/preflight.ts` for scenarios that require optional external resources such as a stable model provider, plugin/tool credential, or knowledge base seed. Prefer an explicit `Given` step that returns a skipped result with a clear blocked-precondition reason over hidden setup in hooks. Keep `support/preflight.ts` as the public barrel and put resource checks in the matching module under `support/preflight/`: `models.ts`, `agents.ts`, `datasets.ts`, `tools.ts`, or `access.ts`. +Use `support/preflight.ts` for scenarios that require optional external resources such as a model provider, plugin/tool credential, knowledge base seed, or fixed app. Prefer an explicit `Given` step that returns a skipped result with a clear blocked-precondition reason over hidden setup in hooks. -Keep Agent Builder step definitions grouped by user capability, not by DOM component or Cucumber keyword. `configure.steps.ts` owns common configure navigation, refresh, and draft assertions; `build-draft.steps.ts` owns Build mode checkout, apply, discard, and Build draft isolation; `files.steps.ts` owns Files upload and file-list assertions; `advanced-settings.steps.ts` owns Env and advanced settings behavior; `agent-edit.steps.ts` owns saved Agent detail display assertions; `publish.steps.ts` owns publish state; `access-point.steps.ts` owns Access Point behavior; `preflight.steps.ts` should remain the explicit `Given` entrypoint for preflight resources. +Keep `support/preflight.ts` as the public barrel when feature-specific checks need implementation modules underneath it. The step wording should stay explicit in the feature area, while support code owns API checks, readiness polling, and blocked-precondition messages. -Use `the Agent Builder stable chat model is available` before scenarios that must run an Agent with a real model. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` define the single usable model for the E2E environment. The step defaults the type to `llm`, verifies the model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilder.preflight.stableModel`. Any Agent Builder scenario that needs a usable model should explicitly apply this stored model during its own setup instead of hard-coding provider or model names in feature files or hooks. +Use typed cleanup fields on `DifyWorld` for resource types created by scenarios, and use `DifyWorld.registerCleanup(...)` when a scenario creates any resource type that is not covered by typed cleanup fields. Cleanup callbacks run after typed cleanup queues, even when the scenario fails. -Use `the Agent Builder broken chat model is available` before model-recovery scenarios that intentionally start from an invalid model. The step requires `E2E_BROKEN_MODEL_PROVIDER`, defaults `E2E_BROKEN_MODEL_NAME` to `e2e-broken-model`, defaults `E2E_BROKEN_MODEL_TYPE` to `llm`, and only verifies that the model entry exists. The scenario must still assert the user-visible failure and recovery behavior. - -Use `the Agent Builder preseeded Agent "{name}" is available`, `the Agent Builder preseeded workflow "{name}" is available`, `the Agent Builder preseeded dataset "{name}" is available`, and `the Agent Builder preseeded tool "{provider} / {tool}" is available` when a scenario depends on a fixed environment resource. These steps verify the resource through Console APIs, store the result in `DifyWorld.agentBuilder.preflight.preseededResources`, and return `skipped` with a blocked-precondition attachment when the resource is missing. The tool step checks built-in tool availability only; credential-validity scenarios still need explicit credential-state setup or assertions. - -Use `the Agent Builder preseeded dataset "{name}" is indexed and ready` for knowledge retrieval scenarios that require a completed knowledge base. It verifies that the dataset exists, has documents, all listed documents are available, and every document indexing status is `completed`. Use `the Agent Builder preseeded dataset "{name}" is indexing` for failure-recovery scenarios that require an indexing or queued knowledge base; it verifies at least one document is in `waiting`, `parsing`, `cleaning`, `splitting`, or `indexing`. Fixed-content assertions such as `AGENT_KNOWLEDGE_PASS` belong in the dependent runtime scenario, where the user-visible Agent reply can prove retrieval actually hit the expected content. - -Use `the Agent Builder preseeded Agent "{agent}" includes drive skill "{skill}"` to verify that a fixed Agent has a drive-backed Skill attached. Use `the Agent Builder preseeded Agent "{agent}" has Backend service API access with an API key` to verify that a fixed Agent has Backend service API enabled and at least one key. The API key step does not validate a human-readable key name because the Console API key response does not expose one. - -Use `the Agent Builder preseeded Agent "{agent}" includes the core fixture configuration` for the fixed Full Config Agent prerequisite. It composes the stable model, Summary Skill, JSON Replace tool, and indexed knowledge-base preflights, then reads `/console/api/agent/{agent_id}/composer` to verify the Agent Soul contains the selected model, prompt success token, required file fixtures, JSON Replace tool entry, and knowledge dataset reference. Do not use this step for Agent node output variables; those live in workflow node-job `declared_outputs`, not the roster Agent App composer response. - -Use `the Agent Builder preseeded Agent "{agent}" includes the tool state fixture configuration` for the fixed Tool States Agent prerequisite. It composes the Summary Skill, JSON Replace tool, and Tavily Search tool preflights, then reads `/console/api/agent/{agent_id}/composer` to verify the Agent Soul includes JSON Replace, Tavily Search, and a Tavily credential reference. This proves the seed is configured to exercise tool status UI; keep actual invalid-credential errors in dependent user-visible configuration or runtime scenarios. - -Use `the Agent Builder preseeded Agent "{agent}" includes the dual retrieval fixture configuration` for the fixed Dual Retrieval Agent prerequisite. It composes the indexed knowledge-base preflight, then reads `/console/api/agent/{agent_id}/composer` to verify `agent_soul.knowledge.sets` includes both an Agent-decide generated query set and a custom user-query set using the fixed custom query. This proves the seed is configured to exercise the Knowledge Retrieval display; keep retrieval hit results, labels, expand/collapse behavior, and runtime assertions in dependent user-visible scenarios. - -Use `the Agent Builder preseeded Agent "{agent}" includes the file tree fixture files` for file-tree display prerequisites. It verifies the Agent drive contains every file from `agentBuilderFileTreeFixtureFiles` through `/console/api/agent/{agent_id}/drive/files?prefix=files/`. This proves the committed drive file set is ready; keep visual tree expansion, nesting, and preview assertions in dependent user-visible scenarios. - -Use `the Agent Builder preseeded Agent "{agent}" includes the current flat file fixture configuration` for the current Agent Edit Files section. Agent config files are still a flat `config_files` list and reject path separators, so this preflight verifies the fixture file basenames are present in the Agent Soul. Treat this as partial coverage for tree-display requirements until the product supports hierarchical config files in the visible Files section. - -Use `the Agent Builder preseeded Agent "{agent}" has published Web app access` to verify that a fixed Agent is published, Web app access is enabled, and the Agent detail response includes the site token and base URL needed to open the Web app. Keep Web app launch and runtime assertions in dependent user-visible scenarios. - -Use `the Agent Builder preseeded Agent "{agent}" is referenced by workflow "{workflow}"` to verify Workflow access prerequisites. It checks both fixed resources exist, then uses `/console/api/agent/{agent_id}/referencing-workflows`, the same Console API used by the Access Point Workflow references table, to verify the workflow references the Agent through at least one published Agent node. - -Run `pnpm -C e2e e2e -- --tags @agent-v2-preflight` against a seeded environment to verify Agent Builder preseeded resource readiness before running dependent scenarios. Keep each resource as a separate preflight scenario so a missing resource marks only its dependent precondition as blocked instead of hiding the rest of the readiness report. - -Use `DifyWorld.createdDatasetIds` for datasets created by a scenario, `DifyWorld.createdAgentDriveFiles` for Agent drive files committed during a scenario, and `DifyWorld.createdBuiltinToolCredentials` for built-in tool credentials created during a scenario. The shared `After` hook deletes Agent drive files first so file cleanup also works for scenarios that upload into a preseeded Agent, then deletes created Agents and Apps before deleting dependent datasets and tool credentials. Use `DifyWorld.registerCleanup(...)` when a scenario creates any other resource type that is not covered by the typed cleanup fields. Cleanup callbacks run after the typed cleanup queues, even when the scenario fails. +Feature-specific seed contracts, resource readiness rules, tags, and scenario ownership can be documented in one scoped `AGENTS.md` at the feature root when a module becomes large enough to need it. Do not add deeper `AGENTS.md` files unless the nested module becomes independently owned. ## Reusing existing steps @@ -336,19 +312,3 @@ Or browse the step definition files directly: - `features/step-definitions/common/` — auth guards and navigation assertions shared by all features - `features/step-definitions//` — domain-specific steps scoped to a single feature area - -## Agent v2 scenarios - -Agent v2 scenarios live under `features/agent-v2/` and use the `@agent-v2` capability tag. - -The E2E web environment enables Agent v2 through `NEXT_PUBLIC_ENABLE_AGENT_V2=true` in `scripts/common.ts`, because `/roster` routes are guarded by that feature flag. - -Use `support/agent.ts` for Agent v2 API fixtures. It owns roster-shaped Agent IDs, configure/access route helpers, composer draft sync, build-draft helpers, publish, API access toggles, Agent drive file cleanup, and Agent cleanup. Store created roster Agent IDs in `DifyWorld.createdAgentIds`; the shared `After` hook deletes them after each scenario. - -Keep Agent v2 step definitions under `features/step-definitions/agent-v2/`. Prefer API setup for prerequisite state, then use Playwright only for user-observable navigation, editing, and assertions. - -Use `a basic configured Agent v2 test agent has been created via API` when a scenario only needs a created Agent with a composer draft. Do not use that basic shell for runtime, model, tool, skill, knowledge, environment variable, moderation, or output-variable coverage until those resources have explicit seed helpers and readiness checks. - -Use `a runnable Agent v2 test agent has been created via API` after `the Agent Builder stable chat model is available` when a scenario needs a real model-backed Agent. The step writes the preflight model into the Agent Soul model config through `support/agent.ts` with deterministic E2E model settings; do not duplicate provider/model payload construction in individual steps. - -Use `the Agent v2 configuration should be saved automatically` after UI edits that rely on Configure autosave. It waits for the user-visible publish bar saved state; do not replace it with network-idle waits or internal store checks. diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md new file mode 100644 index 00000000000000..bd11d9be21fc33 --- /dev/null +++ b/e2e/features/agent-v2/AGENTS.md @@ -0,0 +1,151 @@ +# Agent V2 E2E + +This file scopes Agent v2 E2E conventions for `e2e/features/agent-v2/` and its step definitions under `e2e/features/step-definitions/agent-v2/`. Keep package-wide runner, lifecycle, hook, fixture, locator, assertion, and cleanup rules in `e2e/AGENTS.md`. + +Do not add deeper `AGENTS.md` files unless an Agent v2 submodule becomes independently owned. + +## Scope + +Agent v2 scenarios live under `features/agent-v2/` and use the `@agent-v2` capability tag. + +The E2E web environment enables Agent v2 through `NEXT_PUBLIC_ENABLE_AGENT_V2=true` in `scripts/common.ts`, because `/roster` routes are guarded by that feature flag. + +Preview/Test Run scenarios are not part of the current build-mode slice unless explicitly requested. Current Agent v2 coverage should prioritize Configure, Build draft, saved configuration display, publish state, Access Point, preflight, files, advanced settings, and other build-mode behavior. + +Use API setup for prerequisite state, then use Playwright only for user-observable navigation, editing, and assertions. Do not make assertions pass by mirroring the current implementation blindly; if a failure exposes a product ambiguity, resource gap, or test-quality problem, identify the owner before changing the test. + +## Tags + +- `@agent-v2` — required capability tag for all Agent v2 scenarios. +- `@core` — stable scenario expected to run in the regular Agent v2 suite when its explicit preconditions are met. +- `@infra` — infrastructure or readiness checks. +- `@build` — Build mode and Build draft behavior. +- `@files` — Files section upload, display, and fixture behavior. +- `@advanced-settings` — Env Editor, Content Moderation, and related Advanced Settings behavior. +- `@agent-edit` — saved Agent detail/configuration display surfaces. +- `@publish` — publish and publish-bar state. +- `@access-point` — Web app, Backend service API, and Workflow access surfaces. +- `@feature-gated` — product capability is optional. This tag alone does not skip execution; the scenario must include an explicit step that returns `skipped` with a blocked-precondition reason when the feature is unavailable. + +## Step Organization + +Keep Agent v2 step definitions grouped by user capability, not by DOM component or Cucumber keyword: + +- `configure.steps.ts` — common configure navigation, refresh, autosave, and normal draft assertions. +- `build-draft.steps.ts` — Build mode checkout, apply, discard, supported writeback, and Build draft isolation. +- `files.steps.ts` — Files upload, display, and fixture-list assertions. +- `advanced-settings.steps.ts` — Env Editor, Content Moderation, and Advanced Settings behavior. +- `agent-edit.steps.ts` — saved Agent detail display assertions. +- `publish.steps.ts` — publish and publish-bar assertions. +- `access-point.steps.ts` — Access Point behavior. +- `preflight.steps.ts` — explicit `Given` entrypoints for Agent Builder preflight resources. + +Cucumber step definitions are globally registered. Do not duplicate the same step text across files, even if one is written as `Given` and another as `Then`. + +## World State + +`DifyWorld` owns generic scenario state such as `page`, `context`, errors, downloads, cleanup queues, and created resource IDs. + +Agent v2 business state belongs under `world.agentBuilder`; do not keep adding Agent v2-specific fields to the top level of `DifyWorld`. + +Use the existing namespace shape: + +- `world.agentBuilder.preflight.stableModel` +- `world.agentBuilder.preflight.brokenModel` +- `world.agentBuilder.preflight.preseededResources` +- `world.agentBuilder.accessPoint.serviceApiBaseURL` +- `world.agentBuilder.accessPoint.generatedApiKey` +- `world.agentBuilder.accessPoint.apiReferencePage` +- `world.agentBuilder.accessPoint.webAppPage` +- `world.agentBuilder.accessPoint.webAppURL` +- `world.agentBuilder.accessPoint.workflowReferencePage` +- `world.agentBuilder.accessPoint.composerDraftSnapshot` +- `world.agentBuilder.workflow.outputVariables` + +Use `support/agent.ts` for Agent v2 API fixtures. It owns roster-shaped Agent IDs, configure/access route helpers, composer draft sync, build-draft helpers, publish, API access toggles, Agent drive file cleanup, and Agent cleanup. Store created roster Agent IDs in `DifyWorld.createdAgentIds`; the shared `After` hook deletes them after each scenario. + +Use `DifyWorld.createdAgentDriveFiles` for Agent drive files committed during a scenario and `DifyWorld.createdBuiltinToolCredentials` for built-in tool credentials created during a scenario. The shared `After` hook deletes Agent drive files first so cleanup also works for scenarios that upload into a preseeded Agent. + +## Setup Boundary + +Use `a basic configured Agent v2 test agent has been created via API` when a scenario only needs a created Agent with a composer draft. Do not use that basic shell for runtime, model, tool, skill, knowledge, environment variable, moderation, or output-variable coverage until those resources have explicit seed helpers and readiness checks. + +Use `a runnable Agent v2 test agent has been created via API` after `the Agent Builder stable chat model is available` when a scenario needs a real model-backed Agent. The step writes the preflight model into the Agent Soul model config through `support/agent.ts` with deterministic E2E model settings; do not duplicate provider/model payload construction in individual steps. + +Use `the Agent v2 configuration should be saved automatically` after UI edits that rely on Configure autosave. It waits for the user-visible publish bar saved state; do not replace it with network-idle waits or internal store checks. + +API setup is acceptable for creating scenario-owned Agents, enabling Backend service API, writing composer drafts, seeding Build drafts, and preparing fixed state. The scenario must still assert user-visible behavior or a real persisted product contract through the public Console API. Do not assert only that a setup API call succeeded. + +## Build Mode And Preview + +Build mode scope means: + +- Configure page behavior. +- Build draft checkout, pending state, apply, discard, and route isolation. +- Supported Build draft writeback for files, skills, and env. +- Saved configuration display after apply/discard/refresh. + +Preview/Test Run scope means: + +- Model-backed runtime responses. +- Duplicate run prevention. +- Runtime failure recovery. +- Tool/knowledge hit behavior proven through Agent replies. + +Keep Preview/Test Run scenarios out of the current build-mode slice unless the task explicitly reintroduces them. + +## Preflight Resources + +Keep `support/preflight.ts` as the public barrel. Agent Builder resource checks live under `support/preflight/`: + +- `models.ts` +- `agents.ts` +- `datasets.ts` +- `tools.ts` +- `access.ts` + +`preflight.steps.ts` should remain the explicit `Given` entrypoint. Do not move preflight into hidden hooks. + +Use `the Agent Builder stable chat model is available` before scenarios that must run an Agent with a real model. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` define the single usable model for the E2E environment. The step defaults the type to `llm`, verifies the model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilder.preflight.stableModel`. + +Use `the Agent Builder broken chat model is available` before model-recovery scenarios that intentionally start from an invalid model. The step requires `E2E_BROKEN_MODEL_PROVIDER`, defaults `E2E_BROKEN_MODEL_NAME` to `e2e-broken-model`, defaults `E2E_BROKEN_MODEL_TYPE` to `llm`, and only verifies that the model entry exists. The scenario must still assert the user-visible failure and recovery behavior. + +Use `the Agent Builder preseeded Agent "{name}" is available`, `the Agent Builder preseeded workflow "{name}" is available`, `the Agent Builder preseeded dataset "{name}" is available`, and `the Agent Builder preseeded tool "{provider} / {tool}" is available` when a scenario depends on a fixed environment resource. These steps verify the resource through Console APIs, store the result in `DifyWorld.agentBuilder.preflight.preseededResources`, and return `skipped` when the resource is missing. + +Use `the Agent Builder preseeded dataset "{name}" is indexed and ready` for knowledge retrieval scenarios that require a completed knowledge base. It verifies that the dataset exists, has documents, all listed documents are available, and every document indexing status is `completed`. + +Use `the Agent Builder preseeded dataset "{name}" is indexing` for failure-recovery scenarios that require an indexing or queued knowledge base. It verifies at least one document is in `waiting`, `parsing`, `cleaning`, `splitting`, or `indexing`. + +Use `the Agent Builder preseeded Agent "{agent}" includes drive skill "{skill}"` to verify that a fixed Agent has a drive-backed Skill attached. + +Use `the Agent Builder preseeded Agent "{agent}" has Backend service API access with an API key` to verify that a fixed Agent has Backend service API enabled and at least one key. The API key step does not validate a human-readable key name because the Console API key response does not expose one. + +Use `the Agent Builder preseeded Agent "{agent}" includes the core fixture configuration` for the fixed Full Config Agent prerequisite. It composes the stable model, Summary Skill, JSON Replace tool, and indexed knowledge-base preflights, then reads `/console/api/agent/{agent_id}/composer` to verify the Agent Soul contains the selected model, prompt success token, required file fixtures, JSON Replace tool entry, and knowledge dataset reference. Do not use this step for Agent node output variables; those live in workflow node-job `declared_outputs`, not the roster Agent App composer response. + +Use `the Agent Builder preseeded Agent "{agent}" includes the tool state fixture configuration` for the fixed Tool States Agent prerequisite. It composes the Summary Skill, JSON Replace tool, and Tavily Search tool preflights, then reads `/console/api/agent/{agent_id}/composer` to verify the Agent Soul includes JSON Replace, Tavily Search, and a Tavily credential reference. This proves the seed is configured to exercise tool status UI; keep actual invalid-credential errors in dependent user-visible configuration or runtime scenarios. + +Use `the Agent Builder preseeded Agent "{agent}" includes the dual retrieval fixture configuration` for the fixed Dual Retrieval Agent prerequisite. It composes the indexed knowledge-base preflight, then reads `/console/api/agent/{agent_id}/composer` to verify `agent_soul.knowledge.sets` includes both an Agent-decide generated query set and a custom user-query set using the fixed custom query. + +Use `the Agent Builder preseeded Agent "{agent}" includes the file tree fixture files` for file-tree display prerequisites. It verifies the Agent drive contains every file from `agentBuilderFileTreeFixtureFiles` through `/console/api/agent/{agent_id}/drive/files?prefix=files/`. + +Use `the Agent Builder preseeded Agent "{agent}" includes the current flat file fixture configuration` for the current Agent Edit Files section. Agent config files are still a flat `config_files` list and reject path separators, so this preflight verifies the fixture file basenames are present in the Agent Soul. Treat this as partial coverage for tree-display requirements until the product supports hierarchical config files in the visible Files section. + +Use `the Agent Builder preseeded Agent "{agent}" has published Web app access` to verify that a fixed Agent is published, Web app access is enabled, and the Agent detail response includes the site token and base URL needed to open the Web app. + +Use `the Agent Builder preseeded Agent "{agent}" is referenced by workflow "{workflow}"` to verify Workflow access prerequisites. It checks both fixed resources exist, then uses `/console/api/agent/{agent_id}/referencing-workflows`, the same Console API used by the Access Point Workflow references table, to verify the workflow references the Agent through at least one published Agent node. + +Run `pnpm -C e2e e2e -- --tags @agent-v2-preflight` against a seeded environment to verify Agent Builder preseeded resource readiness before running dependent scenarios. Keep each resource as a separate preflight scenario so a missing resource marks only its dependent precondition as blocked instead of hiding the rest of the readiness report. + +## Blocked And Partial Policy + +Use explicit skipped steps for missing resources, disabled feature flags, and product capabilities that are not currently implemented. `@feature-gated` is only a label; it is not execution semantics. + +Blocked messages should be specific enough to route ownership: + +```text +Blocked precondition: missing . Owner: seed/product. Remediation: . +``` + +Use partial coverage only when current product behavior is intentionally narrower than the written requirement and the test still asserts a real user-visible behavior. Example: Files are currently flat in Agent config files, so the flat Files list can be asserted while tree display remains blocked until product support exists. + +Do not mark a scenario as complete if it only proves setup state and does not assert the user-visible behavior or persisted product contract required by the case. From 2739772f0a443c00f4d6ffc49c29ca3033c29846 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 20:48:59 +0800 Subject: [PATCH 084/185] fix(ui): name content moderation switches --- .../new-feature-panel/moderation/moderation-content.tsx | 8 ++++++-- .../orchestrate/advanced/content-moderation.tsx | 7 ++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/web/app/components/base/features/new-feature-panel/moderation/moderation-content.tsx b/web/app/components/base/features/new-feature-panel/moderation/moderation-content.tsx index b008490ae6ff9a..8d9f870b226deb 100644 --- a/web/app/components/base/features/new-feature-panel/moderation/moderation-content.tsx +++ b/web/app/components/base/features/new-feature-panel/moderation/moderation-content.tsx @@ -40,7 +40,10 @@ const ModerationContent: FC = ({ } return ( -
+
{title}
@@ -52,6 +55,7 @@ const ModerationContent: FC = ({ handleConfigChange('enabled', v)} + aria-label={title} />
@@ -85,7 +89,7 @@ const ModerationContent: FC = ({
) } -
+ ) } diff --git a/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/content-moderation.tsx b/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/content-moderation.tsx index 57c1ec9d5439f2..4eafa263f7d884 100644 --- a/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/content-moderation.tsx +++ b/web/features/agent-v2/agent-detail/configure/components/orchestrate/advanced/content-moderation.tsx @@ -129,7 +129,12 @@ function AgentContentModerationSettingsContent() { )}
- +
) : undefined} From 0670aee45b5f6780bcc4f0cb2b29bba76c36fb18 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 20:51:52 +0800 Subject: [PATCH 085/185] test(e2e): gate content moderation settings coverage --- .../agent-v2/advanced-settings.feature | 14 +++ .../agent-v2/advanced-settings.steps.ts | 115 +++++++++++++++++- e2e/support/agent-builder-resources.ts | 1 + 3 files changed, 129 insertions(+), 1 deletion(-) diff --git a/e2e/features/agent-v2/advanced-settings.feature b/e2e/features/agent-v2/advanced-settings.feature index 2e7a1edbd2546a..d485e82e674fa5 100644 --- a/e2e/features/agent-v2/advanced-settings.feature +++ b/e2e/features/agent-v2/advanced-settings.feature @@ -53,3 +53,17 @@ Feature: Agent v2 advanced settings And the Agent v2 configuration should be saved automatically When I refresh the current page Then I should see the Agent v2 environment variables from the invalid import in Advanced Settings + + @content-moderation @feature-gated + Scenario: Content Moderation keyword preset replies are saved and restored + Given I am signed in as the default E2E admin + And a basic configured Agent v2 test agent has been created via API + When I open the Agent v2 configure page + And I expand Agent v2 Advanced Settings + Then Agent v2 Content Moderation Settings should be available + When I configure Agent v2 Content Moderation keyword preset replies + Then Agent v2 Content Moderation keyword preset replies should be saved in the Agent v2 draft + And the Agent v2 configuration should be saved automatically + When I refresh the current page + And I expand Agent v2 Advanced Settings + Then I should see the Agent v2 Content Moderation keyword preset replies in Advanced Settings diff --git a/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts b/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts index 80041582424750..ea321973e0f008 100644 --- a/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts +++ b/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts @@ -1,3 +1,4 @@ +import type { Locator } from '@playwright/test' import type { DifyWorld } from '../../support/world' import { Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' @@ -15,6 +16,17 @@ import { openAgentAdvancedSettings, } from './configure-helpers' +const getModerationSettingsDialog = (world: DifyWorld) => + world.getPage().getByRole('dialog').filter({ hasText: 'Content moderation settings' }) + +const getContentModerationRegion = (world: DifyWorld) => + world.getPage().getByRole('region', { name: 'Content moderation' }) + +const ensureSwitchChecked = async (switchLocator: Locator) => { + if (await switchLocator.getAttribute('aria-checked') !== 'true') + await switchLocator.click() +} + When( 'I add the plain Agent v2 environment variable from Advanced Settings', async function (this: DifyWorld) { @@ -100,6 +112,45 @@ When('I expand Agent v2 Advanced Settings', async function (this: DifyWorld) { await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() }) +When( + 'I configure Agent v2 Content Moderation keyword preset replies', + async function (this: DifyWorld) { + const page = this.getPage() + const contentModeration = getContentModerationRegion(this) + const enabledSwitch = contentModeration.getByRole('switch', { name: 'Content moderation' }) + + if (await enabledSwitch.getAttribute('aria-checked') === 'true') + await contentModeration.getByRole('button', { name: 'Settings' }).click() + else + await enabledSwitch.click() + + const dialog = getModerationSettingsDialog(this) + await expect(dialog).toBeVisible() + await dialog.getByRole('button', { name: 'Keywords' }).click() + await dialog + .getByRole('textbox', { name: 'Keywords' }) + .fill(agentBuilderFixedInputs.moderationKeyword) + + const inputModeration = dialog.getByRole('region', { name: 'Moderate INPUT Content' }) + const outputModeration = dialog.getByRole('region', { name: 'Moderate OUTPUT Content' }) + await ensureSwitchChecked(inputModeration.getByRole('switch', { name: 'Moderate INPUT Content' })) + + await dialog.getByRole('button', { name: 'Save' }).click() + await expect(page.getByText('Preset replies cannot be empty')).toBeVisible() + await expect(dialog).toBeVisible() + + await inputModeration + .getByRole('textbox', { name: 'Preset replies' }) + .fill(agentBuilderFixedInputs.inputModerationReply) + await ensureSwitchChecked(outputModeration.getByRole('switch', { name: 'Moderate OUTPUT Content' })) + await outputModeration + .getByRole('textbox', { name: 'Preset replies' }) + .fill(agentBuilderFixedInputs.outputModerationReply) + await dialog.getByRole('button', { name: 'Save' }).click() + await expect(dialog).not.toBeVisible() + }, +) + Then( 'Agent v2 Advanced Settings should describe supported entries while collapsed', async function (this: DifyWorld) { @@ -271,11 +322,73 @@ Then('Agent v2 Content Moderation Settings should be available', async function catch { return skipBlockedPrecondition( this, - 'Feature not enabled: Agent v2 Content Moderation Settings is not available in this build.', + 'Agent v2 Content Moderation Settings is not available in this build. Owner: product. Remediation: enable ENABLE_AGENT_CONTENT_MODERATION or keep this scenario feature-gated.', ) } }) +Then( + 'Agent v2 Content Moderation keyword preset replies should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect + .poll(async () => { + const draft = await getAgentComposerDraft(agentId) + const appFeatures = draft.agent_soul?.app_features as Record | undefined + const moderation = appFeatures?.sensitive_word_avoidance as Record | undefined + const config = moderation?.config as Record | undefined + const inputsConfig = config?.inputs_config as Record | undefined + const outputsConfig = config?.outputs_config as Record | undefined + + return { + enabled: moderation?.enabled, + inputEnabled: inputsConfig?.enabled, + inputPreset: inputsConfig?.preset_response, + keywords: config?.keywords, + outputEnabled: outputsConfig?.enabled, + outputPreset: outputsConfig?.preset_response, + type: moderation?.type, + } + }, { + timeout: 30_000, + }) + .toEqual({ + enabled: true, + inputEnabled: true, + inputPreset: agentBuilderFixedInputs.inputModerationReply, + keywords: agentBuilderFixedInputs.moderationKeyword, + outputEnabled: true, + outputPreset: agentBuilderFixedInputs.outputModerationReply, + type: 'keywords', + }) + }, +) + +Then( + 'I should see the Agent v2 Content Moderation keyword preset replies in Advanced Settings', + async function (this: DifyWorld) { + const contentModeration = getContentModerationRegion(this) + + await expect(contentModeration).toContainText('Keywords') + await expect(contentModeration).toContainText('INPUT & OUTPUT') + await contentModeration.getByRole('button', { name: 'Settings' }).click() + + const dialog = getModerationSettingsDialog(this) + await expect(dialog).toBeVisible() + await expect(dialog.getByRole('textbox', { name: 'Keywords' })) + .toHaveValue(agentBuilderFixedInputs.moderationKeyword) + await expect(dialog.getByRole('region', { name: 'Moderate INPUT Content' }) + .getByRole('textbox', { name: 'Preset replies' })) + .toHaveValue(agentBuilderFixedInputs.inputModerationReply) + await expect(dialog.getByRole('region', { name: 'Moderate OUTPUT Content' }) + .getByRole('textbox', { name: 'Preset replies' })) + .toHaveValue(agentBuilderFixedInputs.outputModerationReply) + await dialog.getByRole('button', { name: 'Cancel' }).click() + await expect(dialog).not.toBeVisible() + }, +) + Then( 'I should see the Agent v2 environment variables from the valid import in Advanced Settings', async function (this: DifyWorld) { diff --git a/e2e/support/agent-builder-resources.ts b/e2e/support/agent-builder-resources.ts index 3c87fdc094fa11..34097059d22a70 100644 --- a/e2e/support/agent-builder-resources.ts +++ b/e2e/support/agent-builder-resources.ts @@ -30,6 +30,7 @@ export const agentBuilderFixedInputs = { envModeValue: 'plain', envAfterInvalidImportKey: 'E2E_AGENT_AFTER_INVALID', envAfterInvalidImportValue: 'still-valid', + moderationKeyword: 'E2E_BLOCKED_KEYWORD', inputModerationReply: 'E2E_INPUT_BLOCKED_REPLY', outputModerationReply: 'E2E_OUTPUT_BLOCKED_REPLY', previewSuccessQuery: '请回复测试成功', From 80171d05c543542cd4c9804b0445e80872a7dcfa Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 21:04:00 +0800 Subject: [PATCH 086/185] test(e2e): cover agent v2 publish state --- e2e/features/agent-v2/publish.feature | 25 +++++++++++ .../agent-v2/configure.steps.ts | 8 ++++ .../agent-v2/publish.steps.ts | 45 ++++++++++++++++++- e2e/support/agent.ts | 21 +++++++++ 4 files changed, 98 insertions(+), 1 deletion(-) diff --git a/e2e/features/agent-v2/publish.feature b/e2e/features/agent-v2/publish.feature index c3a8fc3f7b794a..7cd901de4c5101 100644 --- a/e2e/features/agent-v2/publish.feature +++ b/e2e/features/agent-v2/publish.feature @@ -7,3 +7,28 @@ Feature: Agent v2 publish When I open the Agent v2 configure page And I publish the Agent v2 draft Then the Agent v2 draft should be published and up to date + + Scenario: Publish action follows unpublished changes + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API + When I open the Agent v2 configure page + Then the Agent v2 publish action should be available for unpublished changes + When I publish the Agent v2 draft + Then the Agent v2 draft should be published and up to date + And the Agent v2 publish action should be unavailable while up to date + When I fill the Agent v2 prompt editor with the updated E2E prompt + Then the Agent v2 configuration should be saved automatically + And the Agent v2 publish action should be available for unpublished changes + + Scenario: Published Agent v2 version remains isolated from draft edits + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API + When I open the Agent v2 configure page + And I publish the Agent v2 draft + Then the Agent v2 draft should be published and up to date + When I fill the Agent v2 prompt editor with the updated E2E prompt + Then the Agent v2 configuration should be saved automatically + And the normal Agent v2 draft should use the updated E2E prompt + And the active published Agent v2 version should still use the normal E2E prompt diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index b89e7461879596..64d56a2edd03a6 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -124,6 +124,14 @@ When('I fill the Agent v2 prompt editor with the normal E2E prompt', async funct await promptSection.getByRole('textbox', { name: 'Prompt' }).fill(normalAgentPrompt) }) +When('I fill the Agent v2 prompt editor with the updated E2E prompt', async function (this: DifyWorld) { + const page = this.getPage() + const promptSection = page.getByRole('region', { name: 'Prompt' }) + + await expect(promptSection).toBeVisible({ timeout: 30_000 }) + await promptSection.getByRole('textbox', { name: 'Prompt' }).fill(updatedAgentPrompt) +}) + Then('I should be on the Agent v2 configure page', async function (this: DifyWorld) { const agentId = getCurrentAgentId(this) diff --git a/e2e/features/step-definitions/agent-v2/publish.steps.ts b/e2e/features/step-definitions/agent-v2/publish.steps.ts index 13699f6578ab04..2e0b36ae6fa9c5 100644 --- a/e2e/features/step-definitions/agent-v2/publish.steps.ts +++ b/e2e/features/step-definitions/agent-v2/publish.steps.ts @@ -1,7 +1,7 @@ import type { DifyWorld } from '../../support/world' import { Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' -import { getTestAgent } from '../../../support/agent' +import { getAgentVersionDetail, getTestAgent, normalAgentPrompt } from '../../../support/agent' import { waitForAgentConfigureAutosaved } from '../../../support/agent-configure' import { getCurrentAgentId } from './configure-helpers' @@ -22,6 +22,49 @@ Then('the Agent v2 draft should be published and up to date', async function (th const agentId = getCurrentAgentId(this) await expect(page.getByRole('button', { name: 'Published' })).toBeVisible({ timeout: 30_000 }) + await expect(page.getByRole('status', { name: /^Up to date\./ })).toBeVisible() await expect(page.getByText('Up to date')).toBeVisible() await expect.poll(async () => (await getTestAgent(agentId)).active_config_is_published).toBe(true) }) + +Then( + 'the Agent v2 publish action should be available for unpublished changes', + async function (this: DifyWorld) { + const page = this.getPage() + const agentId = getCurrentAgentId(this) + + await expect(page.getByRole('status', { name: /^(?:Draft|Unpublished changes)\./ })) + .toBeVisible({ timeout: 30_000 }) + await expect(page.getByRole('button', { name: /^Publish(?: update)?$/ })) + .toBeEnabled() + await expect.poll(async () => (await getTestAgent(agentId)).active_config_is_published) + .toBe(false) + }, +) + +Then('the Agent v2 publish action should be unavailable while up to date', async function (this: DifyWorld) { + const page = this.getPage() + + await expect(page.getByRole('status', { name: /^Up to date\./ })).toBeVisible({ timeout: 30_000 }) + await expect(page.getByRole('button', { name: 'Published' })).toBeDisabled() +}) + +Then( + 'the active published Agent v2 version should still use the normal E2E prompt', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect.poll(async () => (await getTestAgent(agentId)).active_config_is_published, { + timeout: 30_000, + }).toBe(false) + + const agent = await getTestAgent(agentId) + const activeSnapshotId = agent?.active_config_snapshot_id + if (!activeSnapshotId) + throw new Error(`Agent v2 ${agentId} does not have an active published snapshot.`) + + const version = await getAgentVersionDetail(agentId, activeSnapshotId) + + expect(version.config_snapshot.prompt).toEqual({ system_prompt: normalAgentPrompt }) + }, +) diff --git a/e2e/support/agent.ts b/e2e/support/agent.ts index 11b57754c08f10..60d8f805f09902 100644 --- a/e2e/support/agent.ts +++ b/e2e/support/agent.ts @@ -6,6 +6,7 @@ import { assertE2EResourceName, createE2EResourceName } from './naming' export type AgentSeed = { active_config_is_published?: boolean + active_config_snapshot_id?: string | null app_id?: string backing_app_id?: string description?: string @@ -80,6 +81,11 @@ export type AgentApiKey = { token?: string } +export type AgentConfigSnapshotDetail = { + config_snapshot: AgentSoulConfig + id: string +} + export type AgentReferencingWorkflow = { app_id: string app_name: string @@ -353,6 +359,21 @@ export async function getTestAgent(agentId: string): Promise { } } +export async function getAgentVersionDetail( + agentId: string, + versionId: string, +): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agentId}/versions/${versionId}`) + await expectApiResponseOK(response, `Get Agent v2 version ${versionId} for ${agentId}`) + return (await response.json()) as AgentConfigSnapshotDetail + } + finally { + await ctx.dispose() + } +} + export async function deleteTestAgent(agentId: string): Promise { const ctx = await createApiContext() try { From 2a73e3ad2f75503c51606ebf6aa61779398e1e68 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 21:09:58 +0800 Subject: [PATCH 087/185] refactor(e2e): scope agent v2 support --- e2e/AGENTS.md | 4 +- e2e/features/agent-v2/AGENTS.md | 6 +-- .../support/agent-builder-resources.ts | 0 e2e/{ => features/agent-v2}/support/agent.ts | 4 +- .../agent-v2}/support/preflight/access.ts | 4 +- .../agent-v2}/support/preflight/agents.ts | 4 +- .../agent-v2}/support/preflight/common.ts | 4 +- .../agent-v2}/support/preflight/datasets.ts | 4 +- .../agent-v2}/support/preflight/models.ts | 4 +- .../agent-v2}/support/preflight/tools.ts | 4 +- .../agent-v2/support/test-materials.ts | 52 +++++++++++++++++++ .../agent-v2/access-point.steps.ts | 4 +- .../agent-v2/advanced-settings.steps.ts | 8 +-- .../agent-v2/agent-edit.steps.ts | 4 +- .../agent-v2/build-draft.steps.ts | 8 +-- .../agent-v2/configure-helpers.ts | 6 +-- .../agent-v2/configure.steps.ts | 4 +- .../step-definitions/agent-v2/files.steps.ts | 2 +- .../agent-v2/output-variables.steps.ts | 2 +- .../agent-v2/preflight.steps.ts | 24 +++++---- .../agent-v2/publish.steps.ts | 2 +- .../agent-v2/workflow-node.steps.ts | 10 ++-- e2e/features/support/hooks.ts | 2 +- e2e/support/preflight.ts | 1 - e2e/support/preflight/index.ts | 6 --- e2e/support/test-materials.ts | 50 ------------------ 26 files changed, 112 insertions(+), 111 deletions(-) rename e2e/{ => features/agent-v2}/support/agent-builder-resources.ts (100%) rename e2e/{ => features/agent-v2}/support/agent.ts (99%) rename e2e/{ => features/agent-v2}/support/preflight/access.ts (97%) rename e2e/{ => features/agent-v2}/support/preflight/agents.ts (99%) rename e2e/{ => features/agent-v2}/support/preflight/common.ts (96%) rename e2e/{ => features/agent-v2}/support/preflight/datasets.ts (96%) rename e2e/{ => features/agent-v2}/support/preflight/models.ts (97%) rename e2e/{ => features/agent-v2}/support/preflight/tools.ts (95%) create mode 100644 e2e/features/agent-v2/support/test-materials.ts delete mode 100644 e2e/support/preflight.ts delete mode 100644 e2e/support/preflight/index.ts diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index b9a24c6004d0e6..66f90a1ad668f6 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -296,9 +296,9 @@ Use `support/naming.ts` for generated test resource names. New app, Agent, datas Use `fixtures/test-materials/` for checked-in files that scenarios upload, preview, index, or retrieve. Keep these fixtures small and deterministic, and use `support/test-materials.ts` to resolve their absolute paths. -Use `support/preflight.ts` for scenarios that require optional external resources such as a model provider, plugin/tool credential, knowledge base seed, or fixed app. Prefer an explicit `Given` step that returns a skipped result with a clear blocked-precondition reason over hidden setup in hooks. +Use scoped feature support for scenarios that require optional external resources such as a model provider, plugin/tool credential, knowledge base seed, or fixed app. Prefer an explicit `Given` step that returns a skipped result with a clear blocked-precondition reason over hidden setup in hooks. -Keep `support/preflight.ts` as the public barrel when feature-specific checks need implementation modules underneath it. The step wording should stay explicit in the feature area, while support code owns API checks, readiness polling, and blocked-precondition messages. +Keep package-level support limited to broadly reusable primitives such as API clients, naming, fixture path resolution, and cleanup helpers. Feature-specific seed contracts and preflight checks belong under the owning feature's support folder. Use typed cleanup fields on `DifyWorld` for resource types created by scenarios, and use `DifyWorld.registerCleanup(...)` when a scenario creates any resource type that is not covered by typed cleanup fields. Cleanup callbacks run after typed cleanup queues, even when the scenario fails. diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index bd11d9be21fc33..dee75f9e5d38fb 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -62,7 +62,7 @@ Use the existing namespace shape: - `world.agentBuilder.accessPoint.composerDraftSnapshot` - `world.agentBuilder.workflow.outputVariables` -Use `support/agent.ts` for Agent v2 API fixtures. It owns roster-shaped Agent IDs, configure/access route helpers, composer draft sync, build-draft helpers, publish, API access toggles, Agent drive file cleanup, and Agent cleanup. Store created roster Agent IDs in `DifyWorld.createdAgentIds`; the shared `After` hook deletes them after each scenario. +Use `features/agent-v2/support/agent.ts` for Agent v2 API fixtures. It owns roster-shaped Agent IDs, configure/access route helpers, composer draft sync, build-draft helpers, publish, API access toggles, Agent drive file cleanup, and Agent cleanup. Store created roster Agent IDs in `DifyWorld.createdAgentIds`; the shared `After` hook deletes them after each scenario. Use `DifyWorld.createdAgentDriveFiles` for Agent drive files committed during a scenario and `DifyWorld.createdBuiltinToolCredentials` for built-in tool credentials created during a scenario. The shared `After` hook deletes Agent drive files first so cleanup also works for scenarios that upload into a preseeded Agent. @@ -70,7 +70,7 @@ Use `DifyWorld.createdAgentDriveFiles` for Agent drive files committed during a Use `a basic configured Agent v2 test agent has been created via API` when a scenario only needs a created Agent with a composer draft. Do not use that basic shell for runtime, model, tool, skill, knowledge, environment variable, moderation, or output-variable coverage until those resources have explicit seed helpers and readiness checks. -Use `a runnable Agent v2 test agent has been created via API` after `the Agent Builder stable chat model is available` when a scenario needs a real model-backed Agent. The step writes the preflight model into the Agent Soul model config through `support/agent.ts` with deterministic E2E model settings; do not duplicate provider/model payload construction in individual steps. +Use `a runnable Agent v2 test agent has been created via API` after `the Agent Builder stable chat model is available` when a scenario needs a real model-backed Agent. The step writes the preflight model into the Agent Soul model config through `features/agent-v2/support/agent.ts` with deterministic E2E model settings; do not duplicate provider/model payload construction in individual steps. Use `the Agent v2 configuration should be saved automatically` after UI edits that rely on Configure autosave. It waits for the user-visible publish bar saved state; do not replace it with network-idle waits or internal store checks. @@ -96,7 +96,7 @@ Keep Preview/Test Run scenarios out of the current build-mode slice unless the t ## Preflight Resources -Keep `support/preflight.ts` as the public barrel. Agent Builder resource checks live under `support/preflight/`: +Agent Builder resource checks live under `features/agent-v2/support/preflight/`. Import from the specific module that owns the resource contract; do not add a preflight barrel file: - `models.ts` - `agents.ts` diff --git a/e2e/support/agent-builder-resources.ts b/e2e/features/agent-v2/support/agent-builder-resources.ts similarity index 100% rename from e2e/support/agent-builder-resources.ts rename to e2e/features/agent-v2/support/agent-builder-resources.ts diff --git a/e2e/support/agent.ts b/e2e/features/agent-v2/support/agent.ts similarity index 99% rename from e2e/support/agent.ts rename to e2e/features/agent-v2/support/agent.ts index 60d8f805f09902..a3b22a57f77755 100644 --- a/e2e/support/agent.ts +++ b/e2e/features/agent-v2/support/agent.ts @@ -1,8 +1,8 @@ import { Buffer } from 'node:buffer' import { readFile } from 'node:fs/promises' import path from 'node:path' -import { createApiContext, expectApiResponseOK, setAppSiteEnabled } from './api' -import { assertE2EResourceName, createE2EResourceName } from './naming' +import { createApiContext, expectApiResponseOK, setAppSiteEnabled } from '../../../support/api' +import { assertE2EResourceName, createE2EResourceName } from '../../../support/naming' export type AgentSeed = { active_config_is_published?: boolean diff --git a/e2e/support/preflight/access.ts b/e2e/features/agent-v2/support/preflight/access.ts similarity index 97% rename from e2e/support/preflight/access.ts rename to e2e/features/agent-v2/support/preflight/access.ts index e04be75ec799ae..f721fc716b3728 100644 --- a/e2e/support/preflight/access.ts +++ b/e2e/features/agent-v2/support/preflight/access.ts @@ -1,6 +1,6 @@ -import type { DifyWorld } from '../../features/support/world' +import type { DifyWorld } from '../../../support/world' import type { PreseededResource } from './common' -import { createApiContext, expectApiResponseOK } from '../api' +import { createApiContext, expectApiResponseOK } from '../../../../support/api' import { skipMissingPreseededAgent, skipMissingPreseededWorkflow } from './agents' import { skipBlockedPrecondition } from './common' diff --git a/e2e/support/preflight/agents.ts b/e2e/features/agent-v2/support/preflight/agents.ts similarity index 99% rename from e2e/support/preflight/agents.ts rename to e2e/features/agent-v2/support/preflight/agents.ts index 90bdde7a12e62d..2c06376a1c6b04 100644 --- a/e2e/support/preflight/agents.ts +++ b/e2e/features/agent-v2/support/preflight/agents.ts @@ -1,11 +1,11 @@ -import type { DifyWorld } from '../../features/support/world' +import type { DifyWorld } from '../../../support/world' import type { AgentComposerResponse, PreseededResource } from './common' +import { createApiContext, expectApiResponseOK } from '../../../../support/api' import { agentBuilderExpectedTokens, agentBuilderFixedInputs, agentBuilderPreseededResources, } from '../agent-builder-resources' -import { createApiContext, expectApiResponseOK } from '../api' import { agentBuilderFileTreeFixtureFileNames, agentBuilderFileTreeFixtureFiles, diff --git a/e2e/support/preflight/common.ts b/e2e/features/agent-v2/support/preflight/common.ts similarity index 96% rename from e2e/support/preflight/common.ts rename to e2e/features/agent-v2/support/preflight/common.ts index 90b491338b7f55..bb6150ff7a56a5 100644 --- a/e2e/support/preflight/common.ts +++ b/e2e/features/agent-v2/support/preflight/common.ts @@ -1,6 +1,6 @@ -import type { DifyWorld } from '../../features/support/world' +import type { DifyWorld } from '../../../support/world' +import { createApiContext, expectApiResponseOK } from '../../../../support/api' import { agentBuilderPreseededResources } from '../agent-builder-resources' -import { createApiContext, expectApiResponseOK } from '../api' export type PreseededResource = NonNullable< DifyWorld['agentBuilder']['preflight']['preseededResources'][string] diff --git a/e2e/support/preflight/datasets.ts b/e2e/features/agent-v2/support/preflight/datasets.ts similarity index 96% rename from e2e/support/preflight/datasets.ts rename to e2e/features/agent-v2/support/preflight/datasets.ts index 81c2bf5208eb10..dc992ecdb5bbb4 100644 --- a/e2e/support/preflight/datasets.ts +++ b/e2e/features/agent-v2/support/preflight/datasets.ts @@ -1,6 +1,6 @@ -import type { DifyWorld } from '../../features/support/world' +import type { DifyWorld } from '../../../support/world' import type { NamedResource, PreseededResource } from './common' -import { createApiContext, expectApiResponseOK } from '../api' +import { createApiContext, expectApiResponseOK } from '../../../../support/api' import { buildQuery, findConsoleResourceByName, diff --git a/e2e/support/preflight/models.ts b/e2e/features/agent-v2/support/preflight/models.ts similarity index 97% rename from e2e/support/preflight/models.ts rename to e2e/features/agent-v2/support/preflight/models.ts index 05e544c3ab20f8..0176947543f807 100644 --- a/e2e/support/preflight/models.ts +++ b/e2e/features/agent-v2/support/preflight/models.ts @@ -1,6 +1,6 @@ -import type { DifyWorld } from '../../features/support/world' +import type { DifyWorld } from '../../../support/world' +import { createApiContext, expectApiResponseOK } from '../../../../support/api' import { agentBuilderPreseededResources } from '../agent-builder-resources' -import { createApiContext, expectApiResponseOK } from '../api' import { skipBlockedPrecondition } from './common' const stableChatModelProviderEnv = 'E2E_STABLE_MODEL_PROVIDER' diff --git a/e2e/support/preflight/tools.ts b/e2e/features/agent-v2/support/preflight/tools.ts similarity index 95% rename from e2e/support/preflight/tools.ts rename to e2e/features/agent-v2/support/preflight/tools.ts index b188c9fe24e072..9f20ee0e8e5a63 100644 --- a/e2e/support/preflight/tools.ts +++ b/e2e/features/agent-v2/support/preflight/tools.ts @@ -1,6 +1,6 @@ -import type { DifyWorld } from '../../features/support/world' +import type { DifyWorld } from '../../../support/world' import type { LocalizedLabel, PreseededResource } from './common' -import { createApiContext, expectApiResponseOK } from '../api' +import { createApiContext, expectApiResponseOK } from '../../../../support/api' import { asRecord, asString, diff --git a/e2e/features/agent-v2/support/test-materials.ts b/e2e/features/agent-v2/support/test-materials.ts new file mode 100644 index 00000000000000..eb6497918b05ec --- /dev/null +++ b/e2e/features/agent-v2/support/test-materials.ts @@ -0,0 +1,52 @@ +import path from 'node:path' +import { getGeneratedTextMaterialPath, getTestMaterialPath } from '../../../support/test-materials' + +export const agentBuilderTestMaterials = { + smallFile: 'agent-small-file.txt', + knowledgeSource: 'agent-knowledge-source.txt', + emptyFile: 'agent-empty-file.txt', + unsupportedFile: 'agent-unsupported-file.exe', + specialFilename: 'agent-special-filename-中文 @#$%.txt', + validEnv: 'agent-valid.env', + invalidEnv: 'agent-invalid.env', + buildInstruction: 'agent-build-instruction.txt', + summarySkill: 'e2e-summary-skill/SKILL.md', + fileTreeFixture: 'file_tree_fixture', + countBatch5: 'count_batch_5_valid_files', + countBatch6: 'count_batch_6_valid_files', + countTotal50: 'count_total_50_valid_files', + countTotalExtra1: 'count_total_extra_1_valid_file', +} as const + +export const agentBuilderGeneratedTestMaterials = { + slowUploadFile: 'agent-slow-upload-file.txt', + tooLargeFile: 'agent-too-large-file.txt', +} as const + +export const agentBuilderFileTreeFixtureFiles = [ + 'assets/sample.csv', + 'docs/中文说明.md', + 'public/index.html', + 'src/main.txt', + 'web-game/README.md', +] as const + +export const agentBuilderFileTreeFixtureFileNames = agentBuilderFileTreeFixtureFiles + .map(filePath => path.basename(filePath)) + +export const getAgentBuilderTestMaterialPath = (material: keyof typeof agentBuilderTestMaterials) => + getTestMaterialPath(agentBuilderTestMaterials[material]) + +export const getTooLargeAgentFilePath = () => + getGeneratedTextMaterialPath({ + fileName: 'agent-too-large-file.txt', + sizeBytes: 16 * 1024 * 1024, + seedText: 'E2E_TOO_LARGE_FILE_FIXTURE', + }) + +export const getSlowUploadAgentFilePath = () => + getGeneratedTextMaterialPath({ + fileName: 'agent-slow-upload-file.txt', + sizeBytes: 2 * 1024 * 1024, + seedText: 'E2E_SLOW_UPLOAD_FILE_FIXTURE', + }) diff --git a/e2e/features/step-definitions/agent-v2/access-point.steps.ts b/e2e/features/step-definitions/agent-v2/access-point.steps.ts index 39f66ea282d345..f9ac836119fd96 100644 --- a/e2e/features/step-definitions/agent-v2/access-point.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point.steps.ts @@ -7,8 +7,8 @@ import { getAgentReferencingWorkflows, setAgentApiAccess, setAgentSiteAccessAndGetURL, -} from '../../../support/agent' -import { agentBuilderPreseededResources } from '../../../support/agent-builder-resources' +} from '../../agent-v2/support/agent' +import { agentBuilderPreseededResources } from '../../agent-v2/support/agent-builder-resources' const getCurrentAgentId = (world: DifyWorld) => { const agentId = world.createdAgentIds.at(-1) diff --git a/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts b/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts index ea321973e0f008..530d2db5839378 100644 --- a/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts +++ b/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts @@ -2,10 +2,10 @@ import type { Locator } from '@playwright/test' import type { DifyWorld } from '../../support/world' import { Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' -import { getAgentComposerDraft } from '../../../support/agent' -import { agentBuilderFixedInputs } from '../../../support/agent-builder-resources' -import { skipBlockedPrecondition } from '../../../support/preflight' -import { getAgentBuilderTestMaterialPath } from '../../../support/test-materials' +import { getAgentComposerDraft } from '../../agent-v2/support/agent' +import { agentBuilderFixedInputs } from '../../agent-v2/support/agent-builder-resources' +import { skipBlockedPrecondition } from '../../agent-v2/support/preflight/common' +import { getAgentBuilderTestMaterialPath } from '../../agent-v2/support/test-materials' import { expectAgentEnvVariableHidden, expectAgentEnvVariableVisible, diff --git a/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts b/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts index ff65e1fb28397a..9f1a2a9a845e20 100644 --- a/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts +++ b/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts @@ -1,8 +1,8 @@ import type { DifyWorld } from '../../support/world' import { Then } from '@cucumber/cucumber' import { expect } from '@playwright/test' -import { agentBuilderExpectedTokens, agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../../support/agent-builder-resources' -import { agentBuilderTestMaterials } from '../../../support/test-materials' +import { agentBuilderExpectedTokens, agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../agent-v2/support/agent-builder-resources' +import { agentBuilderTestMaterials } from '../../agent-v2/support/test-materials' import { expectProviderToolActionVisible, openAgentKnowledgeRetrievalDialog } from './configure-helpers' Then('I should see the Agent v2 full-config fixture sections', async function (this: DifyWorld) { diff --git a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts index 728f9916b208f5..bff2b8c2e95748 100644 --- a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts +++ b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts @@ -12,10 +12,10 @@ import { updatedAgentPrompt, updatedAgentSoulConfig, uploadAgentConfigFileToDraft, -} from '../../../support/agent' -import { agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../../support/agent-builder-resources' -import { skipBlockedPrecondition } from '../../../support/preflight' -import { agentBuilderTestMaterials, getAgentBuilderTestMaterialPath } from '../../../support/test-materials' +} from '../../agent-v2/support/agent' +import { agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../agent-v2/support/agent-builder-resources' +import { skipBlockedPrecondition } from '../../agent-v2/support/preflight/common' +import { agentBuilderTestMaterials, getAgentBuilderTestMaterialPath } from '../../agent-v2/support/test-materials' import { getAgentEnvVariableValue, getCurrentAgentId, diff --git a/e2e/features/step-definitions/agent-v2/configure-helpers.ts b/e2e/features/step-definitions/agent-v2/configure-helpers.ts index 19d5fcbc478e91..d5b78834071a1f 100644 --- a/e2e/features/step-definitions/agent-v2/configure-helpers.ts +++ b/e2e/features/step-definitions/agent-v2/configure-helpers.ts @@ -1,16 +1,16 @@ import type { Locator } from '@playwright/test' -import type { AgentComposerEnvVariable } from '../../../support/agent' +import type { AgentComposerEnvVariable } from '../../agent-v2/support/agent' import type { DifyWorld } from '../../support/world' import { expect } from '@playwright/test' import { getAgentComposerDraft, normalAgentPrompt, uploadAgentConfigSkillToDraft, -} from '../../../support/agent' +} from '../../agent-v2/support/agent' import { agentBuilderTestMaterials, getAgentBuilderTestMaterialPath, -} from '../../../support/test-materials' +} from '../../agent-v2/support/test-materials' export const getCurrentAgentId = (world: DifyWorld) => { const agentId = world.createdAgentIds.at(-1) diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 64d56a2edd03a6..b559d0a3413a20 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -13,8 +13,8 @@ import { saveAgentComposerDraft, updatedAgentPrompt, uploadAgentDriveSkill, -} from '../../../support/agent' -import { agentBuilderTestMaterials, getAgentBuilderTestMaterialPath } from '../../../support/test-materials' +} from '../../agent-v2/support/agent' +import { agentBuilderTestMaterials, getAgentBuilderTestMaterialPath } from '../../agent-v2/support/test-materials' import { expectNormalAgentPromptDraft, getCurrentAgentId, diff --git a/e2e/features/step-definitions/agent-v2/files.steps.ts b/e2e/features/step-definitions/agent-v2/files.steps.ts index 8c77b59d4f3b8c..990e4335ae2047 100644 --- a/e2e/features/step-definitions/agent-v2/files.steps.ts +++ b/e2e/features/step-definitions/agent-v2/files.steps.ts @@ -1,7 +1,7 @@ import type { DifyWorld } from '../../support/world' import { Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' -import { agentBuilderFileTreeFixtureFileNames } from '../../../support/test-materials' +import { agentBuilderFileTreeFixtureFileNames } from '../../agent-v2/support/test-materials' import { expectAgentConfigFileHidden, expectAgentConfigFileSaved, diff --git a/e2e/features/step-definitions/agent-v2/output-variables.steps.ts b/e2e/features/step-definitions/agent-v2/output-variables.steps.ts index b091ff46edfff7..134219814f22a2 100644 --- a/e2e/features/step-definitions/agent-v2/output-variables.steps.ts +++ b/e2e/features/step-definitions/agent-v2/output-variables.steps.ts @@ -1,7 +1,7 @@ import type { DifyWorld } from '../../support/world' import { Then } from '@cucumber/cucumber' import { expect } from '@playwright/test' -import { skipBlockedPrecondition } from '../../../support/preflight' +import { skipBlockedPrecondition } from '../../agent-v2/support/preflight/common' Then('Agent v2 standalone Output Variables should be available', async function (this: DifyWorld) { const page = this.getPage() diff --git a/e2e/features/step-definitions/agent-v2/preflight.steps.ts b/e2e/features/step-definitions/agent-v2/preflight.steps.ts index 86a499061c2e5c..09080d7a873597 100644 --- a/e2e/features/step-definitions/agent-v2/preflight.steps.ts +++ b/e2e/features/step-definitions/agent-v2/preflight.steps.ts @@ -1,24 +1,30 @@ import type { DifyWorld } from '../../support/world' import { Given } from '@cucumber/cucumber' import { - skipMissingAgentBuilderBrokenChatModel, - skipMissingAgentBuilderStableChatModel, - skipMissingIndexingPreseededDataset, - skipMissingPreseededAgent, skipMissingPreseededAgentBackendApiKey, + skipMissingPreseededAgentPublishedWebApp, + skipMissingPreseededAgentWorkflowReference, +} from '../../agent-v2/support/preflight/access' +import { + skipMissingPreseededAgent, skipMissingPreseededAgentDriveSkill, skipMissingPreseededAgentFileTreeFixture, skipMissingPreseededAgentFlatFileFixtureConfiguration, - skipMissingPreseededAgentPublishedWebApp, - skipMissingPreseededAgentWorkflowReference, - skipMissingPreseededDataset, skipMissingPreseededDualRetrievalAgentConfiguration, skipMissingPreseededFullConfigAgentCoreConfiguration, - skipMissingPreseededTool, skipMissingPreseededToolStatesAgentConfiguration, skipMissingPreseededWorkflow, +} from '../../agent-v2/support/preflight/agents' +import { + skipMissingIndexingPreseededDataset, + skipMissingPreseededDataset, skipMissingReadyPreseededDataset, -} from '../../../support/preflight' +} from '../../agent-v2/support/preflight/datasets' +import { + skipMissingAgentBuilderBrokenChatModel, + skipMissingAgentBuilderStableChatModel, +} from '../../agent-v2/support/preflight/models' +import { skipMissingPreseededTool } from '../../agent-v2/support/preflight/tools' Given('the Agent Builder stable chat model is available', async function (this: DifyWorld) { const stableModel = await skipMissingAgentBuilderStableChatModel(this) diff --git a/e2e/features/step-definitions/agent-v2/publish.steps.ts b/e2e/features/step-definitions/agent-v2/publish.steps.ts index 2e0b36ae6fa9c5..a1a7b2abe274a5 100644 --- a/e2e/features/step-definitions/agent-v2/publish.steps.ts +++ b/e2e/features/step-definitions/agent-v2/publish.steps.ts @@ -1,8 +1,8 @@ import type { DifyWorld } from '../../support/world' import { Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' -import { getAgentVersionDetail, getTestAgent, normalAgentPrompt } from '../../../support/agent' import { waitForAgentConfigureAutosaved } from '../../../support/agent-configure' +import { getAgentVersionDetail, getTestAgent, normalAgentPrompt } from '../../agent-v2/support/agent' import { getCurrentAgentId } from './configure-helpers' When('I publish the Agent v2 draft', async function (this: DifyWorld) { diff --git a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts index 433d525de2cd2c..44a3125ac42d3f 100644 --- a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts +++ b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts @@ -2,17 +2,17 @@ import type { DataTable } from '@cucumber/cucumber' import type { AgentV2WorkflowOutputVariable, DifyWorld } from '../../support/world' import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' -import { - createAgentSoulConfigWithModel, - createConfiguredTestAgent, - normalAgentSoulConfig, -} from '../../../support/agent' import { createTestApp, getWorkflowDraft, syncAgentV2WorkflowDraft, } from '../../../support/api' import { createE2EResourceName } from '../../../support/naming' +import { + createAgentSoulConfigWithModel, + createConfiguredTestAgent, + normalAgentSoulConfig, +} from '../../agent-v2/support/agent' const agentV2WorkflowNodeId = 'agent-v2' diff --git a/e2e/features/support/hooks.ts b/e2e/features/support/hooks.ts index 393cbab72c31d6..87be4ac4460465 100644 --- a/e2e/features/support/hooks.ts +++ b/e2e/features/support/hooks.ts @@ -7,11 +7,11 @@ import { fileURLToPath } from 'node:url' import { After, AfterAll, Before, BeforeAll, setDefaultTimeout, Status } from '@cucumber/cucumber' import { chromium } from '@playwright/test' import { AUTH_BOOTSTRAP_TIMEOUT_MS, ensureAuthenticatedState } from '../../fixtures/auth' -import { deleteAgentConfigFile, deleteAgentConfigSkill, deleteAgentDriveFile, deleteTestAgent } from '../../support/agent' import { deleteTestApp } from '../../support/api' import { deleteTestDataset } from '../../support/datasets' import { deleteBuiltinToolCredential } from '../../support/tools' import { baseURL, cucumberHeadless, cucumberSlowMo } from '../../test-env' +import { deleteAgentConfigFile, deleteAgentConfigSkill, deleteAgentDriveFile, deleteTestAgent } from '../agent-v2/support/agent' const e2eRoot = fileURLToPath(new URL('../..', import.meta.url)) const artifactsDir = path.join(e2eRoot, 'cucumber-report', 'artifacts') diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts deleted file mode 100644 index fa1640a966b1bd..00000000000000 --- a/e2e/support/preflight.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './preflight/index' diff --git a/e2e/support/preflight/index.ts b/e2e/support/preflight/index.ts deleted file mode 100644 index 4c3ada4720af65..00000000000000 --- a/e2e/support/preflight/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './access' -export * from './agents' -export * from './common' -export * from './datasets' -export * from './models' -export * from './tools' diff --git a/e2e/support/test-materials.ts b/e2e/support/test-materials.ts index 031d31d51f5d95..891f17f7d22346 100644 --- a/e2e/support/test-materials.ts +++ b/e2e/support/test-materials.ts @@ -12,42 +12,6 @@ export const generatedTestMaterialsDir = fileURLToPath( export const getTestMaterialPath = (fileName: string) => path.join(testMaterialsDir, fileName) -export const agentBuilderTestMaterials = { - smallFile: 'agent-small-file.txt', - knowledgeSource: 'agent-knowledge-source.txt', - emptyFile: 'agent-empty-file.txt', - unsupportedFile: 'agent-unsupported-file.exe', - specialFilename: 'agent-special-filename-中文 @#$%.txt', - validEnv: 'agent-valid.env', - invalidEnv: 'agent-invalid.env', - buildInstruction: 'agent-build-instruction.txt', - summarySkill: 'e2e-summary-skill/SKILL.md', - fileTreeFixture: 'file_tree_fixture', - countBatch5: 'count_batch_5_valid_files', - countBatch6: 'count_batch_6_valid_files', - countTotal50: 'count_total_50_valid_files', - countTotalExtra1: 'count_total_extra_1_valid_file', -} as const - -export const agentBuilderGeneratedTestMaterials = { - slowUploadFile: 'agent-slow-upload-file.txt', - tooLargeFile: 'agent-too-large-file.txt', -} as const - -export const agentBuilderFileTreeFixtureFiles = [ - 'assets/sample.csv', - 'docs/中文说明.md', - 'public/index.html', - 'src/main.txt', - 'web-game/README.md', -] as const - -export const agentBuilderFileTreeFixtureFileNames = agentBuilderFileTreeFixtureFiles - .map(filePath => path.basename(filePath)) - -export const getAgentBuilderTestMaterialPath = (material: keyof typeof agentBuilderTestMaterials) => - getTestMaterialPath(agentBuilderTestMaterials[material]) - export async function getGeneratedTextMaterialPath({ fileName, sizeBytes, @@ -67,17 +31,3 @@ export async function getGeneratedTextMaterialPath({ return targetPath } - -export const getTooLargeAgentFilePath = () => - getGeneratedTextMaterialPath({ - fileName: 'agent-too-large-file.txt', - sizeBytes: 16 * 1024 * 1024, - seedText: 'E2E_TOO_LARGE_FILE_FIXTURE', - }) - -export const getSlowUploadAgentFilePath = () => - getGeneratedTextMaterialPath({ - fileName: 'agent-slow-upload-file.txt', - sizeBytes: 2 * 1024 * 1024, - seedText: 'E2E_SLOW_UPLOAD_FILE_FIXTURE', - }) From bbd998b80deddc879e697e2f11609665635e2e29 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 21:15:04 +0800 Subject: [PATCH 088/185] fix(api): type agent build draft schema --- api/controllers/console/agent/roster.py | 16 ++-- .../generated/api/console/agent/types.gen.ts | 44 ++++------ .../generated/api/console/agent/zod.gen.ts | 82 +++++++++---------- 3 files changed, 69 insertions(+), 73 deletions(-) diff --git a/api/controllers/console/agent/roster.py b/api/controllers/console/agent/roster.py index ec7f32c760927b..d4a72b75d92122 100644 --- a/api/controllers/console/agent/roster.py +++ b/api/controllers/console/agent/roster.py @@ -39,9 +39,11 @@ ) from extensions.ext_database import db from fields.agent_fields import ( + AgentConfigDraftSummaryResponse, AgentConfigSnapshotDetailResponse, AgentConfigSnapshotListResponse, AgentConfigSnapshotRestoreResponse, + AgentConfigSnapshotSummaryResponse, AgentInviteOptionsResponse, AgentLogListResponse, AgentLogMessageListResponse, @@ -50,11 +52,13 @@ AgentRosterListResponse, AgentStatisticSummaryEnvelopeResponse, ) +from fields.base import ResponseModel from libs.datetime_utils import parse_time_range from libs.helper import dump_response from libs.login import login_required from models import Account from models.agent import Agent, AgentStatus +from models.agent_config_entities import AgentSoulConfig from models.enums import ApiTokenType from models.model import ApiToken, App, IconType from services.agent.composer_service import AgentComposerService @@ -264,21 +268,21 @@ class AgentPublishPayload(BaseModel): version_note: str | None = Field(default=None, description="Optional note for this published Agent version") -class AgentPublishResponse(BaseModel): +class AgentPublishResponse(ResponseModel): result: str active_config_snapshot_id: str - active_config_snapshot: dict[str, object] | None = None - draft: dict[str, object] | None = None + active_config_snapshot: AgentConfigSnapshotSummaryResponse | None = None + draft: AgentConfigDraftSummaryResponse | None = None class AgentBuildDraftCheckoutPayload(BaseModel): force: bool = Field(default=False, description="Overwrite the existing current-user build draft") -class AgentBuildDraftResponse(BaseModel): +class AgentBuildDraftResponse(ResponseModel): variant: str - draft: dict[str, object] - agent_soul: dict[str, object] + draft: AgentConfigDraftSummaryResponse + agent_soul: AgentSoulConfig class AgentBuildDraftApplyResponse(BaseModel): diff --git a/packages/contracts/generated/api/console/agent/types.gen.ts b/packages/contracts/generated/api/console/agent/types.gen.ts index 08b9fe90e1d75e..036cccc5da7b8c 100644 --- a/packages/contracts/generated/api/console/agent/types.gen.ts +++ b/packages/contracts/generated/api/console/agent/types.gen.ts @@ -120,12 +120,8 @@ export type AgentSimpleResultResponse = { } export type AgentBuildDraftResponse = { - agent_soul: { - [key: string]: unknown - } - draft: { - [key: string]: unknown - } + agent_soul: AgentSoulConfig + draft: AgentConfigDraftSummaryResponse variant: string } @@ -411,13 +407,9 @@ export type AgentPublishPayload = { } export type AgentPublishResponse = { - active_config_snapshot?: { - [key: string]: unknown - } | null + active_config_snapshot?: AgentConfigSnapshotSummaryResponse | null active_config_snapshot_id: string - draft?: { - [key: string]: unknown - } | null + draft?: AgentConfigDraftSummaryResponse | null result: string } @@ -650,6 +642,18 @@ export type AgentSoulConfig = { tools?: AgentSoulToolsConfig } +export type AgentConfigDraftSummaryResponse = { + account_id?: string | null + agent_id: string + base_snapshot_id?: string | null + created_at?: number | null + created_by?: string | null + draft_type: AgentConfigDraftType + id: string + updated_at?: number | null + updated_by?: string | null +} + export type ComposerBindingPayload = { agent_id?: string | null binding_type: 'inline_agent' | 'roster_agent' @@ -711,18 +715,6 @@ export type AgentComposerAgentResponse = { status: AgentStatus } -export type AgentConfigDraftSummaryResponse = { - account_id?: string | null - agent_id: string - base_snapshot_id?: string | null - created_at?: number | null - created_by?: string | null - draft_type: AgentConfigDraftType - id: string - updated_at?: number | null - updated_by?: string | null -} - export type ComposerValidationFindingsResponse = { knowledge_retrieval_placeholder?: Array warnings?: Array @@ -1202,6 +1194,8 @@ export type AgentSoulToolsConfig = { dify_tools?: Array } +export type AgentConfigDraftType = 'debug_build' | 'draft' + export type DeclaredOutputConfig = { array_item?: DeclaredArrayItem | null check?: DeclaredOutputCheckConfig | null @@ -1268,8 +1262,6 @@ export type WorkflowPreviousNodeOutputRef = { [key: string]: unknown } -export type AgentConfigDraftType = 'debug_build' | 'draft' - export type DeclaredOutputType = 'array' | 'boolean' | 'file' | 'number' | 'object' | 'string' export type AgentCliToolConfig = { diff --git a/packages/contracts/generated/api/console/agent/zod.gen.ts b/packages/contracts/generated/api/console/agent/zod.gen.ts index d14cd5c7e89b00..f68a2ce99e161d 100644 --- a/packages/contracts/generated/api/console/agent/zod.gen.ts +++ b/packages/contracts/generated/api/console/agent/zod.gen.ts @@ -61,15 +61,6 @@ export const zAgentSimpleResultResponse = z.object({ result: z.string(), }) -/** - * AgentBuildDraftResponse - */ -export const zAgentBuildDraftResponse = z.object({ - agent_soul: z.record(z.string(), z.unknown()), - draft: z.record(z.string(), z.unknown()), - variant: z.string(), -}) - /** * AgentBuildDraftApplyResponse */ @@ -194,16 +185,6 @@ export const zAgentPublishPayload = z.object({ version_note: z.string().nullish(), }) -/** - * AgentPublishResponse - */ -export const zAgentPublishResponse = z.object({ - active_config_snapshot: z.record(z.string(), z.unknown()).nullish(), - active_config_snapshot_id: z.string(), - draft: z.record(z.string(), z.unknown()).nullish(), - result: z.string(), -}) - /** * SandboxReadResponse */ @@ -1233,6 +1214,38 @@ export const zAgentSoulPromptConfig = z.object({ system_prompt: z.string().optional().default(''), }) +/** + * AgentConfigDraftType + * + * Editable Agent Soul draft workspace type. + */ +export const zAgentConfigDraftType = z.enum(['debug_build', 'draft']) + +/** + * AgentConfigDraftSummaryResponse + */ +export const zAgentConfigDraftSummaryResponse = z.object({ + account_id: z.string().nullish(), + agent_id: z.string(), + base_snapshot_id: z.string().nullish(), + created_at: z.int().nullish(), + created_by: z.string().nullish(), + draft_type: zAgentConfigDraftType, + id: z.string(), + updated_at: z.int().nullish(), + updated_by: z.string().nullish(), +}) + +/** + * AgentPublishResponse + */ +export const zAgentPublishResponse = z.object({ + active_config_snapshot: zAgentConfigSnapshotSummaryResponse.nullish(), + active_config_snapshot_id: z.string(), + draft: zAgentConfigDraftSummaryResponse.nullish(), + result: z.string(), +}) + /** * AgentHumanContactConfig */ @@ -1271,28 +1284,6 @@ export const zWorkflowPreviousNodeOutputRef = z.object({ .nullish(), }) -/** - * AgentConfigDraftType - * - * Editable Agent Soul draft workspace type. - */ -export const zAgentConfigDraftType = z.enum(['debug_build', 'draft']) - -/** - * AgentConfigDraftSummaryResponse - */ -export const zAgentConfigDraftSummaryResponse = z.object({ - account_id: z.string().nullish(), - agent_id: z.string(), - base_snapshot_id: z.string().nullish(), - created_at: z.int().nullish(), - created_by: z.string().nullish(), - draft_type: zAgentConfigDraftType, - id: z.string(), - updated_at: z.int().nullish(), - updated_by: z.string().nullish(), -}) - /** * DeclaredOutputType */ @@ -2384,6 +2375,15 @@ export const zAgentSoulConfig = z.object({ tools: zAgentSoulToolsConfig.optional(), }) +/** + * AgentBuildDraftResponse + */ +export const zAgentBuildDraftResponse = z.object({ + agent_soul: zAgentSoulConfig, + draft: zAgentConfigDraftSummaryResponse, + variant: z.string(), +}) + /** * ComposerSavePayload */ From 52c7e76a175f111802d8a5b643a9676fd91c6122 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Wed, 1 Jul 2026 13:21:29 +0000 Subject: [PATCH 089/185] [autofix.ci] apply automated fixes --- api/openapi/markdown/console-openapi.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/openapi/markdown/console-openapi.md b/api/openapi/markdown/console-openapi.md index 0c113e237bf62b..8f9ae90496ce2a 100644 --- a/api/openapi/markdown/console-openapi.md +++ b/api/openapi/markdown/console-openapi.md @@ -13079,8 +13079,8 @@ default (the config form sends the full desired feature state on save). | Name | Type | Description | Required | | ---- | ---- | ----------- | -------- | -| agent_soul | object | | Yes | -| draft | object | | Yes | +| agent_soul | [AgentSoulConfig](#agentsoulconfig) | | Yes | +| draft | [AgentConfigDraftSummaryResponse](#agentconfigdraftsummaryresponse) | | Yes | | variant | string | | Yes | #### AgentCliToolAuthorizationStatus @@ -14155,9 +14155,9 @@ section may be empty, which is how callers express "no knowledge layer". | Name | Type | Description | Required | | ---- | ---- | ----------- | -------- | -| active_config_snapshot | object | | No | +| active_config_snapshot | [AgentConfigSnapshotSummaryResponse](#agentconfigsnapshotsummaryresponse) | | No | | active_config_snapshot_id | string | | Yes | -| draft | object | | No | +| draft | [AgentConfigDraftSummaryResponse](#agentconfigdraftsummaryresponse) | | No | | result | string | | Yes | #### AgentPublishedReferenceResponse From 11c52ddf09916857d2b828846541214d7f500e33 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 21:24:43 +0800 Subject: [PATCH 090/185] test(e2e): use generated agent api contracts --- e2e/AGENTS.md | 2 + e2e/features/agent-v2/AGENTS.md | 6 + e2e/features/agent-v2/support/agent.ts | 217 ++++++------------ .../agent-v2/support/preflight/access.ts | 41 +--- .../agent-v2/support/preflight/agents.ts | 34 +-- .../agent-v2/support/preflight/common.ts | 8 +- .../agent-v2/support/preflight/datasets.ts | 26 +-- .../agent-v2/support/preflight/models.ts | 18 +- .../agent-v2/build-draft.steps.ts | 10 +- .../agent-v2/configure.steps.ts | 6 +- .../agent-v2/workflow-node.steps.ts | 2 +- e2e/package.json | 1 + pnpm-lock.yaml | 3 + 13 files changed, 126 insertions(+), 248 deletions(-) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 66f90a1ad668f6..599bd4021b75db 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -300,6 +300,8 @@ Use scoped feature support for scenarios that require optional external resource Keep package-level support limited to broadly reusable primitives such as API clients, naming, fixture path resolution, and cleanup helpers. Feature-specific seed contracts and preflight checks belong under the owning feature's support folder. +Use generated API contracts for Console/Web/Service API request, response, and payload shapes. Import the concrete type directly from `@dify/contracts/.../types.gen` when it exists, and do not hand-write duplicate response shapes or wrap generated types in local aliases just to preserve an older helper name. Keep local E2E types only for scenario state, fixture registries, helper input options, preflight resource state, and intentionally narrowed test view models that are not complete API responses. + Use typed cleanup fields on `DifyWorld` for resource types created by scenarios, and use `DifyWorld.registerCleanup(...)` when a scenario creates any resource type that is not covered by typed cleanup fields. Cleanup callbacks run after typed cleanup queues, even when the scenario fails. Feature-specific seed contracts, resource readiness rules, tags, and scenario ownership can be documented in one scoped `AGENTS.md` at the feature root when a module becomes large enough to need it. Do not add deeper `AGENTS.md` files unless the nested module becomes independently owned. diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index dee75f9e5d38fb..9e342a75fb7342 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -76,6 +76,12 @@ Use `the Agent v2 configuration should be saved automatically` after UI edits th API setup is acceptable for creating scenario-owned Agents, enabling Backend service API, writing composer drafts, seeding Build drafts, and preparing fixed state. The scenario must still assert user-visible behavior or a real persisted product contract through the public Console API. Do not assert only that a setup API call succeeded. +## API Contract Types + +Agent v2 support helpers consume Console API contracts from `@dify/contracts/api/console/.../types.gen`. When a generated request, response, or payload type exists, import and use that exact type name at the helper boundary. Do not keep an old local response type name as an alias for the generated type. + +Keep local types for Agent v2 E2E-owned state only, such as `DifyWorld.agentBuilder` state, scenario preflight resource records, fixture registry entries, helper input options, and deliberately narrowed test view models. If an endpoint response needs a field that the generated contract does not expose yet, fix the backend schema and regenerate contracts before broadening E2E types. + ## Build Mode And Preview Build mode scope means: diff --git a/e2e/features/agent-v2/support/agent.ts b/e2e/features/agent-v2/support/agent.ts index a3b22a57f77755..7c3cefd29274df 100644 --- a/e2e/features/agent-v2/support/agent.ts +++ b/e2e/features/agent-v2/support/agent.ts @@ -1,118 +1,52 @@ +import type { + AgentApiAccessResponse, + AgentAppComposerResponse, + AgentAppDetailWithSite, + AgentBuildDraftResponse, + AgentConfigFileRefConfig, + AgentConfigFileUploadResponse, + AgentConfigSkillRefConfig, + AgentConfigSkillUploadResponse, + AgentConfigSnapshotDetailResponse, + AgentDriveSkillItemResponse, + AgentDriveSkillListResponse, + AgentReferencingWorkflowResponse, + AgentReferencingWorkflowsResponse, + AgentSkillUploadResponse, + AgentSoulConfig, + ApiKeyItem, +} from '@dify/contracts/api/console/agent/types.gen' import { Buffer } from 'node:buffer' import { readFile } from 'node:fs/promises' import path from 'node:path' import { createApiContext, expectApiResponseOK, setAppSiteEnabled } from '../../../support/api' import { assertE2EResourceName, createE2EResourceName } from '../../../support/naming' -export type AgentSeed = { - active_config_is_published?: boolean +export type AgentSeed = Pick< + AgentAppDetailWithSite, + | 'active_config_is_published' + | 'app_id' + | 'backing_app_id' + | 'description' + | 'enable_site' + | 'id' + | 'name' + | 'role' + | 'site' +> & { active_config_snapshot_id?: string | null - app_id?: string - backing_app_id?: string - description?: string - enable_site?: boolean - id: string - name: string - role?: string - site?: { - access_token?: string | null - app_base_url?: string | null - code?: string | null - } | null } -export type AgentComposerConfigFile = { - file_id?: string | null - file_kind?: string | null - hash?: string | null - mime_type?: string | null - name: string - size?: number | null -} -export type AgentComposerConfigSkill = { - description?: string | null - file_id?: string | null - file_kind?: string | null - hash?: string | null - mime_type?: string | null - name: string - size?: number | null -} -export type AgentComposerEnvVariable = { - id?: string | null - key?: string | null - name?: string | null - value?: unknown - variable?: string | null -} +export type { AgentSoulConfig } -export type AgentSoulConfig = Record & { - config_files?: AgentComposerConfigFile[] - config_skills?: AgentComposerConfigSkill[] - env?: { - secret_refs?: unknown[] - variables?: AgentComposerEnvVariable[] - } -} +export type AgentComposerEnvVariable = NonNullable< + NonNullable['variables'] +>[number] export type AgentModelSelection = { name: string provider: string } -export type AgentComposerResponse = { - agent_soul?: AgentSoulConfig -} - -export type AgentBuildDraftResponse = { - agent_soul: AgentSoulConfig - draft: Record - variant: 'agent_app' -} - -export type AgentApiAccess = { - api_key_count: number - enabled: boolean - files_upload_endpoint: string - service_api_base_url: string -} - -export type AgentApiKey = { - id: string - token?: string -} - -export type AgentConfigSnapshotDetail = { - config_snapshot: AgentSoulConfig - id: string -} - -export type AgentReferencingWorkflow = { - app_id: string - app_name: string - app_updated_at?: number | null - node_ids?: string[] - workflow_id: string - workflow_version: string -} - -export type AgentReferencingWorkflowsResponse = { - data: AgentReferencingWorkflow[] -} - -export type AgentDriveSkillUpload = { - skill: { - archive_key?: string | null - description: string - name: string - path: string - skill_md_key: string - } -} - -export type AgentConfigSkillUpload = { - skill: AgentComposerConfigSkill -} - export type UploadedConsoleFile = { id: string mime_type?: string | null @@ -120,20 +54,6 @@ export type UploadedConsoleFile = { size?: number | null } -export type AgentConfigFileUpload = { - file: AgentComposerConfigFile -} - -export type AgentDriveSkill = { - description?: string | null - name: string - path: string -} - -export type AgentDriveSkillListResponse = { - items: AgentDriveSkill[] -} - export type CreateTestAgentOptions = { description?: string name?: string @@ -362,12 +282,12 @@ export async function getTestAgent(agentId: string): Promise { export async function getAgentVersionDetail( agentId: string, versionId: string, -): Promise { +): Promise { const ctx = await createApiContext() try { const response = await ctx.get(`/console/api/agent/${agentId}/versions/${versionId}`) await expectApiResponseOK(response, `Get Agent v2 version ${versionId} for ${agentId}`) - return (await response.json()) as AgentConfigSnapshotDetail + return (await response.json()) as AgentConfigSnapshotDetailResponse } finally { await ctx.dispose() @@ -388,7 +308,7 @@ export async function deleteTestAgent(agentId: string): Promise { export async function saveAgentComposerDraft( agentId: string, agentSoul: AgentSoulConfig = defaultAgentSoulConfig, -): Promise { +): Promise { const ctx = await createApiContext() try { const response = await ctx.put(`/console/api/agent/${agentId}/composer`, { @@ -399,7 +319,7 @@ export async function saveAgentComposerDraft( }, }) await expectApiResponseOK(response, `Save Agent v2 composer draft for ${agentId}`) - return (await response.json()) as AgentComposerResponse + return (await response.json()) as AgentAppComposerResponse } finally { await ctx.dispose() @@ -414,7 +334,7 @@ export async function uploadAgentDriveSkill({ agentId: string fileName: string filePath: string -}): Promise { +}): Promise { const ctx = await createApiContext() try { const upload = await toSkillArchiveUpload({ fileName, filePath }) @@ -428,7 +348,7 @@ export async function uploadAgentDriveSkill({ }, }) await expectApiResponseOK(response, `Upload Agent v2 drive skill ${fileName} for ${agentId}`) - return (await response.json()) as AgentDriveSkillUpload + return (await response.json()) as AgentSkillUploadResponse } finally { await ctx.dispose() @@ -443,7 +363,7 @@ export async function uploadAgentConfigFileToDraft({ agentId: string fileName: string filePath: string -}): Promise { +}): Promise { const ctx = await createApiContext() try { const uploadResponse = await ctx.post('/console/api/files/upload', { @@ -464,12 +384,18 @@ export async function uploadAgentConfigFileToDraft({ }, }) await expectApiResponseOK(commitResponse, `Commit Agent v2 config file ${fileName} for ${agentId}`) - const body = (await commitResponse.json()) as AgentConfigFileUpload - const { id: _id, ...file } = body.file as AgentComposerConfigFile & { id?: string } + const body = (await commitResponse.json()) as AgentConfigFileUploadResponse + const file = body.file + if (!file.file_id) + throw new Error(`Agent v2 config file ${fileName} did not return a file_id.`) return { - ...file, - file_kind: file.file_kind ?? 'upload_file', + file_id: file.file_id, + file_kind: 'upload_file', + hash: file.hash, + mime_type: file.mime_type, + name: file.name, + size: file.size, } } finally { @@ -485,7 +411,7 @@ export async function uploadAgentConfigSkillToDraft({ agentId: string fileName: string filePath: string -}): Promise { +}): Promise { const ctx = await createApiContext() try { const upload = await toSkillArchiveUpload({ fileName, filePath }) @@ -499,12 +425,19 @@ export async function uploadAgentConfigSkillToDraft({ }, }) await expectApiResponseOK(response, `Upload Agent v2 config skill ${fileName} for ${agentId}`) - const body = (await response.json()) as AgentConfigSkillUpload - const { id: _id, ...skill } = body.skill as AgentComposerConfigSkill & { id?: string } + const body = (await response.json()) as AgentConfigSkillUploadResponse + const skill = body.skill + if (!skill.file_id) + throw new Error(`Agent v2 config skill ${fileName} did not return a file_id.`) return { - ...skill, - file_kind: skill.file_kind ?? 'tool_file', + description: skill.description, + file_id: skill.file_id, + file_kind: 'tool_file', + hash: skill.hash, + mime_type: skill.mime_type, + name: skill.name, + size: skill.size, } } finally { @@ -512,38 +445,38 @@ export async function uploadAgentConfigSkillToDraft({ } } -export async function getAgentDriveSkills(agentId: string): Promise { +export async function getAgentDriveSkills(agentId: string): Promise { const ctx = await createApiContext() try { const response = await ctx.get(`/console/api/agent/${agentId}/drive/skills`) await expectApiResponseOK(response, `Get Agent v2 drive skills for ${agentId}`) const body = (await response.json()) as AgentDriveSkillListResponse - return body.items + return body.items ?? [] } finally { await ctx.dispose() } } -export async function getAgentReferencingWorkflows(agentId: string): Promise { +export async function getAgentReferencingWorkflows(agentId: string): Promise { const ctx = await createApiContext() try { const response = await ctx.get(`/console/api/agent/${agentId}/referencing-workflows`) await expectApiResponseOK(response, `Get Agent v2 referencing workflows for ${agentId}`) const body = (await response.json()) as AgentReferencingWorkflowsResponse - return body.data + return body.data ?? [] } finally { await ctx.dispose() } } -export async function getAgentComposerDraft(agentId: string): Promise { +export async function getAgentComposerDraft(agentId: string): Promise { const ctx = await createApiContext() try { const response = await ctx.get(`/console/api/agent/${agentId}/composer`) await expectApiResponseOK(response, `Get Agent v2 composer draft for ${agentId}`) - return (await response.json()) as AgentComposerResponse + return (await response.json()) as AgentAppComposerResponse } finally { await ctx.dispose() @@ -629,12 +562,12 @@ export async function setAgentSiteAccessAndGetURL( return `${baseURL.replace(/\/$/, '')}/agent/${token}` } -export async function getAgentApiAccess(agentId: string): Promise { +export async function getAgentApiAccess(agentId: string): Promise { const ctx = await createApiContext() try { const response = await ctx.get(`/console/api/agent/${agentId}/api-access`) await expectApiResponseOK(response, `Get Agent v2 API access for ${agentId}`) - return (await response.json()) as AgentApiAccess + return (await response.json()) as AgentApiAccessResponse } finally { await ctx.dispose() @@ -644,7 +577,7 @@ export async function getAgentApiAccess(agentId: string): Promise { +): Promise { const ctx = await createApiContext() try { const response = await ctx.post(`/console/api/agent/${agentId}/api-enable`, { @@ -654,19 +587,19 @@ export async function setAgentApiAccess( response, `${enabled ? 'Enable' : 'Disable'} Agent v2 API access for ${agentId}`, ) - return (await response.json()) as AgentApiAccess + return (await response.json()) as AgentApiAccessResponse } finally { await ctx.dispose() } } -export async function createAgentApiKey(agentId: string): Promise { +export async function createAgentApiKey(agentId: string): Promise { const ctx = await createApiContext() try { const response = await ctx.post(`/console/api/agent/${agentId}/api-keys`) await expectApiResponseOK(response, `Create Agent v2 API key for ${agentId}`) - return (await response.json()) as AgentApiKey + return (await response.json()) as ApiKeyItem } finally { await ctx.dispose() diff --git a/e2e/features/agent-v2/support/preflight/access.ts b/e2e/features/agent-v2/support/preflight/access.ts index f721fc716b3728..a315f09b92abb5 100644 --- a/e2e/features/agent-v2/support/preflight/access.ts +++ b/e2e/features/agent-v2/support/preflight/access.ts @@ -1,38 +1,15 @@ +import type { + AgentApiAccessResponse, + AgentAppDetailWithSite, + AgentReferencingWorkflowsResponse, + ApiKeyList, +} from '@dify/contracts/api/console/agent/types.gen' import type { DifyWorld } from '../../../support/world' import type { PreseededResource } from './common' import { createApiContext, expectApiResponseOK } from '../../../../support/api' import { skipMissingPreseededAgent, skipMissingPreseededWorkflow } from './agents' import { skipBlockedPrecondition } from './common' -type AgentApiAccessResponse = { - api_key_count: number - enabled: boolean -} - -type AgentApiKeyListResponse = { - data: Array<{ - id: string - }> -} - -type AgentReferencingWorkflowsResponse = { - data: Array<{ - app_id: string - app_name: string - node_ids?: string[] - }> -} - -type PreseededAgentDetailResponse = { - active_config_is_published?: boolean - enable_site?: boolean - site?: { - access_token?: string | null - app_base_url?: string | null - code?: string | null - } | null -} - export async function skipMissingPreseededAgentBackendApiKey( world: DifyWorld, agentName: string, @@ -55,7 +32,7 @@ export async function skipMissingPreseededAgentBackendApiKey( const keyResponse = await ctx.get(`/console/api/agent/${agent.id}/api-keys`) await expectApiResponseOK(keyResponse, `Check preseeded Agent API key ${agentName}`) - const keys = (await keyResponse.json()) as AgentApiKeyListResponse + const keys = (await keyResponse.json()) as ApiKeyList const key = keys.data.at(0) if (!key) { return skipBlockedPrecondition( @@ -87,7 +64,7 @@ export async function skipMissingPreseededAgentPublishedWebApp( try { const response = await ctx.get(`/console/api/agent/${agent.id}`) await expectApiResponseOK(response, `Check preseeded Agent published Web app ${agentName}`) - const detail = (await response.json()) as PreseededAgentDetailResponse + const detail = (await response.json()) as AgentAppDetailWithSite if (detail.active_config_is_published !== true) { return skipBlockedPrecondition(world, `Preseeded Agent "${agentName}" is not published.`) } @@ -136,7 +113,7 @@ export async function skipMissingPreseededAgentWorkflowReference( const response = await ctx.get(`/console/api/agent/${agent.id}/referencing-workflows`) await expectApiResponseOK(response, `Check preseeded Agent workflow reference ${agentName}`) const references = (await response.json()) as AgentReferencingWorkflowsResponse - const reference = references.data.find( + const reference = references.data?.find( item => item.app_id === workflow.id || item.app_name === workflow.name, ) diff --git a/e2e/features/agent-v2/support/preflight/agents.ts b/e2e/features/agent-v2/support/preflight/agents.ts index 2c06376a1c6b04..bc9a5cdf960f88 100644 --- a/e2e/features/agent-v2/support/preflight/agents.ts +++ b/e2e/features/agent-v2/support/preflight/agents.ts @@ -1,5 +1,10 @@ +import type { + AgentAppComposerResponse, + AgentDriveListResponse, + AgentDriveSkillListResponse, +} from '@dify/contracts/api/console/agent/types.gen' import type { DifyWorld } from '../../../support/world' -import type { AgentComposerResponse, PreseededResource } from './common' +import type { PreseededResource } from './common' import { createApiContext, expectApiResponseOK } from '../../../../support/api' import { agentBuilderExpectedTokens, @@ -12,14 +17,12 @@ import { agentBuilderTestMaterials, } from '../test-materials' import { - asArray, asRecord, asString, buildQuery, findConsoleResourceByName, hasNamedOrKeyedEntry, - skipBlockedPrecondition, } from './common' import { skipMissingReadyPreseededDataset } from './datasets' @@ -32,19 +35,6 @@ import { splitToolDisplayName, } from './tools' -type AgentDriveSkillListResponse = { - items: Array<{ - name: string - path: string - }> -} - -type AgentDriveFileListResponse = { - items?: Array<{ - key: string - }> -} - const hasKnowledgeDataset = ( soul: Record, dataset: PreseededResource, @@ -150,7 +140,7 @@ export async function skipMissingPreseededAgentDriveSkill( const response = await ctx.get(`/console/api/agent/${agent.id}/drive/skills`) await expectApiResponseOK(response, `Check preseeded Agent skill ${skillName}`) const body = (await response.json()) as AgentDriveSkillListResponse - const skill = body.items.find(item => item.name === skillName) + const skill = body.items?.find(item => item.name === skillName) if (!skill) { return skipBlockedPrecondition( @@ -208,7 +198,7 @@ export async function skipMissingPreseededFullConfigAgentCoreConfiguration( try { const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) await expectApiResponseOK(response, `Check preseeded Agent core configuration ${agentName}`) - const body = (await response.json()) as AgentComposerResponse + const body = (await response.json()) as AgentAppComposerResponse const soul = body.agent_soul ?? {} const missing: string[] = [] @@ -294,7 +284,7 @@ export async function skipMissingPreseededToolStatesAgentConfiguration( try { const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) await expectApiResponseOK(response, `Check preseeded Agent tool states ${agentName}`) - const body = (await response.json()) as AgentComposerResponse + const body = (await response.json()) as AgentAppComposerResponse const soul = body.agent_soul ?? {} const toolItems = asArray(asRecord(soul.tools).dify_tools) const missing: string[] = [] @@ -364,7 +354,7 @@ export async function skipMissingPreseededDualRetrievalAgentConfiguration( try { const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) await expectApiResponseOK(response, `Check preseeded Agent dual retrieval ${agentName}`) - const body = (await response.json()) as AgentComposerResponse + const body = (await response.json()) as AgentAppComposerResponse const soul = body.agent_soul ?? {} const missing: string[] = [] @@ -407,7 +397,7 @@ export async function skipMissingPreseededAgentFileTreeFixture( const query = buildQuery({ prefix: 'files/' }) const response = await ctx.get(`/console/api/agent/${agent.id}/drive/files?${query}`) await expectApiResponseOK(response, `Check preseeded Agent file tree ${agentName}`) - const body = (await response.json()) as AgentDriveFileListResponse + const body = (await response.json()) as AgentDriveListResponse const keys = (body.items ?? []).map(item => item.key) const missingFiles = agentBuilderFileTreeFixtureFiles.filter( filePath => @@ -444,7 +434,7 @@ export async function skipMissingPreseededAgentFlatFileFixtureConfiguration( try { const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) await expectApiResponseOK(response, `Check preseeded Agent flat file fixture ${agentName}`) - const body = (await response.json()) as AgentComposerResponse + const body = (await response.json()) as AgentAppComposerResponse const configFiles = Array.isArray(body.agent_soul?.config_files) ? body.agent_soul.config_files : [] diff --git a/e2e/features/agent-v2/support/preflight/common.ts b/e2e/features/agent-v2/support/preflight/common.ts index bb6150ff7a56a5..ea9ed1236d22fe 100644 --- a/e2e/features/agent-v2/support/preflight/common.ts +++ b/e2e/features/agent-v2/support/preflight/common.ts @@ -21,7 +21,7 @@ export type NamedResource = { name: string } -export type NamedResourceListResponse = { +export type NamedResourceCollection = { data: T[] } @@ -30,10 +30,6 @@ export type LocalizedLabel = { zh_Hans?: string } -export type AgentComposerResponse = { - agent_soul?: Record -} - export const readRequiredEnvResource = ( envName: string, description: string, @@ -94,7 +90,7 @@ export const findConsoleResourceByName = async + const body = (await response.json()) as NamedResourceCollection return body.data.find(item => item.name === resourceName) } diff --git a/e2e/features/agent-v2/support/preflight/datasets.ts b/e2e/features/agent-v2/support/preflight/datasets.ts index dc992ecdb5bbb4..a2677f21683ef0 100644 --- a/e2e/features/agent-v2/support/preflight/datasets.ts +++ b/e2e/features/agent-v2/support/preflight/datasets.ts @@ -1,5 +1,9 @@ +import type { + DatasetListItemResponse, + DocumentStatusListResponse, +} from '@dify/contracts/api/console/datasets/types.gen' import type { DifyWorld } from '../../../support/world' -import type { NamedResource, PreseededResource } from './common' +import type { PreseededResource } from './common' import { createApiContext, expectApiResponseOK } from '../../../../support/api' import { buildQuery, @@ -8,11 +12,6 @@ import { skipBlockedPrecondition, } from './common' -type DatasetResource = NamedResource & { - document_count: number - total_available_documents: number -} - type DocumentIndexingStatus = | 'cleaning' | 'completed' @@ -21,13 +20,6 @@ type DocumentIndexingStatus | 'splitting' | 'waiting' -type DatasetIndexingStatusResponse = { - data: Array<{ - id: string - indexing_status?: string - }> -} - const completedDocumentIndexingStatus: DocumentIndexingStatus = 'completed' const activeDocumentIndexingStatuses = new Set([ 'cleaning', @@ -40,7 +32,7 @@ const activeDocumentIndexingStatuses = new Set([ export const getPreseededDataset = async (resourceName: string) => { const query = buildQuery({ keyword: resourceName, limit: '20', page: '1' }) - return findConsoleResourceByName({ + return findConsoleResourceByName({ action: `Check preseeded dataset ${resourceName}`, path: `/console/api/datasets?${query}`, resourceName, @@ -52,7 +44,7 @@ const getDatasetIndexingStatuses = async (datasetId: string, resourceName: strin try { const response = await ctx.get(`/console/api/datasets/${datasetId}/indexing-status`) await expectApiResponseOK(response, `Check preseeded dataset indexing status ${resourceName}`) - const body = (await response.json()) as DatasetIndexingStatusResponse + const body = (await response.json()) as DocumentStatusListResponse return body.data } @@ -61,9 +53,7 @@ const getDatasetIndexingStatuses = async (datasetId: string, resourceName: strin } } -export const toDatasetResource = ( - resource: NamedResource, -): PreseededResource => ({ +export const toDatasetResource = (resource: DatasetListItemResponse): PreseededResource => ({ id: resource.id, kind: 'dataset', name: resource.name, diff --git a/e2e/features/agent-v2/support/preflight/models.ts b/e2e/features/agent-v2/support/preflight/models.ts index 0176947543f807..16f1438cf14a29 100644 --- a/e2e/features/agent-v2/support/preflight/models.ts +++ b/e2e/features/agent-v2/support/preflight/models.ts @@ -1,3 +1,4 @@ +import type { ProviderWithModelsDataResponse } from '@dify/contracts/api/console/workspaces/types.gen' import type { DifyWorld } from '../../../support/world' import { createApiContext, expectApiResponseOK } from '../../../../support/api' import { agentBuilderPreseededResources } from '../agent-builder-resources' @@ -13,21 +14,6 @@ const activeModelStatus = 'active' const defaultStableChatModelType = 'llm' const defaultBrokenChatModelName = agentBuilderPreseededResources.brokenModel -type ModelTypeListResponse = { - data: Array<{ - provider: string - models: Array<{ - label?: { - en_US?: string - zh_Hans?: string - } - model: string - status?: string - }> - status?: string - }> -} - type ModelPreflightConfig = | { ok: true @@ -107,7 +93,7 @@ async function skipMissingAgentBuilderModel( `/console/api/workspaces/current/models/model-types/${config.type}`, ) await expectApiResponseOK(response, `Check ${config.resourceName}`) - const body = (await response.json()) as ModelTypeListResponse + const body = (await response.json()) as ProviderWithModelsDataResponse const provider = body.data.find(item => item.provider === config.provider) const model = provider?.models.find( item => diff --git a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts index bff2b8c2e95748..3ac0b05ce920d6 100644 --- a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts +++ b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts @@ -48,10 +48,7 @@ Given( await saveAgentBuildDraft(agentId, { ...updatedConfig, config_files: [configFile], - config_skills: [{ - ...skill, - file_kind: skill.file_kind ?? 'tool_file', - }], + config_skills: [skill], env: { secret_refs: [], variables: [{ @@ -76,10 +73,7 @@ Given( const updatedConfig = this.agentBuilder.preflight.stableModel ? createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilder.preflight.stableModel) : updatedAgentSoulConfig - const configSkills = [{ - ...skill, - file_kind: skill.file_kind ?? 'tool_file', - }] + const configSkills = [skill] await saveAgentComposerDraft(getCurrentAgentId(this), { ...normalConfig, diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index b559d0a3413a20..856a986615cb57 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -25,7 +25,7 @@ Given('an Agent v2 test agent has been created via API', async function (this: D const agent = await createTestAgent() this.createdAgentIds.push(agent.id) this.lastCreatedAgentName = agent.name - this.lastCreatedAgentRole = agent.role + this.lastCreatedAgentRole = agent.role ?? undefined }) Given( @@ -34,7 +34,7 @@ Given( const agent = await createConfiguredTestAgent() this.createdAgentIds.push(agent.id) this.lastCreatedAgentName = agent.name - this.lastCreatedAgentRole = agent.role + this.lastCreatedAgentRole = agent.role ?? undefined }, ) @@ -50,7 +50,7 @@ Given('a runnable Agent v2 test agent has been created via API', async function }) this.createdAgentIds.push(agent.id) this.lastCreatedAgentName = agent.name - this.lastCreatedAgentRole = agent.role + this.lastCreatedAgentRole = agent.role ?? undefined }) Given('a minimal Agent v2 composer draft has been synced', async function (this: DifyWorld) { diff --git a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts index 44a3125ac42d3f..23e6269ee665e1 100644 --- a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts +++ b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts @@ -55,7 +55,7 @@ Given( }) this.createdAgentIds.push(agent.id) this.lastCreatedAgentName = agent.name - this.lastCreatedAgentRole = agent.role + this.lastCreatedAgentRole = agent.role ?? undefined const app = await createTestApp(createE2EResourceName('App', 'workflow-agent-v2'), 'workflow') this.createdAppIds.push(app.id) diff --git a/e2e/package.json b/e2e/package.json index d435754376765b..9210fc6a81b75f 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -15,6 +15,7 @@ }, "devDependencies": { "@cucumber/cucumber": "catalog:", + "@dify/contracts": "workspace:*", "@dify/tsconfig": "workspace:*", "@playwright/test": "catalog:", "@types/node": "catalog:", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 21cf3372f0c103..91e4c3d8e8a9e4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -763,6 +763,9 @@ importers: '@cucumber/cucumber': specifier: 'catalog:' version: 13.0.0 + '@dify/contracts': + specifier: workspace:* + version: link:../packages/contracts '@dify/tsconfig': specifier: workspace:* version: link:../packages/tsconfig From 8f4155d9c099acd8932f6b73db227aa226813c8f Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 21:33:17 +0800 Subject: [PATCH 091/185] test(e2e): cover empty agent file upload --- e2e/features/agent-v2/files.feature | 12 ++++++++++ .../agent-v2/configure-helpers.ts | 23 +++++++++++++++---- .../step-definitions/agent-v2/files.steps.ts | 15 ++++++++++++ 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/e2e/features/agent-v2/files.feature b/e2e/features/agent-v2/files.feature index 01f1862283683d..77e6f07a74c348 100644 --- a/e2e/features/agent-v2/files.feature +++ b/e2e/features/agent-v2/files.feature @@ -12,6 +12,18 @@ Feature: Agent v2 files When I refresh the current page Then I should see the small Agent v2 file in the Files section + Scenario: Uploading an empty file keeps a zero-byte file in the Agent configuration + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API + When I open the Agent v2 configure page + And I upload the empty Agent v2 file from the Files section + Then I should see the empty Agent v2 file in the Files section + And the empty Agent v2 file should be saved as a zero-byte file in the Agent v2 draft + And the Agent v2 configuration should be saved automatically + When I refresh the current page + Then I should see the empty Agent v2 file in the Files section + Scenario: Uploading a special-name file keeps the filename readable Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available diff --git a/e2e/features/step-definitions/agent-v2/configure-helpers.ts b/e2e/features/step-definitions/agent-v2/configure-helpers.ts index d5b78834071a1f..d6a77db353796c 100644 --- a/e2e/features/step-definitions/agent-v2/configure-helpers.ts +++ b/e2e/features/step-definitions/agent-v2/configure-helpers.ts @@ -112,17 +112,32 @@ export const expectAgentConfigFileHidden = async ( export const expectAgentConfigFileSaved = async ( world: DifyWorld, material: keyof typeof agentBuilderTestMaterials, + options?: { + size?: number + }, ) => { const agentId = getCurrentAgentId(world) const fileName = agentBuilderTestMaterials[material] await expect - .poll(async () => ( - await getAgentComposerDraft(agentId) - ).agent_soul?.config_files?.map(file => file.name) ?? [], { + .poll(async () => { + const file = (await getAgentComposerDraft(agentId)).agent_soul?.config_files?.find( + file => file.name === fileName, + ) + + return file + ? { + name: file.name, + size: file.size, + } + : undefined + }, { timeout: 30_000, }) - .toContain(fileName) + .toEqual({ + name: fileName, + size: options?.size ?? expect.anything(), + }) } export const uploadSummaryConfigSkillForBuildDraft = async (world: DifyWorld) => { diff --git a/e2e/features/step-definitions/agent-v2/files.steps.ts b/e2e/features/step-definitions/agent-v2/files.steps.ts index 990e4335ae2047..f8f6fefaf78414 100644 --- a/e2e/features/step-definitions/agent-v2/files.steps.ts +++ b/e2e/features/step-definitions/agent-v2/files.steps.ts @@ -13,6 +13,10 @@ When('I upload the small Agent v2 file from the Files section', async function ( await uploadAgentConfigFile(this, 'smallFile') }) +When('I upload the empty Agent v2 file from the Files section', async function (this: DifyWorld) { + await uploadAgentConfigFile(this, 'emptyFile') +}) + When('I upload the special-name Agent v2 file from the Files section', async function (this: DifyWorld) { await uploadAgentConfigFile(this, 'specialFilename') }) @@ -45,6 +49,10 @@ Then('I should see the small Agent v2 file in the Files section', async function await expectAgentConfigFileVisible(this, 'smallFile') }) +Then('I should see the empty Agent v2 file in the Files section', async function (this: DifyWorld) { + await expectAgentConfigFileVisible(this, 'emptyFile') +}) + Then('I should not see the small Agent v2 file in the Files section', async function (this: DifyWorld) { await expectAgentConfigFileHidden(this, 'smallFile') }) @@ -59,6 +67,13 @@ Then( }, ) +Then( + 'the empty Agent v2 file should be saved as a zero-byte file in the Agent v2 draft', + async function (this: DifyWorld) { + await expectAgentConfigFileSaved(this, 'emptyFile', { size: 0 }) + }, +) + Then( 'the special-name Agent v2 file should be saved in the Agent v2 draft', async function (this: DifyWorld) { From 5c68a165e9f8ef52d5c7bc7e67eaa40e13a4ecca Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 21:37:14 +0800 Subject: [PATCH 092/185] test(e2e): cover workflow agent console navigation --- e2e/features/agent-v2/agent-edit.feature | 11 +++ .../agent-v2/workflow-node.steps.ts | 77 +++++++++++++++++++ e2e/features/support/world.ts | 1 + 3 files changed, 89 insertions(+) diff --git a/e2e/features/agent-v2/agent-edit.feature b/e2e/features/agent-v2/agent-edit.feature index 5dfcbf52d0c150..589cdd127d53df 100644 --- a/e2e/features/agent-v2/agent-edit.feature +++ b/e2e/features/agent-v2/agent-edit.feature @@ -29,3 +29,14 @@ Feature: Agent v2 Agent Edit page And the Agent Builder preseeded Agent "E2E Agent With Dual Retrieval" includes the dual retrieval fixture configuration When I open the preseeded Agent v2 configure page for "E2E Agent With Dual Retrieval" from the Agent Roster Then I should see the Agent v2 dual retrieval fixture settings + + Scenario: Workflow Agent edit opens the same Agent in Agent Console + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a workflow app with an Agent v2 node has been created via API + When I open the app from the app list + And I open the Agent v2 workflow node panel + And I open the Agent v2 workflow Agent details + Then I should see the Agent v2 workflow Agent details for the created Agent + When I open the Agent v2 workflow Agent in Agent Console + Then the Agent v2 Agent Console should open for the same workflow Agent diff --git a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts index 23e6269ee665e1..5c59dc07b27f2a 100644 --- a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts +++ b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts @@ -11,6 +11,7 @@ import { createE2EResourceName } from '../../../support/naming' import { createAgentSoulConfigWithModel, createConfiguredTestAgent, + normalAgentPrompt, normalAgentSoulConfig, } from '../../agent-v2/support/agent' @@ -75,6 +76,31 @@ When('I open the Agent v2 workflow node panel', async function (this: DifyWorld) await expect(page.getByRole('button', { name: 'Output Variables' })).toBeVisible() }) +When('I open the Agent v2 workflow Agent details', async function (this: DifyWorld) { + const page = this.getPage() + const agentName = this.lastCreatedAgentName + if (!agentName) + throw new Error('No Agent v2 name found. Create a workflow Agent v2 node first.') + + await page.getByRole('button', { name: `Open ${agentName} details` }).click() + await expect(page.getByRole('dialog', { name: `${agentName} details` })).toBeVisible() +}) + +When('I open the Agent v2 workflow Agent in Agent Console', async function (this: DifyWorld) { + const page = this.getPage() + const agentName = this.lastCreatedAgentName + if (!agentName) + throw new Error('No Agent v2 name found. Create a workflow Agent v2 node first.') + + const detailsDialog = page.getByRole('dialog', { name: `${agentName} details` }) + const [agentConsolePage] = await Promise.all([ + page.waitForEvent('popup'), + detailsDialog.getByRole('link', { name: 'Edit in Agent Console' }).click(), + ]) + + this.agentBuilder.workflow.agentConsolePage = agentConsolePage +}) + When( 'I add these Agent v2 workflow node output variables', async function (this: DifyWorld, table: DataTable) { @@ -107,6 +133,57 @@ When( }, ) +Then( + 'I should see the Agent v2 workflow Agent details for the created Agent', + async function (this: DifyWorld) { + const page = this.getPage() + const agentName = this.lastCreatedAgentName + const agentRole = this.lastCreatedAgentRole + if (!agentName) + throw new Error('No Agent v2 name found. Create a workflow Agent v2 node first.') + + const detailsDialog = page.getByRole('dialog', { name: `${agentName} details` }) + + await expect(detailsDialog).toBeVisible() + await expect(detailsDialog.getByText(agentName, { exact: true })).toBeVisible() + if (agentRole) + await expect(detailsDialog.getByText(agentRole, { exact: true })).toBeVisible() + await expect(detailsDialog.getByRole('link', { name: 'Edit in Agent Console' })).toHaveAttribute( + 'href', + `/roster/agent/${this.createdAgentIds.at(-1)}/configure`, + ) + }, +) + +Then( + 'the Agent v2 Agent Console should open for the same workflow Agent', + async function (this: DifyWorld) { + const agentConsolePage = this.agentBuilder.workflow.agentConsolePage + const agentId = this.createdAgentIds.at(-1) + const agentName = this.lastCreatedAgentName + const stableModel = this.agentBuilder.preflight.stableModel + if (!agentConsolePage) + throw new Error('Agent Console page was not opened.') + if (!agentId || !agentName) + throw new Error('No Agent v2 ID or name found. Create a workflow Agent v2 node first.') + if (!stableModel) + throw new Error('Stable chat model preflight must run before asserting Agent Console.') + + await expect(agentConsolePage).toHaveURL( + new RegExp(`/roster/agent/${agentId}/configure(?:\\?.*)?$`), + ) + await expect(agentConsolePage.getByRole('heading', { name: 'Configure' })).toBeVisible({ + timeout: 30_000, + }) + await expect(agentConsolePage.getByText(agentName, { exact: true })).toBeVisible() + await expect(agentConsolePage.getByText(stableModel.name, { exact: true })).toBeVisible() + await expect(agentConsolePage.getByText(normalAgentPrompt)).toBeVisible() + + await agentConsolePage.close() + this.agentBuilder.workflow.agentConsolePage = undefined + }, +) + Then( 'the Agent v2 workflow node output variables should be saved in the workflow draft', async function (this: DifyWorld) { diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index 389e9ce88133b6..629f94cf6e9343 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -53,6 +53,7 @@ export const createAgentBuilderWorldState = () => ({ workflowReferencePage: undefined as Page | undefined, }, workflow: { + agentConsolePage: undefined as Page | undefined, outputVariables: [] as AgentV2WorkflowOutputVariable[], }, }) From ab5763469bd760ca146e440e6be8bd6297fd1db0 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 21:45:44 +0800 Subject: [PATCH 093/185] test(e2e): cover missing agent tool search --- e2e/features/agent-v2/AGENTS.md | 1 + e2e/features/agent-v2/tools.feature | 10 ++++ .../step-definitions/agent-v2/tools.steps.ts | 48 +++++++++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 e2e/features/agent-v2/tools.feature create mode 100644 e2e/features/step-definitions/agent-v2/tools.steps.ts diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 9e342a75fb7342..7d2fc5b7de5773 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -34,6 +34,7 @@ Keep Agent v2 step definitions grouped by user capability, not by DOM component - `configure.steps.ts` — common configure navigation, refresh, autosave, and normal draft assertions. - `build-draft.steps.ts` — Build mode checkout, apply, discard, supported writeback, and Build draft isolation. - `files.steps.ts` — Files upload, display, and fixture-list assertions. +- `tools.steps.ts` — Tools selector, search, and configuration-boundary behavior. - `advanced-settings.steps.ts` — Env Editor, Content Moderation, and Advanced Settings behavior. - `agent-edit.steps.ts` — saved Agent detail display assertions. - `publish.steps.ts` — publish and publish-bar assertions. diff --git a/e2e/features/agent-v2/tools.feature b/e2e/features/agent-v2/tools.feature new file mode 100644 index 00000000000000..7b1cc73f9c7eaa --- /dev/null +++ b/e2e/features/agent-v2/tools.feature @@ -0,0 +1,10 @@ +@agent-v2 @authenticated @tools @core +Feature: Agent v2 tools + Scenario: Tool selector shows an empty state for a missing tool search + Given I am signed in as the default E2E admin + And a basic configured Agent v2 test agent has been created via API + When I open the Agent v2 configure page + And I search for the missing Agent v2 tool from the Tools selector + Then I should see the Agent v2 tool selector empty state + When I clear the Agent v2 tool selector search + Then I should see the Agent v2 tool selector ready for another search diff --git a/e2e/features/step-definitions/agent-v2/tools.steps.ts b/e2e/features/step-definitions/agent-v2/tools.steps.ts new file mode 100644 index 00000000000000..02e56d23a6fbe7 --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/tools.steps.ts @@ -0,0 +1,48 @@ +import type { DifyWorld } from '../../support/world' +import { Then, When } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { agentBuilderFixedInputs } from '../../agent-v2/support/agent-builder-resources' + +const getToolsSection = (world: DifyWorld) => + world.getPage().getByRole('region', { name: 'Tools' }) + +const getToolSelectorSearch = (world: DifyWorld) => + world.getPage().getByRole('textbox', { name: 'Search integrations...' }) + +When( + 'I search for the missing Agent v2 tool from the Tools selector', + async function (this: DifyWorld) { + const toolsSection = getToolsSection(this) + + await expect(toolsSection).toBeVisible({ timeout: 30_000 }) + await toolsSection.getByRole('button', { name: 'Add tool' }).click() + await this.getPage().getByRole('button', { name: /^Tool\b/ }).click() + + const search = getToolSelectorSearch(this) + await expect(search).toBeVisible() + await search.fill(agentBuilderFixedInputs.missingToolSearchWithSuffix) + }, +) + +When('I clear the Agent v2 tool selector search', async function (this: DifyWorld) { + const search = getToolSelectorSearch(this) + + await search.fill('') +}) + +Then('I should see the Agent v2 tool selector empty state', async function (this: DifyWorld) { + const page = this.getPage() + + await expect(page.getByText('No integrations were found')).toBeVisible({ timeout: 30_000 }) + await expect(page.getByRole('link', { name: 'Requests to the community' })).toBeVisible() + await expect(page.getByText(agentBuilderFixedInputs.missingToolSearchWithSuffix)).not.toBeVisible() +}) + +Then('I should see the Agent v2 tool selector ready for another search', async function (this: DifyWorld) { + const page = this.getPage() + const search = getToolSelectorSearch(this) + + await expect(search).toHaveValue('') + await expect(page.getByText('No integrations were found')).not.toBeVisible() + await expect(page.getByText('All tools')).toBeVisible() +}) From 119fac3077e07123f69972dcc37be8f7af4a99ea Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 21:50:38 +0800 Subject: [PATCH 094/185] test(e2e): tighten agent contract type boundary --- e2e/features/agent-v2/support/agent.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/e2e/features/agent-v2/support/agent.ts b/e2e/features/agent-v2/support/agent.ts index 7c3cefd29274df..b23f57228edc16 100644 --- a/e2e/features/agent-v2/support/agent.ts +++ b/e2e/features/agent-v2/support/agent.ts @@ -37,8 +37,6 @@ export type AgentSeed = Pick< active_config_snapshot_id?: string | null } -export type { AgentSoulConfig } - export type AgentComposerEnvVariable = NonNullable< NonNullable['variables'] >[number] From d1f21b2986aebc667b840577ae4c45cfd8fe48cd Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 21:55:02 +0800 Subject: [PATCH 095/185] test(e2e): cover configure leave autosave protection --- e2e/features/agent-v2/configure-persistence.feature | 12 ++++++++++++ .../step-definitions/agent-v2/configure.steps.ts | 7 +++++++ 2 files changed, 19 insertions(+) diff --git a/e2e/features/agent-v2/configure-persistence.feature b/e2e/features/agent-v2/configure-persistence.feature index b0053eff45a023..a5e8038397b685 100644 --- a/e2e/features/agent-v2/configure-persistence.feature +++ b/e2e/features/agent-v2/configure-persistence.feature @@ -14,3 +14,15 @@ Feature: Agent v2 configure persistence Then I should see the normal E2E prompt in the Agent v2 prompt editor And I should see the stable E2E model in the Agent v2 model selector And the Agent v2 draft should use the stable E2E model + + @configure-persistence + Scenario: Leaving Configure before autosave completes preserves prompt changes + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API + When I open the Agent v2 configure page + And I fill the Agent v2 prompt editor with the updated E2E prompt + And I leave the Agent v2 configure page before autosave completes + When I open the Agent v2 configure page from the Agent Roster + Then I should see the updated E2E prompt in the Agent v2 prompt editor + And the normal Agent v2 draft should use the updated E2E prompt diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 856a986615cb57..da923de58fd32a 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -90,6 +90,13 @@ When('I switch to the Agent v2 Configure section', async function (this: DifyWor await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) }) +When('I leave the Agent v2 configure page before autosave completes', async function (this: DifyWorld) { + const page = this.getPage() + + await page.goto('/roster') + await expect(page).toHaveURL(/\/roster(?:\?.*)?$/) +}) + When('I open the Agent v2 configure page from the Agent Roster', async function (this: DifyWorld) { const page = this.getPage() const agentId = getCurrentAgentId(this) From 3116ca39a01915bd3804fc2b42983974c2ea7c5b Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 22:04:12 +0800 Subject: [PATCH 096/185] test(e2e): standardize agent v2 blocked messages --- e2e/features/agent-v2/support/preflight/common.ts | 13 +++++++++++-- .../agent-v2/advanced-settings.steps.ts | 6 +++++- .../step-definitions/agent-v2/build-draft.steps.ts | 4 ++++ .../agent-v2/output-variables.steps.ts | 4 ++++ 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/e2e/features/agent-v2/support/preflight/common.ts b/e2e/features/agent-v2/support/preflight/common.ts index ea9ed1236d22fe..5334b8601e7511 100644 --- a/e2e/features/agent-v2/support/preflight/common.ts +++ b/e2e/features/agent-v2/support/preflight/common.ts @@ -44,8 +44,17 @@ export const readRequiredEnvResource = ( } } -export function skipBlockedPrecondition(world: DifyWorld, reason: string): 'skipped' { - const message = `Blocked precondition: ${reason}` +export function skipBlockedPrecondition( + world: DifyWorld, + reason: string, + options: { + owner?: string + remediation?: string + } = {}, +): 'skipped' { + const owner = options.owner ?? 'seed/product' + const remediation = options.remediation ?? 'Seed the required resource or align the product capability before running this scenario.' + const message = `Blocked precondition: ${reason} Owner: ${owner}. Remediation: ${remediation}` console.warn(`[e2e] ${message}`) world.attach(message, 'text/plain') return 'skipped' diff --git a/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts b/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts index 530d2db5839378..8f76ee64d9784e 100644 --- a/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts +++ b/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts @@ -322,7 +322,11 @@ Then('Agent v2 Content Moderation Settings should be available', async function catch { return skipBlockedPrecondition( this, - 'Agent v2 Content Moderation Settings is not available in this build. Owner: product. Remediation: enable ENABLE_AGENT_CONTENT_MODERATION or keep this scenario feature-gated.', + 'Agent v2 Content Moderation Settings is not available in this build.', + { + owner: 'product', + remediation: 'Enable ENABLE_AGENT_CONTENT_MODERATION or keep this scenario feature-gated.', + }, ) } }) diff --git a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts index 3ac0b05ce920d6..572cb4145b0b72 100644 --- a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts +++ b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts @@ -161,6 +161,10 @@ Then('Agent v2 Build chat Dify Tool writeback should be available', async functi return skipBlockedPrecondition( this, 'Build draft Dify Tool writeback is not available: Build draft currently supports files, skills, and env only.', + { + owner: 'product', + remediation: 'Define and implement Build draft Tool writeback before enabling this scenario.', + }, ) }) diff --git a/e2e/features/step-definitions/agent-v2/output-variables.steps.ts b/e2e/features/step-definitions/agent-v2/output-variables.steps.ts index 134219814f22a2..31e2564416933b 100644 --- a/e2e/features/step-definitions/agent-v2/output-variables.steps.ts +++ b/e2e/features/step-definitions/agent-v2/output-variables.steps.ts @@ -11,5 +11,9 @@ Then('Agent v2 standalone Output Variables should be available', async function return skipBlockedPrecondition( this, 'Standalone Agent Output Variables are not available: output variables currently belong to Workflow Agent v2 nodes.', + { + owner: 'product', + remediation: 'Expose standalone Agent Output Variables or keep this scenario excluded until the product path exists.', + }, ) }) From 9ff5f79418023001dc424eb7ee4922cda79721f4 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 22:08:09 +0800 Subject: [PATCH 097/185] test(e2e): tighten agent edit console navigation --- e2e/features/agent-v2/agent-edit.feature | 2 +- .../step-definitions/agent-v2/workflow-node.steps.ts | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/e2e/features/agent-v2/agent-edit.feature b/e2e/features/agent-v2/agent-edit.feature index 589cdd127d53df..ce440219191c75 100644 --- a/e2e/features/agent-v2/agent-edit.feature +++ b/e2e/features/agent-v2/agent-edit.feature @@ -30,7 +30,7 @@ Feature: Agent v2 Agent Edit page When I open the preseeded Agent v2 configure page for "E2E Agent With Dual Retrieval" from the Agent Roster Then I should see the Agent v2 dual retrieval fixture settings - Scenario: Workflow Agent edit opens the same Agent in Agent Console + Scenario: Agent Edit opens the same Agent in Agent Console Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available And a workflow app with an Agent v2 node has been created via API diff --git a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts index 5c59dc07b27f2a..1406a5fe683f4d 100644 --- a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts +++ b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts @@ -139,8 +139,11 @@ Then( const page = this.getPage() const agentName = this.lastCreatedAgentName const agentRole = this.lastCreatedAgentRole + const stableModel = this.agentBuilder.preflight.stableModel if (!agentName) throw new Error('No Agent v2 name found. Create a workflow Agent v2 node first.') + if (!stableModel) + throw new Error('Stable chat model preflight must run before asserting workflow Agent details.') const detailsDialog = page.getByRole('dialog', { name: `${agentName} details` }) @@ -148,6 +151,8 @@ Then( await expect(detailsDialog.getByText(agentName, { exact: true })).toBeVisible() if (agentRole) await expect(detailsDialog.getByText(agentRole, { exact: true })).toBeVisible() + await expect(detailsDialog.getByText(stableModel.name, { exact: true })).toBeVisible() + await expect(detailsDialog.getByText(normalAgentPrompt)).toBeVisible() await expect(detailsDialog.getByRole('link', { name: 'Edit in Agent Console' })).toHaveAttribute( 'href', `/roster/agent/${this.createdAgentIds.at(-1)}/configure`, From 0c3a4d4695b3d7baa1d120f5175f2b557ea253f9 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 22:14:01 +0800 Subject: [PATCH 098/185] test(e2e): cover agent web app publish runtime --- e2e/features/agent-v2/agent-edit.feature | 8 +++ e2e/features/agent-v2/publish.feature | 18 +++++++ .../agent-v2/access-point.steps.ts | 53 ++++++++++++++++++- .../agent-v2/agent-edit.steps.ts | 15 ++++++ 4 files changed, 93 insertions(+), 1 deletion(-) diff --git a/e2e/features/agent-v2/agent-edit.feature b/e2e/features/agent-v2/agent-edit.feature index ce440219191c75..453015ba3b5027 100644 --- a/e2e/features/agent-v2/agent-edit.feature +++ b/e2e/features/agent-v2/agent-edit.feature @@ -15,6 +15,14 @@ Feature: Agent v2 Agent Edit page When I open the preseeded Agent v2 configure page for "E2E New Agent Builder Tool States" from the Agent Roster Then I should see the Agent v2 tool state fixture tools + @tool-error-state @feature-gated + Scenario: Tool credential error states are visible on the Agent Edit page + Given I am signed in as the default E2E admin + And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" is available + And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" includes the tool state fixture configuration + When I open the preseeded Agent v2 configure page for "E2E New Agent Builder Tool States" from the Agent Roster + Then Agent v2 Tool credential error state should be available + Scenario: File fixture entries are visible in the current flat Files list Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E Agent With File Tree" is available diff --git a/e2e/features/agent-v2/publish.feature b/e2e/features/agent-v2/publish.feature index 7cd901de4c5101..4592d6c0e7f7ce 100644 --- a/e2e/features/agent-v2/publish.feature +++ b/e2e/features/agent-v2/publish.feature @@ -32,3 +32,21 @@ Feature: Agent v2 publish Then the Agent v2 configuration should be saved automatically And the normal Agent v2 draft should use the updated E2E prompt And the active published Agent v2 version should still use the normal E2E prompt + + @web-app-runtime + Scenario: Published Web app uses the latest Agent v2 published configuration + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API + When I open the Agent v2 configure page + And I publish the Agent v2 draft + Then the Agent v2 draft should be published and up to date + When I fill the Agent v2 prompt editor with the updated E2E prompt + Then the Agent v2 configuration should be saved automatically + And the normal Agent v2 draft should use the updated E2E prompt + When I publish the Agent v2 draft + Then the Agent v2 draft should be published and up to date + When Agent v2 Web app access has been enabled via API + And I open the Agent v2 Web app URL + And I send an E2E message in the Agent v2 Web app + Then the Agent v2 Web app response should include the updated E2E marker diff --git a/e2e/features/step-definitions/agent-v2/access-point.steps.ts b/e2e/features/step-definitions/agent-v2/access-point.steps.ts index f9ac836119fd96..3f8672e63a2596 100644 --- a/e2e/features/step-definitions/agent-v2/access-point.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point.steps.ts @@ -8,7 +8,10 @@ import { setAgentApiAccess, setAgentSiteAccessAndGetURL, } from '../../agent-v2/support/agent' -import { agentBuilderPreseededResources } from '../../agent-v2/support/agent-builder-resources' +import { + agentBuilderExpectedTokens, + agentBuilderPreseededResources, +} from '../../agent-v2/support/agent-builder-resources' const getCurrentAgentId = (world: DifyWorld) => { const agentId = world.createdAgentIds.at(-1) @@ -65,6 +68,16 @@ Given( }, ) +When( + 'Agent v2 Web app access has been enabled via API', + async function (this: DifyWorld) { + this.agentBuilder.accessPoint.webAppURL = await setAgentSiteAccessAndGetURL( + getCurrentAgentId(this), + true, + ) + }, +) + When('I open the Agent v2 Access Point page', async function (this: DifyWorld) { await this.getPage().goto(getAgentAccessPath(getCurrentAgentId(this))) }) @@ -167,6 +180,30 @@ When('I launch the Agent v2 Web app', async function (this: DifyWorld) { this.agentBuilder.accessPoint.webAppPage = webAppPage }) +When('I open the Agent v2 Web app URL', async function (this: DifyWorld) { + const webAppURL = this.agentBuilder.accessPoint.webAppURL + if (!webAppURL) + throw new Error('No Agent v2 Web app URL was recorded.') + if (!this.context) + throw new Error('Playwright browser context has not been initialized.') + + const webAppPage = await this.context.newPage() + await webAppPage.goto(webAppURL) + + this.agentBuilder.accessPoint.webAppPage = webAppPage +}) + +When('I send an E2E message in the Agent v2 Web app', async function (this: DifyWorld) { + const webAppPage = this.agentBuilder.accessPoint.webAppPage + if (!webAppPage) + throw new Error('No Agent v2 Web app page was opened.') + + const messageInput = webAppPage.getByRole('textbox').last() + await expect(messageInput).toBeEditable({ timeout: 30_000 }) + await messageInput.fill('Please reply with the test success marker.') + await messageInput.press('Enter') +}) + Then('the Agent v2 Web app should open in a new tab', async function (this: DifyWorld) { const webAppPage = this.agentBuilder.accessPoint.webAppPage const webAppURL = this.agentBuilder.accessPoint.webAppURL @@ -179,6 +216,20 @@ Then('the Agent v2 Web app should open in a new tab', async function (this: Dify this.agentBuilder.accessPoint.webAppURL = undefined }) +Then( + 'the Agent v2 Web app response should include the updated E2E marker', + async function (this: DifyWorld) { + const webAppPage = this.agentBuilder.accessPoint.webAppPage + if (!webAppPage) + throw new Error('No Agent v2 Web app page was opened.') + + await expect(webAppPage.getByText(agentBuilderExpectedTokens.updatedAgentReply)) + .toBeVisible({ timeout: 120_000 }) + await webAppPage.close() + this.agentBuilder.accessPoint.webAppPage = undefined + }, +) + When('I open Agent v2 Embedded configuration', async function (this: DifyWorld) { await getWebAppCard(this).getByRole('button', { name: 'Embedded' }).click() }) diff --git a/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts b/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts index 9f1a2a9a845e20..a2732947d6158c 100644 --- a/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts +++ b/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts @@ -2,6 +2,7 @@ import type { DifyWorld } from '../../support/world' import { Then } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { agentBuilderExpectedTokens, agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../agent-v2/support/agent-builder-resources' +import { skipBlockedPrecondition } from '../../agent-v2/support/preflight/common' import { agentBuilderTestMaterials } from '../../agent-v2/support/test-materials' import { expectProviderToolActionVisible, openAgentKnowledgeRetrievalDialog } from './configure-helpers' @@ -83,6 +84,20 @@ Then('I should see the Agent v2 tool state fixture tools', async function (this: ) }) +Then('Agent v2 Tool credential error state should be available', async function (this: DifyWorld) { + const toolsSection = this.getPage().getByRole('region', { name: 'Tools' }) + + await expect(toolsSection).toBeVisible({ timeout: 30_000 }) + return skipBlockedPrecondition( + this, + 'Agent v2 Tool credential error state is not covered: the current fixture only proves usable and not-authorized tool states.', + { + owner: 'seed/product', + remediation: 'Define a stable invalid credential fixture and the expected user-visible error label before enabling this scenario.', + }, + ) +}) + Then('I should see the Agent v2 dual retrieval fixture settings', async function (this: DifyWorld) { const page = this.getPage() const knowledgeSection = page.getByRole('region', { name: 'Knowledge Retrieval' }) From 2e5aec22fd793282c6a9a188a5baddf216ba979a Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 22:20:50 +0800 Subject: [PATCH 099/185] test(e2e): cover agent duplicate lifecycle --- e2e/features/agent-v2/agent-edit.feature | 13 ++ .../agent-v2/agent-edit.steps.ts | 153 +++++++++++++++++- 2 files changed, 163 insertions(+), 3 deletions(-) diff --git a/e2e/features/agent-v2/agent-edit.feature b/e2e/features/agent-v2/agent-edit.feature index 453015ba3b5027..b7789d69869a64 100644 --- a/e2e/features/agent-v2/agent-edit.feature +++ b/e2e/features/agent-v2/agent-edit.feature @@ -8,6 +8,19 @@ Feature: Agent v2 Agent Edit page When I open the preseeded Agent v2 configure page for "E2E New Agent Builder Full Config" from the Agent Roster Then I should see the Agent v2 full-config fixture sections + Scenario: Duplicated Agent inherits configuration without changing the original Agent + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" is available + And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" includes the core fixture configuration + When I duplicate the preseeded Agent v2 "E2E New Agent Builder Full Config" from the Agent Roster + Then the duplicated Agent v2 should inherit the full-config fixture from "E2E New Agent Builder Full Config" + When I open the Agent v2 configure page + And I fill the Agent v2 prompt editor with the updated E2E prompt + Then the Agent v2 configuration should be saved automatically + And the normal Agent v2 draft should use the updated E2E prompt + And the preseeded Agent v2 "E2E New Agent Builder Full Config" should still use the normal E2E prompt + Scenario: Tool states are visible on the Agent Edit page Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" is available diff --git a/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts b/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts index a2732947d6158c..e1fe3323495e0e 100644 --- a/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts +++ b/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts @@ -1,10 +1,105 @@ +import type { PostAgentByAgentIdCopyResponse } from '@dify/contracts/api/console/agent/types.gen' import type { DifyWorld } from '../../support/world' -import { Then } from '@cucumber/cucumber' +import { Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' +import { createE2EResourceName } from '../../../support/naming' +import { + getAgentComposerDraft, + getTestAgent, + normalAgentPrompt, +} from '../../agent-v2/support/agent' import { agentBuilderExpectedTokens, agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../agent-v2/support/agent-builder-resources' -import { skipBlockedPrecondition } from '../../agent-v2/support/preflight/common' +import { + asArray, + asRecord, + asString, + skipBlockedPrecondition, +} from '../../agent-v2/support/preflight/common' import { agentBuilderTestMaterials } from '../../agent-v2/support/test-materials' -import { expectProviderToolActionVisible, openAgentKnowledgeRetrievalDialog } from './configure-helpers' +import { + expectProviderToolActionVisible, + getCurrentAgentId, + getPreseededAgent, + openAgentKnowledgeRetrievalDialog, +} from './configure-helpers' + +const getComposerInheritanceSnapshot = async (agentId: string) => { + const draft = await getAgentComposerDraft(agentId) + const soul = draft.agent_soul ?? {} + const model = asRecord(soul.model) + const prompt = asRecord(soul.prompt) + const files = asArray(asRecord(soul.files).files) + const tools = asArray(asRecord(soul.tools).dify_tools) + const knowledgeSets = asArray(asRecord(soul.knowledge).sets) + + return { + fileNames: files.map(file => asString(asRecord(file).name)).filter(Boolean).sort(), + knowledgeDatasetNames: knowledgeSets + .flatMap(set => asArray(asRecord(set).datasets)) + .map(dataset => asString(asRecord(dataset).name)) + .filter(Boolean) + .sort(), + model: { + name: asString(model.model), + provider: asString(model.model_provider), + }, + prompt: asString(prompt.system_prompt), + toolSignatures: tools + .map((tool) => { + const record = asRecord(tool) + const provider = asString(record.provider_id) + || asString(record.provider) + || asString(record.plugin_id) + || asString(record.name) + const toolName = asString(record.tool_name) || asString(record.name) + + return `${provider}/${toolName}` + }) + .filter(signature => signature !== '/') + .sort(), + } +} + +When( + 'I duplicate the preseeded Agent v2 {string} from the Agent Roster', + async function (this: DifyWorld, agentName: string) { + const page = this.getPage() + const agent = getPreseededAgent(this, agentName) + const copyName = createE2EResourceName('Agent', 'copy') + + await page.goto('/roster') + const card = page.locator('article').filter({ + has: page.getByRole('link', { name: agentName }), + }).first() + + await expect(card).toBeVisible({ timeout: 30_000 }) + await card.hover() + await card.getByLabel(`More actions for ${agentName}`).click() + await page.getByRole('menuitem', { name: 'Duplicate' }).click() + + const dialog = page.getByRole('dialog', { name: 'Duplicate agent' }) + await expect(dialog).toBeVisible() + await dialog.getByRole('textbox', { name: /Name/ }).fill(copyName) + + const copyResponsePromise = page.waitForResponse(response => ( + response.request().method() === 'POST' + && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agent.id}/copy`) + )) + await dialog.getByRole('button', { name: 'Duplicate' }).click() + + const copyResponse = await copyResponsePromise + expect(copyResponse.status()).toBe(201) + const copiedAgent = (await copyResponse.json()) as PostAgentByAgentIdCopyResponse + if (!copiedAgent.id) + throw new Error('Agent v2 duplicate response did not include a copied Agent ID.') + + this.createdAgentIds.push(copiedAgent.id) + this.lastCreatedAgentName = copiedAgent.name + this.lastCreatedAgentRole = copiedAgent.role ?? undefined + + await expect(page.getByText('Agent duplicated.')).toBeVisible() + }, +) Then('I should see the Agent v2 full-config fixture sections', async function (this: DifyWorld) { const page = this.getPage() @@ -57,6 +152,58 @@ Then('I should see the Agent v2 full-config fixture sections', async function (t ).toBeVisible() }) +Then( + 'the duplicated Agent v2 should inherit the full-config fixture from {string}', + async function (this: DifyWorld, agentName: string) { + const sourceAgent = getPreseededAgent(this, agentName) + const duplicatedAgentId = getCurrentAgentId(this) + const stableModel = this.agentBuilder.preflight.stableModel + if (!stableModel) + throw new Error('Stable chat model preflight must run before asserting the duplicated Agent.') + + const [sourceDetail, duplicatedDetail, sourceSnapshot, duplicatedSnapshot] = await Promise.all([ + getTestAgent(sourceAgent.id), + getTestAgent(duplicatedAgentId), + getComposerInheritanceSnapshot(sourceAgent.id), + getComposerInheritanceSnapshot(duplicatedAgentId), + ]) + + expect(duplicatedDetail.id).toBe(duplicatedAgentId) + expect(duplicatedDetail.name).toBe(this.lastCreatedAgentName) + expect(duplicatedDetail.active_config_is_published).toBe(sourceDetail.active_config_is_published) + expect(duplicatedSnapshot.model).toEqual({ + name: stableModel.name, + provider: stableModel.provider, + }) + expect(duplicatedSnapshot.model).toEqual(sourceSnapshot.model) + expect(duplicatedSnapshot.prompt).toBe(sourceSnapshot.prompt) + expect(duplicatedSnapshot.fileNames).toEqual(expect.arrayContaining([ + agentBuilderTestMaterials.smallFile, + agentBuilderTestMaterials.specialFilename, + ])) + expect(duplicatedSnapshot.toolSignatures).toEqual(sourceSnapshot.toolSignatures) + expect(duplicatedSnapshot.knowledgeDatasetNames).toEqual(expect.arrayContaining([ + agentBuilderPreseededResources.agentKnowledgeBase, + ])) + }, +) + +Then( + 'the preseeded Agent v2 {string} should still use the normal E2E prompt', + async function (this: DifyWorld, agentName: string) { + const sourceAgent = getPreseededAgent(this, agentName) + + await expect.poll( + async () => { + const draft = await getAgentComposerDraft(sourceAgent.id) + + return asString(asRecord(draft.agent_soul?.prompt).system_prompt) + }, + { timeout: 30_000 }, + ).toBe(normalAgentPrompt) + }, +) + Then('I should see the Agent v2 tool state fixture tools', async function (this: DifyWorld) { const page = this.getPage() const toolsSection = page.getByRole('region', { name: 'Tools' }) From f75dd9b19d1b9ea7482e4efe9f9a737061f44585 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 22:27:56 +0800 Subject: [PATCH 100/185] test(e2e): cover agent version restore --- e2e/features/agent-v2/publish.feature | 21 +++++++++++ .../agent-v2/configure.steps.ts | 10 +++--- .../agent-v2/publish.steps.ts | 36 +++++++++++++++++++ 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/e2e/features/agent-v2/publish.feature b/e2e/features/agent-v2/publish.feature index 4592d6c0e7f7ce..ab692647aaf222 100644 --- a/e2e/features/agent-v2/publish.feature +++ b/e2e/features/agent-v2/publish.feature @@ -33,6 +33,27 @@ Feature: Agent v2 publish And the normal Agent v2 draft should use the updated E2E prompt And the active published Agent v2 version should still use the normal E2E prompt + Scenario: Restoring a published Agent v2 version shows the restored configuration in Builder + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API + When I open the Agent v2 configure page + And I publish the Agent v2 draft + Then the Agent v2 draft should be published and up to date + When I fill the Agent v2 prompt editor with the updated E2E prompt + Then the Agent v2 configuration should be saved automatically + And the normal Agent v2 draft should use the updated E2E prompt + When I publish the Agent v2 draft + Then the Agent v2 draft should be published and up to date + When I open the Agent v2 version history + And I select Agent v2 published version 1 + Then the selected Agent v2 version should be displayed in view-only mode + And I should see the normal E2E prompt in the Agent v2 prompt editor + When I restore the selected Agent v2 version + Then I should see the normal E2E prompt in the Agent v2 prompt editor + And the normal Agent v2 draft should use the normal E2E prompt + And the Agent v2 publish action should be available for unpublished changes + @web-app-runtime Scenario: Published Web app uses the latest Agent v2 published configuration Given I am signed in as the default E2E admin diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index da923de58fd32a..bbb6fa1eab64b8 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -159,9 +159,10 @@ Then( 'I should see the normal E2E prompt in the Agent v2 prompt editor', async function (this: DifyWorld) { const page = this.getPage() + const promptSection = page.getByRole('region', { name: 'Prompt' }) - await expect(page.getByRole('heading', { name: 'Prompt' })).toBeVisible({ timeout: 30_000 }) - await expect(page.getByText(normalAgentPrompt)).toBeVisible() + await expect(promptSection).toBeVisible({ timeout: 30_000 }) + await expect(promptSection.getByRole('textbox', { name: 'Prompt' })).toContainText(normalAgentPrompt) }, ) @@ -181,9 +182,10 @@ Then( 'I should see the updated E2E prompt in the Agent v2 prompt editor', async function (this: DifyWorld) { const page = this.getPage() + const promptSection = page.getByRole('region', { name: 'Prompt' }) - await expect(page.getByRole('heading', { name: 'Prompt' })).toBeVisible({ timeout: 30_000 }) - await expect(page.getByText(updatedAgentPrompt)).toBeVisible() + await expect(promptSection).toBeVisible({ timeout: 30_000 }) + await expect(promptSection.getByRole('textbox', { name: 'Prompt' })).toContainText(updatedAgentPrompt) }, ) diff --git a/e2e/features/step-definitions/agent-v2/publish.steps.ts b/e2e/features/step-definitions/agent-v2/publish.steps.ts index a1a7b2abe274a5..e6f6cb2ee855a5 100644 --- a/e2e/features/step-definitions/agent-v2/publish.steps.ts +++ b/e2e/features/step-definitions/agent-v2/publish.steps.ts @@ -49,6 +49,42 @@ Then('the Agent v2 publish action should be unavailable while up to date', async await expect(page.getByRole('button', { name: 'Published' })).toBeDisabled() }) +When('I open the Agent v2 version history', async function (this: DifyWorld) { + const page = this.getPage() + + await page.getByRole('button', { name: 'Open version history' }).click() + await expect(page.getByRole('heading', { name: 'Versions' })).toBeVisible({ timeout: 30_000 }) +}) + +When('I select Agent v2 published version {int}', async function (this: DifyWorld, versionNumber: number) { + const page = this.getPage() + const versionButton = page.getByRole('button', { name: new RegExp(`\\bVersion ${versionNumber}\\b`) }) + + await expect(versionButton).toBeVisible({ timeout: 30_000 }) + await versionButton.click() +}) + +Then('the selected Agent v2 version should be displayed in view-only mode', async function (this: DifyWorld) { + const page = this.getPage() + + await expect(page.getByText('View Only')).toBeVisible({ timeout: 30_000 }) + await expect(page.getByRole('button', { name: 'Restore' })).toBeEnabled() +}) + +When('I restore the selected Agent v2 version', async function (this: DifyWorld) { + const page = this.getPage() + const agentId = getCurrentAgentId(this) + const restoreResponse = page.waitForResponse(response => + response.request().method() === 'POST' + && response.url().includes(`/console/api/agent/${agentId}/versions/`) + && response.url().endsWith('/restore'), + ) + + await page.getByRole('button', { name: 'Restore' }).click() + const response = await restoreResponse + expect(response.ok()).toBe(true) +}) + Then( 'the active published Agent v2 version should still use the normal E2E prompt', async function (this: DifyWorld) { From 558ada7a7425cdb3250adc7ec6895bc1bea42d01 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 22:33:18 +0800 Subject: [PATCH 101/185] test(e2e): cover concurrent agent edits --- e2e/features/agent-v2/AGENTS.md | 1 + .../agent-v2/configure-persistence.feature | 11 ++ e2e/features/agent-v2/support/agent.ts | 6 + .../agent-v2/configure.steps.ts | 130 ++++++++++++++++-- e2e/features/support/world.ts | 3 + 5 files changed, 136 insertions(+), 15 deletions(-) diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 7d2fc5b7de5773..335797c2659312 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -61,6 +61,7 @@ Use the existing namespace shape: - `world.agentBuilder.accessPoint.webAppURL` - `world.agentBuilder.accessPoint.workflowReferencePage` - `world.agentBuilder.accessPoint.composerDraftSnapshot` +- `world.agentBuilder.configure.concurrentPage` - `world.agentBuilder.workflow.outputVariables` Use `features/agent-v2/support/agent.ts` for Agent v2 API fixtures. It owns roster-shaped Agent IDs, configure/access route helpers, composer draft sync, build-draft helpers, publish, API access toggles, Agent drive file cleanup, and Agent cleanup. Store created roster Agent IDs in `DifyWorld.createdAgentIds`; the shared `After` hook deletes them after each scenario. diff --git a/e2e/features/agent-v2/configure-persistence.feature b/e2e/features/agent-v2/configure-persistence.feature index a5e8038397b685..90a5ea4f34eeda 100644 --- a/e2e/features/agent-v2/configure-persistence.feature +++ b/e2e/features/agent-v2/configure-persistence.feature @@ -26,3 +26,14 @@ Feature: Agent v2 configure persistence When I open the Agent v2 configure page from the Agent Roster Then I should see the updated E2E prompt in the Agent v2 prompt editor And the normal Agent v2 draft should use the updated E2E prompt + + @configure-persistence + Scenario: Concurrent Agent v2 edits converge to one clear saved draft after refresh + Given I am signed in as the default E2E admin + And a basic configured Agent v2 test agent has been created via API + When I open the Agent v2 configure page + And I open the same Agent v2 configure page in another tab + And I save the Agent v2 prompt from the first configure tab + And I save the Agent v2 prompt from the second configure tab + When I refresh both Agent v2 configure tabs + Then both Agent v2 configure tabs and the Agent v2 draft should show one saved concurrent prompt diff --git a/e2e/features/agent-v2/support/agent.ts b/e2e/features/agent-v2/support/agent.ts index b23f57228edc16..b116851ba71615 100644 --- a/e2e/features/agent-v2/support/agent.ts +++ b/e2e/features/agent-v2/support/agent.ts @@ -70,6 +70,12 @@ export const normalAgentPrompt export const updatedAgentPrompt = 'You are a Dify Agent E2E test assistant. Every response must start with E2E_AGENT_UPDATED.' +export const concurrentFirstAgentPrompt + = 'You are a Dify Agent E2E concurrent edit assistant. Always include E2E_CONCURRENT_FIRST in saved instructions.' + +export const concurrentSecondAgentPrompt + = 'You are a Dify Agent E2E concurrent edit assistant. Always include E2E_CONCURRENT_SECOND in saved instructions.' + export const normalAgentSoulConfig: AgentSoulConfig = { prompt: { system_prompt: normalAgentPrompt, diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index bbb6fa1eab64b8..d69e2aa6b47c34 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -1,7 +1,11 @@ +import type { Page } from '@playwright/test' import type { DifyWorld } from '../../support/world' import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' +import { waitForAgentConfigureAutosaved } from '../../../support/agent-configure' import { + concurrentFirstAgentPrompt, + concurrentSecondAgentPrompt, createAgentSoulConfigWithModel, createConfiguredTestAgent, createTestAgent, @@ -21,6 +25,42 @@ import { getPreseededAgent, } from './configure-helpers' +const concurrentAgentPrompts = [ + concurrentFirstAgentPrompt, + concurrentSecondAgentPrompt, +] + +function attachPageDiagnostics(world: DifyWorld, page: Page) { + page.setDefaultTimeout(30_000) + page.on('console', (message) => { + if (message.type() === 'error') + world.consoleErrors.push(message.text()) + }) + page.on('pageerror', (error) => { + world.pageErrors.push(error.message) + }) + page.on('download', (dl) => { + world.capturedDownloads.push(dl) + }) +} + +const getPromptEditor = (page: Page) => + page.getByRole('region', { name: 'Prompt' }).getByRole('textbox', { name: 'Prompt' }) + +async function fillAgentPromptEditor(page: Page, prompt: string) { + const promptSection = page.getByRole('region', { name: 'Prompt' }) + + await expect(promptSection).toBeVisible({ timeout: 30_000 }) + await getPromptEditor(page).fill(prompt) +} + +async function expectAgentComposerPrompt(agentId: string, prompt: string) { + await expect.poll( + async () => (await getAgentComposerDraft(agentId)).agent_soul?.prompt?.system_prompt, + { timeout: 30_000 }, + ).toBe(prompt) +} + Given('an Agent v2 test agent has been created via API', async function (this: DifyWorld) { const agent = await createTestAgent() this.createdAgentIds.push(agent.id) @@ -124,19 +164,58 @@ When( ) When('I fill the Agent v2 prompt editor with the normal E2E prompt', async function (this: DifyWorld) { - const page = this.getPage() - const promptSection = page.getByRole('region', { name: 'Prompt' }) - - await expect(promptSection).toBeVisible({ timeout: 30_000 }) - await promptSection.getByRole('textbox', { name: 'Prompt' }).fill(normalAgentPrompt) + await fillAgentPromptEditor(this.getPage(), normalAgentPrompt) }) When('I fill the Agent v2 prompt editor with the updated E2E prompt', async function (this: DifyWorld) { - const page = this.getPage() - const promptSection = page.getByRole('region', { name: 'Prompt' }) + await fillAgentPromptEditor(this.getPage(), updatedAgentPrompt) +}) - await expect(promptSection).toBeVisible({ timeout: 30_000 }) - await promptSection.getByRole('textbox', { name: 'Prompt' }).fill(updatedAgentPrompt) +When('I open the same Agent v2 configure page in another tab', async function (this: DifyWorld) { + if (!this.context) + throw new Error('Playwright context has not been initialized for this scenario.') + + const agentId = getCurrentAgentId(this) + const concurrentPage = await this.context.newPage() + attachPageDiagnostics(this, concurrentPage) + this.agentBuilder.configure.concurrentPage = concurrentPage + + await concurrentPage.goto(getAgentConfigurePath(agentId)) + await expect(concurrentPage).toHaveURL(new RegExp(`/roster/agent/${agentId}/configure(?:\\?.*)?$`)) + await expect(concurrentPage.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) +}) + +When('I save the Agent v2 prompt from the first configure tab', async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await fillAgentPromptEditor(this.getPage(), concurrentFirstAgentPrompt) + await waitForAgentConfigureAutosaved(this.getPage()) + await expectAgentComposerPrompt(agentId, concurrentFirstAgentPrompt) +}) + +When('I save the Agent v2 prompt from the second configure tab', async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + const concurrentPage = this.agentBuilder.configure.concurrentPage + if (!concurrentPage) + throw new Error('Open the same Agent v2 configure page in another tab before editing it.') + + await fillAgentPromptEditor(concurrentPage, concurrentSecondAgentPrompt) + await waitForAgentConfigureAutosaved(concurrentPage) + await expectAgentComposerPrompt(agentId, concurrentSecondAgentPrompt) +}) + +When('I refresh both Agent v2 configure tabs', async function (this: DifyWorld) { + const page = this.getPage() + const concurrentPage = this.agentBuilder.configure.concurrentPage + if (!concurrentPage) + throw new Error('Open the same Agent v2 configure page in another tab before refreshing it.') + + await Promise.all([ + page.reload(), + concurrentPage.reload(), + ]) + await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) + await expect(concurrentPage.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) }) Then('I should be on the Agent v2 configure page', async function (this: DifyWorld) { @@ -159,10 +238,8 @@ Then( 'I should see the normal E2E prompt in the Agent v2 prompt editor', async function (this: DifyWorld) { const page = this.getPage() - const promptSection = page.getByRole('region', { name: 'Prompt' }) - await expect(promptSection).toBeVisible({ timeout: 30_000 }) - await expect(promptSection.getByRole('textbox', { name: 'Prompt' })).toContainText(normalAgentPrompt) + await expect(getPromptEditor(page)).toContainText(normalAgentPrompt, { timeout: 30_000 }) }, ) @@ -182,10 +259,33 @@ Then( 'I should see the updated E2E prompt in the Agent v2 prompt editor', async function (this: DifyWorld) { const page = this.getPage() - const promptSection = page.getByRole('region', { name: 'Prompt' }) - await expect(promptSection).toBeVisible({ timeout: 30_000 }) - await expect(promptSection.getByRole('textbox', { name: 'Prompt' })).toContainText(updatedAgentPrompt) + await expect(getPromptEditor(page)).toContainText(updatedAgentPrompt, { timeout: 30_000 }) + }, +) + +Then( + 'both Agent v2 configure tabs and the Agent v2 draft should show one saved concurrent prompt', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + const concurrentPage = this.agentBuilder.configure.concurrentPage + if (!concurrentPage) + throw new Error('Open the same Agent v2 configure page in another tab before asserting convergence.') + + let savedPrompt = '' + await expect.poll( + async () => { + const prompt = (await getAgentComposerDraft(agentId)).agent_soul?.prompt?.system_prompt + if (prompt && concurrentAgentPrompts.includes(prompt)) + savedPrompt = prompt + + return !!savedPrompt + }, + { timeout: 30_000 }, + ).toBe(true) + + await expect(getPromptEditor(this.getPage())).toContainText(savedPrompt) + await expect(getPromptEditor(concurrentPage)).toContainText(savedPrompt) }, ) diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index 629f94cf6e9343..a8b11aeda56a5d 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -52,6 +52,9 @@ export const createAgentBuilderWorldState = () => ({ webAppURL: undefined as string | undefined, workflowReferencePage: undefined as Page | undefined, }, + configure: { + concurrentPage: undefined as Page | undefined, + }, workflow: { agentConsolePage: undefined as Page | undefined, outputVariables: [] as AgentV2WorkflowOutputVariable[], From c427f520b582d7dae4475d932f6a1206b15473b3 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 22:41:32 +0800 Subject: [PATCH 102/185] test(e2e): cover backend api access recovery --- e2e/features/agent-v2/AGENTS.md | 1 + e2e/features/agent-v2/access-point.feature | 17 ++++ e2e/features/agent-v2/support/agent.ts | 50 ++++++++++ .../agent-v2/access-point.steps.ts | 95 +++++++++++++++++-- e2e/features/support/world.ts | 1 + 5 files changed, 157 insertions(+), 7 deletions(-) diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 335797c2659312..c57a0cee802635 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -56,6 +56,7 @@ Use the existing namespace shape: - `world.agentBuilder.preflight.preseededResources` - `world.agentBuilder.accessPoint.serviceApiBaseURL` - `world.agentBuilder.accessPoint.generatedApiKey` +- `world.agentBuilder.accessPoint.serviceApiResponse` - `world.agentBuilder.accessPoint.apiReferencePage` - `world.agentBuilder.accessPoint.webAppPage` - `world.agentBuilder.accessPoint.webAppURL` diff --git a/e2e/features/agent-v2/access-point.feature b/e2e/features/agent-v2/access-point.feature index 85faef28d31aaa..f213270e07f310 100644 --- a/e2e/features/agent-v2/access-point.feature +++ b/e2e/features/agent-v2/access-point.feature @@ -74,3 +74,20 @@ Feature: Agent v2 Access Point When I close Agent v2 API key management And I open the Agent v2 API Reference Then the Agent v2 API Reference should open in a new tab + + Scenario: Backend service API can be disabled and restored + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API + And the Agent v2 draft has been published via API + And Agent v2 Backend service API access has been enabled with a key via API + When I open the Agent v2 configure page from the Agent Roster + And I switch to the Agent v2 Access Point section + And I disable Agent v2 Backend service API access + Then Agent v2 Backend service API access should be out of service + When I send the Agent v2 Backend service API minimal request + Then the Agent v2 Backend service API request should be rejected while disabled + When I enable Agent v2 Backend service API access + Then Agent v2 Backend service API access should be in service + When I send the Agent v2 Backend service API minimal request + Then the Agent v2 Backend service API request should succeed with the normal E2E marker diff --git a/e2e/features/agent-v2/support/agent.ts b/e2e/features/agent-v2/support/agent.ts index b116851ba71615..1d863bb5781266 100644 --- a/e2e/features/agent-v2/support/agent.ts +++ b/e2e/features/agent-v2/support/agent.ts @@ -16,9 +16,14 @@ import type { AgentSoulConfig, ApiKeyItem, } from '@dify/contracts/api/console/agent/types.gen' +import type { + ChatRequestPayloadWithUser, + PostChatMessagesResponse, +} from '@dify/contracts/api/service/types.gen' import { Buffer } from 'node:buffer' import { readFile } from 'node:fs/promises' import path from 'node:path' +import { request } from '@playwright/test' import { createApiContext, expectApiResponseOK, setAppSiteEnabled } from '../../../support/api' import { assertE2EResourceName, createE2EResourceName } from '../../../support/naming' @@ -58,6 +63,12 @@ export type CreateTestAgentOptions = { role?: string } +export type AgentServiceApiChatResult = { + body: PostChatMessagesResponse | unknown + ok: boolean + status: number +} + export const defaultAgentSoulConfig: AgentSoulConfig = { prompt: { system_prompt: 'You are a Dify Agent E2E test assistant.', @@ -621,6 +632,45 @@ export async function deleteAgentApiKey(agentId: string, apiKeyId: string): Prom } } +export async function sendAgentServiceApiChatMessage({ + apiKey, + query = 'Please reply with the test success marker.', + serviceApiBaseURL, +}: { + apiKey: string + query?: string + serviceApiBaseURL: string +}): Promise { + const ctx = await request.newContext() + const body = { + inputs: {}, + query, + response_mode: 'blocking', + user: 'e2e-agent-access-point', + } satisfies ChatRequestPayloadWithUser + + try { + const response = await ctx.post(`${serviceApiBaseURL.replace(/\/$/, '')}/chat-messages`, { + data: body, + headers: { + Authorization: `Bearer ${apiKey}`, + }, + }) + const responseBody = await response.json().catch(async () => ({ + message: await response.text().catch(() => ''), + })) + + return { + body: responseBody as PostChatMessagesResponse | unknown, + ok: response.ok(), + status: response.status(), + } + } + finally { + await ctx.dispose() + } +} + export async function deleteAgentConfigFile(agentId: string, name: string): Promise { const ctx = await createApiContext() try { diff --git a/e2e/features/step-definitions/agent-v2/access-point.steps.ts b/e2e/features/step-definitions/agent-v2/access-point.steps.ts index 3f8672e63a2596..6ab05a22a91bb5 100644 --- a/e2e/features/step-definitions/agent-v2/access-point.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point.steps.ts @@ -2,9 +2,12 @@ import type { DifyWorld } from '../../support/world' import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { + createAgentApiKey, getAgentAccessPath, getAgentComposerDraft, getAgentReferencingWorkflows, + publishAgent, + sendAgentServiceApiChatMessage, setAgentApiAccess, setAgentSiteAccessAndGetURL, } from '../../agent-v2/support/agent' @@ -38,6 +41,9 @@ const getAccessRegion = (world: DifyWorld) => const getWebAppCard = (world: DifyWorld) => getAccessRegion(world).locator('article').filter({ hasText: 'Web app' }).first() +const getServiceApiCard = (world: DifyWorld) => + getAccessRegion(world).locator('article').filter({ hasText: 'Backend service API' }).first() + const getDialog = (world: DifyWorld, name: string | RegExp) => world.getPage().getByRole('dialog', { name }) @@ -57,6 +63,22 @@ Given( }, ) +Given('the Agent v2 draft has been published via API', async function (this: DifyWorld) { + await publishAgent(getCurrentAgentId(this)) +}) + +Given( + 'Agent v2 Backend service API access has been enabled with a key via API', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + const apiAccess = await setAgentApiAccess(agentId, true) + const apiKey = await createAgentApiKey(agentId) + + this.agentBuilder.accessPoint.serviceApiBaseURL = apiAccess.service_api_base_url + this.agentBuilder.accessPoint.generatedApiKey = apiKey.token + }, +) + Given( 'Agent v2 Web app access will be restored for {string}', async function (this: DifyWorld, agentName: string) { @@ -429,21 +451,21 @@ Then( ) Then('I should see the Agent v2 Backend service API endpoint', async function (this: DifyWorld) { - const page = this.getPage() + const serviceApiCard = getServiceApiCard(this) if (!this.agentBuilder.accessPoint.serviceApiBaseURL) throw new Error('No Agent v2 service API endpoint found. Enable Backend service API first.') - await expect(page.getByRole('heading', { name: 'Backend service API' })).toBeVisible({ + await expect(serviceApiCard.getByRole('heading', { name: 'Backend service API' })).toBeVisible({ timeout: 30_000, }) - await expect(page.getByText('Service API Endpoint')).toBeVisible() - await expect(page.getByText(this.agentBuilder.accessPoint.serviceApiBaseURL)).toBeVisible() - await expect(page.getByLabel('Copy service API endpoint')).toBeEnabled() + await expect(serviceApiCard.getByText('Service API Endpoint')).toBeVisible() + await expect(serviceApiCard.getByText(this.agentBuilder.accessPoint.serviceApiBaseURL)).toBeVisible() + await expect(serviceApiCard.getByLabel('Copy service API endpoint')).toBeEnabled() }) When('I copy the Agent v2 Backend service API endpoint', async function (this: DifyWorld) { - await this.getPage().getByLabel('Copy service API endpoint').click() + await getServiceApiCard(this).getByLabel('Copy service API endpoint').click() }) Then( @@ -454,7 +476,7 @@ Then( ) When('I open Agent v2 API key management', async function (this: DifyWorld) { - await this.getPage() + await getServiceApiCard(this) .getByRole('button', { name: /^API Key\b/ }) .click() }) @@ -551,3 +573,62 @@ Then('the Agent v2 API Reference should open in a new tab', async function (this await apiReferencePage.close() this.agentBuilder.accessPoint.apiReferencePage = undefined }) + +When('I disable Agent v2 Backend service API access', async function (this: DifyWorld) { + await getServiceApiCard(this).getByLabel('Toggle Backend service API access').click() +}) + +Then('Agent v2 Backend service API access should be out of service', async function (this: DifyWorld) { + const serviceApiCard = getServiceApiCard(this) + + await expect(serviceApiCard.getByText('Out of service')).toBeVisible({ timeout: 30_000 }) +}) + +When('I enable Agent v2 Backend service API access', async function (this: DifyWorld) { + await getServiceApiCard(this).getByLabel('Toggle Backend service API access').click() +}) + +Then('Agent v2 Backend service API access should be in service', async function (this: DifyWorld) { + const serviceApiCard = getServiceApiCard(this) + + await expect(serviceApiCard.getByText('In service')).toBeVisible({ timeout: 30_000 }) +}) + +When('I send the Agent v2 Backend service API minimal request', async function (this: DifyWorld) { + const serviceApiBaseURL = this.agentBuilder.accessPoint.serviceApiBaseURL + const apiKey = this.agentBuilder.accessPoint.generatedApiKey + if (!serviceApiBaseURL) + throw new Error('No Agent v2 service API endpoint found. Enable Backend service API first.') + if (!apiKey) + throw new Error('No Agent v2 API key found. Create a Backend service API key first.') + + this.agentBuilder.accessPoint.serviceApiResponse = await sendAgentServiceApiChatMessage({ + apiKey, + serviceApiBaseURL, + }) +}) + +Then( + 'the Agent v2 Backend service API request should be rejected while disabled', + async function (this: DifyWorld) { + const response = this.agentBuilder.accessPoint.serviceApiResponse + if (!response) + throw new Error('No Agent v2 Backend service API response was recorded.') + + expect(response.ok).toBe(false) + expect(response.status).toBe(403) + expect(JSON.stringify(response.body).toLowerCase()).toContain('disabled') + }, +) + +Then( + 'the Agent v2 Backend service API request should succeed with the normal E2E marker', + async function (this: DifyWorld) { + const response = this.agentBuilder.accessPoint.serviceApiResponse + if (!response) + throw new Error('No Agent v2 Backend service API response was recorded.') + + expect(response.ok).toBe(true) + expect(JSON.stringify(response.body)).toContain(agentBuilderExpectedTokens.agentReply) + }, +) diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index a8b11aeda56a5d..284e8806f75833 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -47,6 +47,7 @@ export const createAgentBuilderWorldState = () => ({ apiReferencePage: undefined as Page | undefined, composerDraftSnapshot: undefined as string | undefined, generatedApiKey: undefined as string | undefined, + serviceApiResponse: undefined as { body: unknown, ok: boolean, status: number } | undefined, serviceApiBaseURL: undefined as string | undefined, webAppPage: undefined as Page | undefined, webAppURL: undefined as string | undefined, From 321d9ade7e49a88d549677e715d4df330ab2ac8f Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 22:50:24 +0800 Subject: [PATCH 103/185] test(e2e): cover knowledge reference removal --- e2e/features/agent-v2/AGENTS.md | 2 + e2e/features/agent-v2/knowledge.feature | 12 +++ e2e/features/agent-v2/support/agent.ts | 26 +++++ .../agent-v2/knowledge.steps.ts | 95 +++++++++++++++++++ 4 files changed, 135 insertions(+) create mode 100644 e2e/features/agent-v2/knowledge.feature create mode 100644 e2e/features/step-definitions/agent-v2/knowledge.steps.ts diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index c57a0cee802635..fb2e38b870eebf 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -21,6 +21,7 @@ Use API setup for prerequisite state, then use Playwright only for user-observab - `@infra` — infrastructure or readiness checks. - `@build` — Build mode and Build draft behavior. - `@files` — Files section upload, display, and fixture behavior. +- `@knowledge` — Knowledge Retrieval configuration display, persistence, and reference cleanup. - `@advanced-settings` — Env Editor, Content Moderation, and related Advanced Settings behavior. - `@agent-edit` — saved Agent detail/configuration display surfaces. - `@publish` — publish and publish-bar state. @@ -34,6 +35,7 @@ Keep Agent v2 step definitions grouped by user capability, not by DOM component - `configure.steps.ts` — common configure navigation, refresh, autosave, and normal draft assertions. - `build-draft.steps.ts` — Build mode checkout, apply, discard, supported writeback, and Build draft isolation. - `files.steps.ts` — Files upload, display, and fixture-list assertions. +- `knowledge.steps.ts` — Knowledge Retrieval configuration persistence and reference cleanup. - `tools.steps.ts` — Tools selector, search, and configuration-boundary behavior. - `advanced-settings.steps.ts` — Env Editor, Content Moderation, and Advanced Settings behavior. - `agent-edit.steps.ts` — saved Agent detail display assertions. diff --git a/e2e/features/agent-v2/knowledge.feature b/e2e/features/agent-v2/knowledge.feature new file mode 100644 index 00000000000000..4462dc6ef26419 --- /dev/null +++ b/e2e/features/agent-v2/knowledge.feature @@ -0,0 +1,12 @@ +@agent-v2 @authenticated @knowledge @core +Feature: Agent v2 Knowledge Retrieval + Scenario: Removing Knowledge Retrieval clears the saved dataset reference + Given I am signed in as the default E2E admin + And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready + And a knowledge-backed Agent v2 test agent has been created via API + When I open the Agent v2 configure page + Then I should see the Agent v2 Knowledge Retrieval "Retrieval 1" + When I remove the Agent v2 Knowledge Retrieval "Retrieval 1" + Then the Agent v2 configuration should be saved automatically + And the Agent v2 draft should no longer reference the Agent Builder knowledge base + And I should not see the Agent v2 Knowledge Retrieval "Retrieval 1" diff --git a/e2e/features/agent-v2/support/agent.ts b/e2e/features/agent-v2/support/agent.ts index 1d863bb5781266..80509fa7f1edac 100644 --- a/e2e/features/agent-v2/support/agent.ts +++ b/e2e/features/agent-v2/support/agent.ts @@ -10,6 +10,7 @@ import type { AgentConfigSnapshotDetailResponse, AgentDriveSkillItemResponse, AgentDriveSkillListResponse, + AgentKnowledgeDatasetConfig, AgentReferencingWorkflowResponse, AgentReferencingWorkflowsResponse, AgentSkillUploadResponse, @@ -244,6 +245,31 @@ export function createAgentSoulConfigWithModel( } } +export function createAgentSoulConfigWithKnowledgeDataset( + agentSoul: AgentSoulConfig, + dataset: AgentKnowledgeDatasetConfig, +): AgentSoulConfig { + return { + ...agentSoul, + knowledge: { + sets: [ + { + datasets: [dataset], + id: 'e2e-knowledge-retrieval', + name: 'Retrieval 1', + query: { + mode: 'generated_query', + }, + retrieval: { + mode: 'multiple', + top_k: 4, + }, + }, + ], + }, + } +} + export async function createTestAgent({ description = 'Created by Dify E2E.', name = createE2EResourceName('Agent'), diff --git a/e2e/features/step-definitions/agent-v2/knowledge.steps.ts b/e2e/features/step-definitions/agent-v2/knowledge.steps.ts new file mode 100644 index 00000000000000..81d1ba29c327bb --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/knowledge.steps.ts @@ -0,0 +1,95 @@ +import type { DifyWorld } from '../../support/world' +import { Given, Then, When } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { + createAgentSoulConfigWithKnowledgeDataset, + createConfiguredTestAgent, + getAgentComposerDraft, + normalAgentSoulConfig, +} from '../../agent-v2/support/agent' +import { agentBuilderPreseededResources } from '../../agent-v2/support/agent-builder-resources' +import { asArray, asRecord } from '../../agent-v2/support/preflight/common' +import { getCurrentAgentId } from './configure-helpers' + +const getPreseededKnowledgeBase = (world: DifyWorld) => { + const resource = world.agentBuilder.preflight.preseededResources[ + agentBuilderPreseededResources.agentKnowledgeBase + ] + if (!resource || resource.kind !== 'dataset') { + throw new Error( + `Preseeded dataset "${agentBuilderPreseededResources.agentKnowledgeBase}" is not available. Run the matching preflight step first.`, + ) + } + + return resource +} + +const getKnowledgeSection = (world: DifyWorld) => + world.getPage().getByRole('region', { name: 'Knowledge Retrieval' }) + +Given( + 'a knowledge-backed Agent v2 test agent has been created via API', + async function (this: DifyWorld) { + const knowledgeBase = getPreseededKnowledgeBase(this) + const agent = await createConfiguredTestAgent({ + agentSoul: createAgentSoulConfigWithKnowledgeDataset( + normalAgentSoulConfig, + { + id: knowledgeBase.id, + name: knowledgeBase.name, + }, + ), + }) + + this.createdAgentIds.push(agent.id) + this.lastCreatedAgentName = agent.name + this.lastCreatedAgentRole = agent.role ?? undefined + }, +) + +Then('I should see the Agent v2 Knowledge Retrieval {string}', async function (this: DifyWorld, name: string) { + const knowledgeSection = getKnowledgeSection(this) + + await expect(knowledgeSection).toBeVisible({ timeout: 30_000 }) + await expect(knowledgeSection.getByText(name, { exact: true })).toBeVisible() +}) + +When('I remove the Agent v2 Knowledge Retrieval {string}', async function (this: DifyWorld, name: string) { + const knowledgeSection = getKnowledgeSection(this) + + await expect(knowledgeSection).toBeVisible({ timeout: 30_000 }) + await knowledgeSection.getByText(name, { exact: true }).hover() + await knowledgeSection.getByRole('button', { + exact: true, + name: `Remove ${name}`, + }).click() +}) + +Then( + 'the Agent v2 draft should no longer reference the Agent Builder knowledge base', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + const knowledgeBase = getPreseededKnowledgeBase(this) + + await expect.poll( + async () => { + const draft = await getAgentComposerDraft(agentId) + const knowledgeSets = asArray(asRecord(draft.agent_soul?.knowledge).sets) + + return knowledgeSets.some(set => asArray(asRecord(set).datasets).some((dataset) => { + const record = asRecord(dataset) + + return record.id === knowledgeBase.id || record.name === knowledgeBase.name + })) + }, + { timeout: 30_000 }, + ).toBe(false) + }, +) + +Then('I should not see the Agent v2 Knowledge Retrieval {string}', async function (this: DifyWorld, name: string) { + const knowledgeSection = getKnowledgeSection(this) + + await expect(knowledgeSection).toBeVisible({ timeout: 30_000 }) + await expect(knowledgeSection.getByText(name, { exact: true })).not.toBeVisible() +}) From b92052d45b6eaa56c62f034ce02d12b6641df096 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 22:56:32 +0800 Subject: [PATCH 104/185] test(e2e): cover nested agent output variables --- .../agent-v2/output-variables.feature | 12 ++ .../agent-v2/workflow-node.steps.ts | 154 +++++++++++++++--- 2 files changed, 147 insertions(+), 19 deletions(-) diff --git a/e2e/features/agent-v2/output-variables.feature b/e2e/features/agent-v2/output-variables.feature index 46716b894efec5..d327ae97b03cea 100644 --- a/e2e/features/agent-v2/output-variables.feature +++ b/e2e/features/agent-v2/output-variables.feature @@ -22,3 +22,15 @@ Feature: Agent v2 output variables When I refresh the current page And I open the Agent v2 workflow node panel Then I should see the Agent v2 workflow node output variables + + Scenario: Workflow Agent v2 nested object output variables persist after refresh + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a workflow app with an Agent v2 node has been created via API + When I open the app from the app list + And I open the Agent v2 workflow node panel + And I add a required Agent v2 workflow node object output variable with text and analysis fields + Then the Agent v2 workflow node nested object output variable should be saved in the workflow draft + When I refresh the current page + And I open the Agent v2 workflow node panel + Then I should see the Agent v2 workflow node nested object output variable diff --git a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts index 1406a5fe683f4d..2322d6c04aae96 100644 --- a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts +++ b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts @@ -1,4 +1,5 @@ import type { DataTable } from '@cucumber/cucumber' +import type { DeclaredOutputConfig } from '@dify/contracts/api/console/apps/types.gen' import type { AgentV2WorkflowOutputVariable, DifyWorld } from '../../support/world' import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' @@ -25,7 +26,7 @@ const getCurrentAppId = (world: DifyWorld) => { return appId } -const getOutputVariablesFromDraft = async (appId: string) => { +const getDeclaredOutputsFromDraft = async (appId: string): Promise => { const draft = await getWorkflowDraft(appId) const agentNode = draft.graph.nodes.find(node => node.id === agentV2WorkflowNodeId) if (!agentNode) @@ -35,11 +36,50 @@ const getOutputVariablesFromDraft = async (appId: string) => { if (!Array.isArray(outputs)) return [] - return outputs as Array<{ - array_item?: { type?: string } - name?: string + return outputs as DeclaredOutputConfig[] +} + +const getOutputVariablesFromDraft = async (appId: string) => getDeclaredOutputsFromDraft(appId) + +const waitForWorkflowDraftSave = (world: DifyWorld, appId: string) => + world.getPage().waitForResponse(response => ( + response.request().method() === 'POST' + && new URL(response.url()).pathname.endsWith(`/console/api/apps/${appId}/workflows/draft`) + )) + +const openWorkflowOutputVariablesPanel = async (world: DifyWorld) => { + const page = world.getPage() + const newOutputButton = page.getByRole('button', { name: 'New output' }) + + if (!await newOutputButton.isVisible().catch(() => false)) + await page.getByRole('button', { name: 'Output Variables' }).click() + + await expect(newOutputButton).toBeVisible() +} + +const fillOutputVariableEditor = async ( + world: DifyWorld, + { + name, + required = false, + type = 'string', + }: { + name: string + required?: boolean type?: string - }> + }, +) => { + const page = world.getPage() + const editor = page.getByRole('form', { name: 'Output variable editor' }) + + await expect(editor).toBeVisible() + await editor.getByRole('textbox', { name: 'Field name' }).fill(name) + if (type !== 'string') { + await editor.getByRole('button', { name: 'Output type' }).click() + await page.getByRole('option', { name: type, exact: true }).click() + } + if (required) + await editor.getByRole('switch', { name: 'Required' }).click() } Given( @@ -109,23 +149,14 @@ When( const rows = table.hashes() as AgentV2WorkflowOutputVariable[] this.agentBuilder.workflow.outputVariables = rows - await page.getByRole('button', { name: 'Output Variables' }).click() + await openWorkflowOutputVariablesPanel(this) for (const row of rows) { await page.getByRole('button', { name: 'New output' }).click() + await fillOutputVariableEditor(this, row) + const editor = page.getByRole('form', { name: 'Output variable editor' }) - await expect(editor).toBeVisible() - - await editor.getByRole('textbox', { name: 'Field name' }).fill(row.name) - if (row.type !== 'string') { - await editor.getByRole('button', { name: 'Output type' }).click() - await page.getByRole('option', { name: row.type }).click() - } - - const saveResponse = page.waitForResponse(response => ( - response.request().method() === 'POST' - && new URL(response.url()).pathname.endsWith(`/console/api/apps/${appId}/workflows/draft`) - )) + const saveResponse = waitForWorkflowDraftSave(this, appId) await editor.getByRole('button', { name: 'Confirm' }).click() expect((await saveResponse).ok()).toBe(true) await expect(editor).not.toBeVisible() @@ -133,6 +164,36 @@ When( }, ) +When( + 'I add a required Agent v2 workflow node object output variable with text and analysis fields', + async function (this: DifyWorld) { + const page = this.getPage() + const appId = getCurrentAppId(this) + + await openWorkflowOutputVariablesPanel(this) + await page.getByRole('button', { name: 'New output' }).click() + await fillOutputVariableEditor(this, { + name: 'response', + required: true, + type: 'object', + }) + + let saveResponse = waitForWorkflowDraftSave(this, appId) + await page.getByRole('form', { name: 'Output variable editor' }).getByRole('button', { name: 'Confirm' }).click() + expect((await saveResponse).ok()).toBe(true) + + for (const fieldName of ['text', 'analysis']) { + await page.getByText('response', { exact: true }).hover() + await page.getByRole('button', { name: 'Add response' }).click() + await fillOutputVariableEditor(this, { name: fieldName }) + + saveResponse = waitForWorkflowDraftSave(this, appId) + await page.getByRole('form', { name: 'Output variable editor' }).getByRole('button', { name: 'Confirm' }).click() + expect((await saveResponse).ok()).toBe(true) + } + }, +) + Then( 'I should see the Agent v2 workflow Agent details for the created Agent', async function (this: DifyWorld) { @@ -223,10 +284,65 @@ Then('I should see the Agent v2 workflow node output variables', async function if (expectedOutputVariables.length === 0) throw new Error('No Agent v2 workflow output variables were recorded for this scenario.') - await page.getByRole('button', { name: 'Output Variables' }).click() + await openWorkflowOutputVariablesPanel(this) for (const output of expectedOutputVariables) { await expect(page.getByText(output.name, { exact: true })).toBeVisible() await expect(page.getByText(output.type, { exact: true })).toBeVisible() } }) + +Then( + 'the Agent v2 workflow node nested object output variable should be saved in the workflow draft', + async function (this: DifyWorld) { + const appId = getCurrentAppId(this) + + await expect + .poll(async () => { + const outputs = await getDeclaredOutputsFromDraft(appId) + const response = outputs.find(output => output.name === 'response') + + return { + children: response?.children?.map(child => ({ + name: child.name, + required: child.required, + type: child.type, + })), + name: response?.name, + required: response?.required, + type: response?.type, + } + }, { + timeout: 30_000, + }) + .toEqual({ + children: [ + { + name: 'text', + required: false, + type: 'string', + }, + { + name: 'analysis', + required: false, + type: 'string', + }, + ], + name: 'response', + required: true, + type: 'object', + }) + }, +) + +Then('I should see the Agent v2 workflow node nested object output variable', async function (this: DifyWorld) { + const page = this.getPage() + + await openWorkflowOutputVariablesPanel(this) + await expect(page.getByText('response', { exact: true })).toBeVisible() + await expect(page.getByText('object', { exact: true })).toBeVisible() + await expect(page.getByText('Required', { exact: true })).toBeVisible() + await expect(page.getByText('text', { exact: true })).toBeVisible() + await expect(page.getByText('analysis', { exact: true })).toBeVisible() + await expect(page.getByText('string', { exact: true })).toBeVisible() +}) From 74cd2d056fc71813c311279b27804855edf6ac5f Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 23:02:47 +0800 Subject: [PATCH 105/185] test(e2e): mark output retry controls blocked --- .../agent-v2/output-variables.feature | 18 ++++++++++ .../agent-v2/output-variables.steps.ts | 34 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/e2e/features/agent-v2/output-variables.feature b/e2e/features/agent-v2/output-variables.feature index d327ae97b03cea..a130631f3c4d53 100644 --- a/e2e/features/agent-v2/output-variables.feature +++ b/e2e/features/agent-v2/output-variables.feature @@ -34,3 +34,21 @@ Feature: Agent v2 output variables When I refresh the current page And I open the Agent v2 workflow node panel Then I should see the Agent v2 workflow node nested object output variable + + @output-retry-strategy @feature-gated + Scenario: Workflow Agent v2 output retry strategy can be saved after refresh + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a workflow app with an Agent v2 node has been created via API + When I open the app from the app list + And I open the Agent v2 workflow node panel + Then Agent v2 workflow output retry strategy should be available + + @output-retry-validation @feature-gated + Scenario: Workflow Agent v2 output retry count validation is enforced + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a workflow app with an Agent v2 node has been created via API + When I open the app from the app list + And I open the Agent v2 workflow node panel + Then Agent v2 workflow output retry count validation should be available diff --git a/e2e/features/step-definitions/agent-v2/output-variables.steps.ts b/e2e/features/step-definitions/agent-v2/output-variables.steps.ts index 31e2564416933b..69442fa24e78d0 100644 --- a/e2e/features/step-definitions/agent-v2/output-variables.steps.ts +++ b/e2e/features/step-definitions/agent-v2/output-variables.steps.ts @@ -17,3 +17,37 @@ Then('Agent v2 standalone Output Variables should be available', async function }, ) }) + +Then('Agent v2 workflow output retry strategy should be available', async function (this: DifyWorld) { + const page = this.getPage() + + await expect(page.getByRole('button', { name: 'Output Variables' })).toBeVisible({ + timeout: 30_000, + }) + + return skipBlockedPrecondition( + this, + 'Agent v2 workflow Output Variables retry strategy is not available in the current editor UI.', + { + owner: 'product', + remediation: 'Expose user-visible retry strategy controls before enabling this scenario.', + }, + ) +}) + +Then('Agent v2 workflow output retry count validation should be available', async function (this: DifyWorld) { + const page = this.getPage() + + await expect(page.getByRole('button', { name: 'Output Variables' })).toBeVisible({ + timeout: 30_000, + }) + + return skipBlockedPrecondition( + this, + 'Agent v2 workflow Output Variables retry count validation is not reachable because retry strategy controls are not available in the current editor UI.', + { + owner: 'product', + remediation: 'Expose retry count controls and validation states before enabling this scenario.', + }, + ) +}) From 791595ea73203681d2b5e979532e4343c79fd86c Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 23:14:22 +0800 Subject: [PATCH 106/185] test(e2e): cover agent output prompt references --- e2e/features/agent-v2/AGENTS.md | 1 + .../agent-v2/output-variables.feature | 24 ++++ .../agent-v2/output-variables.steps.ts | 17 +++ .../agent-v2/workflow-node.steps.ts | 112 +++++++++++++++++- .../agent-v2/components/agent-task-field.tsx | 1 + 5 files changed, 153 insertions(+), 2 deletions(-) diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index fb2e38b870eebf..0a6cca2d1c0a99 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -65,6 +65,7 @@ Use the existing namespace shape: - `world.agentBuilder.accessPoint.workflowReferencePage` - `world.agentBuilder.accessPoint.composerDraftSnapshot` - `world.agentBuilder.configure.concurrentPage` +- `world.agentBuilder.workflow.agentConsolePage` - `world.agentBuilder.workflow.outputVariables` Use `features/agent-v2/support/agent.ts` for Agent v2 API fixtures. It owns roster-shaped Agent IDs, configure/access route helpers, composer draft sync, build-draft helpers, publish, API access toggles, Agent drive file cleanup, and Agent cleanup. Store created roster Agent IDs in `DifyWorld.createdAgentIds`; the shared `After` hook deletes them after each scenario. diff --git a/e2e/features/agent-v2/output-variables.feature b/e2e/features/agent-v2/output-variables.feature index a130631f3c4d53..c6d989d5e2443b 100644 --- a/e2e/features/agent-v2/output-variables.feature +++ b/e2e/features/agent-v2/output-variables.feature @@ -35,6 +35,30 @@ Feature: Agent v2 output variables And I open the Agent v2 workflow node panel Then I should see the Agent v2 workflow node nested object output variable + Scenario: Workflow Agent v2 prompt output reference stays synced when renamed + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a workflow app with an Agent v2 node has been created via API + When I open the app from the app list + And I open the Agent v2 workflow node panel + And I insert a file output reference from the Agent v2 workflow node task editor + Then the Agent v2 workflow node task should reference the file output + When I rename the Agent v2 workflow node task output reference + Then the Agent v2 workflow node task should reference the renamed file output + When I refresh the current page + And I open the Agent v2 workflow node panel + Then the Agent v2 workflow node task should reference the renamed file output + + @output-reference-delete @feature-gated + Scenario: Workflow Agent v2 prompt output reference deletion remains explicit + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a workflow app with an Agent v2 node has been created via API + When I open the app from the app list + And I open the Agent v2 workflow node panel + And I insert a file output reference from the Agent v2 workflow node task editor + Then Agent v2 workflow task output reference deletion consistency should be available + @output-retry-strategy @feature-gated Scenario: Workflow Agent v2 output retry strategy can be saved after refresh Given I am signed in as the default E2E admin diff --git a/e2e/features/step-definitions/agent-v2/output-variables.steps.ts b/e2e/features/step-definitions/agent-v2/output-variables.steps.ts index 69442fa24e78d0..6dde187d6d83c3 100644 --- a/e2e/features/step-definitions/agent-v2/output-variables.steps.ts +++ b/e2e/features/step-definitions/agent-v2/output-variables.steps.ts @@ -35,6 +35,23 @@ Then('Agent v2 workflow output retry strategy should be available', async functi ) }) +Then('Agent v2 workflow task output reference deletion consistency should be available', async function (this: DifyWorld) { + const page = this.getPage() + + await expect(page.getByText('e2e_report.pdf', { exact: true })).toBeVisible({ + timeout: 30_000, + }) + + return skipBlockedPrecondition( + this, + 'Agent v2 workflow task output deletion consistency is not available: deleting an output from the list currently leaves the Prompt token without a stable user-visible invalid-reference state.', + { + owner: 'product', + remediation: 'Define whether deletion should sync the Prompt token, block deletion, or expose an invalid-reference state before enabling this scenario.', + }, + ) +}) + Then('Agent v2 workflow output retry count validation should be available', async function (this: DifyWorld) { const page = this.getPage() diff --git a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts index 2322d6c04aae96..5e73387d63851c 100644 --- a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts +++ b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts @@ -17,6 +17,10 @@ import { } from '../../agent-v2/support/agent' const agentV2WorkflowNodeId = 'agent-v2' +const taskFileOutputName = 'e2e_report.pdf' +const renamedTaskFileOutputName = 'e2e_final_report.pdf' + +const getAgentOutputToken = (name: string) => `[§output:${name}:${name}§]` const getCurrentAppId = (world: DifyWorld) => { const appId = world.createdAppIds.at(-1) @@ -26,13 +30,18 @@ const getCurrentAppId = (world: DifyWorld) => { return appId } -const getDeclaredOutputsFromDraft = async (appId: string): Promise => { +const getAgentV2WorkflowNodeData = async (appId: string) => { const draft = await getWorkflowDraft(appId) const agentNode = draft.graph.nodes.find(node => node.id === agentV2WorkflowNodeId) if (!agentNode) throw new Error(`Workflow draft ${appId} does not include Agent v2 node ${agentV2WorkflowNodeId}.`) - const outputs = agentNode.data?.agent_declared_outputs + return agentNode.data ?? {} +} + +const getDeclaredOutputsFromDraft = async (appId: string): Promise => { + const data = await getAgentV2WorkflowNodeData(appId) + const outputs = data.agent_declared_outputs if (!Array.isArray(outputs)) return [] @@ -141,6 +150,46 @@ When('I open the Agent v2 workflow Agent in Agent Console', async function (this this.agentBuilder.workflow.agentConsolePage = agentConsolePage }) +When( + 'I insert a file output reference from the Agent v2 workflow node task editor', + async function (this: DifyWorld) { + const page = this.getPage() + const appId = getCurrentAppId(this) + const taskEditor = page.getByRole('textbox', { name: 'Agent task' }) + + await expect(taskEditor).toBeVisible() + await taskEditor.click() + await page.getByRole('button', { name: 'Insert' }).click() + await page.getByRole('button', { name: 'New output' }).click() + + const nameInput = page.getByRole('textbox', { name: 'Field name' }) + await expect(nameInput).toBeVisible() + await nameInput.fill(taskFileOutputName) + + const saveResponse = waitForWorkflowDraftSave(this, appId) + await nameInput.press('Enter') + expect((await saveResponse).ok()).toBe(true) + }, +) + +When( + 'I rename the Agent v2 workflow node task output reference', + async function (this: DifyWorld) { + const page = this.getPage() + const appId = getCurrentAppId(this) + + await page.getByText(taskFileOutputName, { exact: true }).hover() + const editor = page.getByRole('form', { name: 'Output variable editor' }) + await expect(editor).toBeVisible() + await editor.getByRole('textbox', { name: 'Field name' }).fill(renamedTaskFileOutputName) + + const saveResponse = waitForWorkflowDraftSave(this, appId) + await editor.getByRole('button', { name: 'Confirm' }).click() + expect((await saveResponse).ok()).toBe(true) + await expect(editor).not.toBeVisible() + }, +) + When( 'I add these Agent v2 workflow node output variables', async function (this: DifyWorld, table: DataTable) { @@ -335,6 +384,20 @@ Then( }, ) +Then( + 'the Agent v2 workflow node task should reference the file output', + async function (this: DifyWorld) { + await expectAgentTaskOutputReference(this, taskFileOutputName) + }, +) + +Then( + 'the Agent v2 workflow node task should reference the renamed file output', + async function (this: DifyWorld) { + await expectAgentTaskOutputReference(this, renamedTaskFileOutputName, taskFileOutputName) + }, +) + Then('I should see the Agent v2 workflow node nested object output variable', async function (this: DifyWorld) { const page = this.getPage() @@ -346,3 +409,48 @@ Then('I should see the Agent v2 workflow node nested object output variable', as await expect(page.getByText('analysis', { exact: true })).toBeVisible() await expect(page.getByText('string', { exact: true })).toBeVisible() }) + +async function expectAgentTaskOutputReference( + world: DifyWorld, + expectedName: string, + unexpectedName?: string, +) { + const page = world.getPage() + const appId = getCurrentAppId(world) + + await expect.poll( + async () => { + const data = await getAgentV2WorkflowNodeData(appId) + const outputs = Array.isArray(data.agent_declared_outputs) + ? data.agent_declared_outputs as DeclaredOutputConfig[] + : [] + const expectedOutput = outputs.find(output => output.name === expectedName) + + return { + agentTask: data.agent_task, + expectedOutput: expectedOutput + ? { + name: expectedOutput.name, + type: expectedOutput.type, + } + : undefined, + unexpectedOutput: unexpectedName + ? outputs.some(output => output.name === unexpectedName) + : false, + } + }, + { timeout: 30_000 }, + ).toEqual({ + agentTask: expect.stringContaining(getAgentOutputToken(expectedName)), + expectedOutput: { + name: expectedName, + type: 'file', + }, + unexpectedOutput: false, + }) + + await expect(page.getByText(expectedName, { exact: true })).toBeVisible() + await expect(page.getByText('file', { exact: true })).toBeVisible() + if (unexpectedName) + await expect(page.getByText(unexpectedName, { exact: true })).toHaveCount(0) +} diff --git a/web/app/components/workflow/nodes/agent-v2/components/agent-task-field.tsx b/web/app/components/workflow/nodes/agent-v2/components/agent-task-field.tsx index ae4a94a3df5e28..bedd96806bc876 100644 --- a/web/app/components/workflow/nodes/agent-v2/components/agent-task-field.tsx +++ b/web/app/components/workflow/nodes/agent-v2/components/agent-task-field.tsx @@ -122,6 +122,7 @@ export function AgentTaskField({ >
Date: Wed, 1 Jul 2026 23:18:49 +0800 Subject: [PATCH 107/185] test(e2e): remove model preflight from agent file uploads --- e2e/features/agent-v2/files.feature | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/e2e/features/agent-v2/files.feature b/e2e/features/agent-v2/files.feature index 77e6f07a74c348..826f8531faa008 100644 --- a/e2e/features/agent-v2/files.feature +++ b/e2e/features/agent-v2/files.feature @@ -2,8 +2,7 @@ Feature: Agent v2 files Scenario: Uploading a small file keeps it in the Agent configuration Given I am signed in as the default E2E admin - And the Agent Builder stable chat model is available - And a runnable Agent v2 test agent has been created via API + And a basic configured Agent v2 test agent has been created via API When I open the Agent v2 configure page And I upload the small Agent v2 file from the Files section Then I should see the small Agent v2 file in the Files section @@ -14,8 +13,7 @@ Feature: Agent v2 files Scenario: Uploading an empty file keeps a zero-byte file in the Agent configuration Given I am signed in as the default E2E admin - And the Agent Builder stable chat model is available - And a runnable Agent v2 test agent has been created via API + And a basic configured Agent v2 test agent has been created via API When I open the Agent v2 configure page And I upload the empty Agent v2 file from the Files section Then I should see the empty Agent v2 file in the Files section @@ -26,8 +24,7 @@ Feature: Agent v2 files Scenario: Uploading a special-name file keeps the filename readable Given I am signed in as the default E2E admin - And the Agent Builder stable chat model is available - And a runnable Agent v2 test agent has been created via API + And a basic configured Agent v2 test agent has been created via API When I open the Agent v2 configure page And I upload the special-name Agent v2 file from the Files section Then I should see the special-name Agent v2 file in the Files section From bc18530a7f3abf86e66edd7d38264d5c133969ff Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 23:22:51 +0800 Subject: [PATCH 108/185] test(e2e): remove model preflight from prompt leave guard --- e2e/features/agent-v2/configure-persistence.feature | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/e2e/features/agent-v2/configure-persistence.feature b/e2e/features/agent-v2/configure-persistence.feature index 90a5ea4f34eeda..23c9182370a1f4 100644 --- a/e2e/features/agent-v2/configure-persistence.feature +++ b/e2e/features/agent-v2/configure-persistence.feature @@ -18,8 +18,7 @@ Feature: Agent v2 configure persistence @configure-persistence Scenario: Leaving Configure before autosave completes preserves prompt changes Given I am signed in as the default E2E admin - And the Agent Builder stable chat model is available - And a runnable Agent v2 test agent has been created via API + And a basic configured Agent v2 test agent has been created via API When I open the Agent v2 configure page And I fill the Agent v2 prompt editor with the updated E2E prompt And I leave the Agent v2 configure page before autosave completes From 86c666ab4080aaef1c16031d9f3c1750c367526d Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 23:26:33 +0800 Subject: [PATCH 109/185] test(e2e): remove model preflight from build draft isolation --- e2e/features/agent-v2/build-draft.feature | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/e2e/features/agent-v2/build-draft.feature b/e2e/features/agent-v2/build-draft.feature index 05f471b1bae053..a495ef2a8c0c77 100644 --- a/e2e/features/agent-v2/build-draft.feature +++ b/e2e/features/agent-v2/build-draft.feature @@ -30,8 +30,7 @@ Feature: Agent v2 build draft Scenario: Discarding a Build draft does not apply supported configuration changes Given I am signed in as the default E2E admin - And the Agent Builder stable chat model is available - And a runnable Agent v2 test agent has been created via API + And a basic configured Agent v2 test agent has been created via API And an Agent v2 Build draft adds the supported E2E files, skills, and env When I open the Agent v2 configure page Then I should see the Agent v2 Build draft pending changes @@ -116,9 +115,8 @@ Feature: Agent v2 build draft Scenario: Pending Build draft remains protected after leaving Configure Given I am signed in as the default E2E admin - And the Agent Builder stable chat model is available - And a runnable Agent v2 test agent has been created via API - And an Agent v2 Build draft uses the updated E2E prompt with the stable E2E model + And a basic configured Agent v2 test agent has been created via API + And an Agent v2 Build draft uses the updated E2E prompt When I open the Agent v2 configure page Then I should see the Agent v2 Build draft pending changes And I should see the updated E2E prompt in the Agent v2 prompt editor From 177ebcec8b9662d34bac4fb5fe22eb43b699c620 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 23:29:28 +0800 Subject: [PATCH 110/185] test(e2e): mark agent file limit cases blocked --- e2e/features/agent-v2/files.feature | 35 +++++++++ .../step-definitions/agent-v2/files.steps.ts | 71 +++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/e2e/features/agent-v2/files.feature b/e2e/features/agent-v2/files.feature index 826f8531faa008..3af8c7df7c3b48 100644 --- a/e2e/features/agent-v2/files.feature +++ b/e2e/features/agent-v2/files.feature @@ -32,3 +32,38 @@ Feature: Agent v2 files And the Agent v2 configuration should be saved automatically When I refresh the current page Then I should see the special-name Agent v2 file in the Files section + + @files-limits @feature-gated + Scenario: Unsupported Agent v2 file formats show a clear rejection reason + Given I am signed in as the default E2E admin + And a basic configured Agent v2 test agent has been created via API + When I open the Agent v2 configure page + Then Agent v2 unsupported file format rejection should be available + + @files-limits @feature-gated + Scenario: Oversized Agent v2 files show a clear rejection reason + Given I am signed in as the default E2E admin + And a basic configured Agent v2 test agent has been created via API + When I open the Agent v2 configure page + Then Agent v2 oversized file rejection should be available + + @files-limits @feature-gated + Scenario: Agent v2 single-batch file count limits are enforced + Given I am signed in as the default E2E admin + And a basic configured Agent v2 test agent has been created via API + When I open the Agent v2 configure page + Then Agent v2 single-batch file count limits should be available + + @files-limits @feature-gated + Scenario: Agent v2 total file count limits are enforced + Given I am signed in as the default E2E admin + And a basic configured Agent v2 test agent has been created via API + When I open the Agent v2 configure page + Then Agent v2 total file count limits should be available + + @files-limits @feature-gated + Scenario: Leaving during Agent v2 file upload keeps a recoverable state + Given I am signed in as the default E2E admin + And a basic configured Agent v2 test agent has been created via API + When I open the Agent v2 configure page + Then Agent v2 in-progress file upload recovery should be available diff --git a/e2e/features/step-definitions/agent-v2/files.steps.ts b/e2e/features/step-definitions/agent-v2/files.steps.ts index f8f6fefaf78414..d972b289c8cf33 100644 --- a/e2e/features/step-definitions/agent-v2/files.steps.ts +++ b/e2e/features/step-definitions/agent-v2/files.steps.ts @@ -1,6 +1,7 @@ import type { DifyWorld } from '../../support/world' import { Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' +import { skipBlockedPrecondition } from '../../agent-v2/support/preflight/common' import { agentBuilderFileTreeFixtureFileNames } from '../../agent-v2/support/test-materials' import { expectAgentConfigFileHidden, @@ -80,3 +81,73 @@ Then( await expectAgentConfigFileSaved(this, 'specialFilename') }, ) + +Then('Agent v2 unsupported file format rejection should be available', async function (this: DifyWorld) { + await expectFilesSectionVisible(this) + + return skipBlockedPrecondition( + this, + 'Agent v2 unsupported file format rejection is not stable: default upload configuration allows arbitrary extensions unless UPLOAD_FILE_EXTENSION_BLACKLIST is seeded.', + { + owner: 'product/seed', + remediation: 'Define Agent config file type restrictions or seed UPLOAD_FILE_EXTENSION_BLACKLIST before enabling this scenario.', + }, + ) +}) + +Then('Agent v2 oversized file rejection should be available', async function (this: DifyWorld) { + await expectFilesSectionVisible(this) + + return skipBlockedPrecondition( + this, + 'Agent v2 oversized file rejection lacks a clear user-visible reason: the current upload dialog collapses upload and commit failures into a generic failure toast.', + { + owner: 'product', + remediation: 'Expose a stable user-visible file-size error before enabling this scenario.', + }, + ) +}) + +Then('Agent v2 single-batch file count limits should be available', async function (this: DifyWorld) { + await expectFilesSectionVisible(this) + + return skipBlockedPrecondition( + this, + 'Agent v2 single-batch file count limits are not reachable: the current Agent config file upload dialog accepts one file per upload.', + { + owner: 'product', + remediation: 'Define multi-file upload behavior and its count-limit error before enabling this scenario.', + }, + ) +}) + +Then('Agent v2 total file count limits should be available', async function (this: DifyWorld) { + await expectFilesSectionVisible(this) + + return skipBlockedPrecondition( + this, + 'Agent v2 total file count limits are not defined for Agent config files in the current product contract.', + { + owner: 'product', + remediation: 'Define the Agent config file total-count limit and user-visible error before enabling this scenario.', + }, + ) +}) + +Then('Agent v2 in-progress file upload recovery should be available', async function (this: DifyWorld) { + await expectFilesSectionVisible(this) + + return skipBlockedPrecondition( + this, + 'Agent v2 in-progress file upload recovery is not stable: the current dialog has no deterministic slow-upload fixture or user-visible navigation guard contract.', + { + owner: 'product/test-infra', + remediation: 'Define upload-in-progress navigation behavior and provide a deterministic slow upload fixture before enabling this scenario.', + }, + ) +}) + +async function expectFilesSectionVisible(world: DifyWorld) { + await expect(world.getPage().getByRole('region', { name: 'Files' })) + .toBeVisible({ timeout: 30_000 }) +} From 1c4822539569e8cb503d33009a7562463d942eaf Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 23:33:43 +0800 Subject: [PATCH 111/185] docs(e2e): document agent file limit gates --- e2e/features/agent-v2/AGENTS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 0a6cca2d1c0a99..82b476fb83590b 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -21,6 +21,7 @@ Use API setup for prerequisite state, then use Playwright only for user-observab - `@infra` — infrastructure or readiness checks. - `@build` — Build mode and Build draft behavior. - `@files` — Files section upload, display, and fixture behavior. +- `@files-limits` — feature-gated file format, size, count, and in-progress upload limit behavior. - `@knowledge` — Knowledge Retrieval configuration display, persistence, and reference cleanup. - `@advanced-settings` — Env Editor, Content Moderation, and related Advanced Settings behavior. - `@agent-edit` — saved Agent detail/configuration display surfaces. @@ -160,4 +161,6 @@ Blocked precondition: missing . Owner: seed/product. Reme Use partial coverage only when current product behavior is intentionally narrower than the written requirement and the test still asserts a real user-visible behavior. Example: Files are currently flat in Agent config files, so the flat Files list can be asserted while tree display remains blocked until product support exists. +File format, size, count, and in-progress upload limit cases are feature-gated until the product exposes stable Agent config file restrictions and user-visible recovery/error states. Do not convert `@files-limits` scenarios to passing tests by relying on default environment behavior; first align the product contract or seed configuration. + Do not mark a scenario as complete if it only proves setup state and does not assert the user-visible behavior or persisted product contract required by the case. From 8a7a9bed0bc65d9c13f1d8b387e2bae4c63baff5 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 23:45:30 +0800 Subject: [PATCH 112/185] test(e2e): align stable model preflight contract --- e2e/features/agent-v2/AGENTS.md | 14 +++++++++++++- e2e/features/agent-v2/preflight.feature | 2 ++ e2e/features/agent-v2/support/preflight/models.ts | 7 ++++++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 82b476fb83590b..e6d02a14d6da88 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -119,7 +119,19 @@ Agent Builder resource checks live under `features/agent-v2/support/preflight/`. `preflight.steps.ts` should remain the explicit `Given` entrypoint. Do not move preflight into hidden hooks. -Use `the Agent Builder stable chat model is available` before scenarios that must run an Agent with a real model. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` define the single usable model for the E2E environment. The step defaults the type to `llm`, verifies the model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilder.preflight.stableModel`. +Use `the Agent Builder stable chat model is available` before scenarios that need a real Agent Soul model configuration. This includes true runtime scenarios and build-mode workflows whose backend API requires model config, such as Agent workflow nodes. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` are selectors for a model already configured in the workspace; they are not provider credentials. The step defaults the type to `llm`, verifies the model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilder.preflight.stableModel`. + +Do not pass model provider API keys through Cucumber or Playwright env vars. Provider credentials belong to the Dify environment seed/admin setup. If the selected provider/model is missing or inactive, the scenario must be blocked by preflight instead of trying to create or patch provider credentials during the test. + +Use this stable OpenAI GPT-5 family selector unless a scenario explicitly needs a different model: + +```bash +E2E_STABLE_MODEL_PROVIDER=openai +E2E_STABLE_MODEL_NAME=gpt-5.4-mini +E2E_STABLE_MODEL_TYPE=llm +``` + +Dify may expose OpenAI as either `openai` or a plugin provider ID such as `langgenius/openai/openai`. The preflight accepts both forms for selection and stores the actual Console API provider ID for Agent Soul setup. Use `the Agent Builder broken chat model is available` before model-recovery scenarios that intentionally start from an invalid model. The step requires `E2E_BROKEN_MODEL_PROVIDER`, defaults `E2E_BROKEN_MODEL_NAME` to `e2e-broken-model`, defaults `E2E_BROKEN_MODEL_TYPE` to `llm`, and only verifies that the model entry exists. The scenario must still assert the user-visible failure and recovery behavior. diff --git a/e2e/features/agent-v2/preflight.feature b/e2e/features/agent-v2/preflight.feature index 4453067ba9dc36..0928eb9be85b8f 100644 --- a/e2e/features/agent-v2/preflight.feature +++ b/e2e/features/agent-v2/preflight.feature @@ -9,10 +9,12 @@ Feature: Agent Builder preseeded environment And I publish the Agent v2 draft Then the Agent v2 draft should be published and up to date + @stable-model Scenario: Stable chat model is available Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available + @broken-model Scenario: Broken chat model is available for recovery scenarios Given I am signed in as the default E2E admin And the Agent Builder broken chat model is available diff --git a/e2e/features/agent-v2/support/preflight/models.ts b/e2e/features/agent-v2/support/preflight/models.ts index 16f1438cf14a29..662008ee8e2935 100644 --- a/e2e/features/agent-v2/support/preflight/models.ts +++ b/e2e/features/agent-v2/support/preflight/models.ts @@ -14,6 +14,11 @@ const activeModelStatus = 'active' const defaultStableChatModelType = 'llm' const defaultBrokenChatModelName = agentBuilderPreseededResources.brokenModel +const getProviderAlias = (provider: string) => provider.split('/').filter(Boolean).at(-1) ?? provider + +const matchesProvider = (actual: string, expected: string) => + actual === expected || getProviderAlias(actual) === getProviderAlias(expected) + type ModelPreflightConfig = | { ok: true @@ -94,7 +99,7 @@ async function skipMissingAgentBuilderModel( ) await expectApiResponseOK(response, `Check ${config.resourceName}`) const body = (await response.json()) as ProviderWithModelsDataResponse - const provider = body.data.find(item => item.provider === config.provider) + const provider = body.data.find(item => matchesProvider(item.provider, config.provider)) const model = provider?.models.find( item => item.model === config.value From c06369f2d94bf920753b1a8bd6697ee39fec8a5a Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 23:49:47 +0800 Subject: [PATCH 113/185] test(e2e): default agent stable model selector --- e2e/features/agent-v2/AGENTS.md | 4 ++-- .../agent-v2/support/preflight/models.ts | 19 ++++--------------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index e6d02a14d6da88..0b3c9251f338ce 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -119,11 +119,11 @@ Agent Builder resource checks live under `features/agent-v2/support/preflight/`. `preflight.steps.ts` should remain the explicit `Given` entrypoint. Do not move preflight into hidden hooks. -Use `the Agent Builder stable chat model is available` before scenarios that need a real Agent Soul model configuration. This includes true runtime scenarios and build-mode workflows whose backend API requires model config, such as Agent workflow nodes. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` are selectors for a model already configured in the workspace; they are not provider credentials. The step defaults the type to `llm`, verifies the model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilder.preflight.stableModel`. +Use `the Agent Builder stable chat model is available` before scenarios that need a real Agent Soul model configuration. This includes true runtime scenarios and build-mode workflows whose backend API requires model config, such as Agent workflow nodes. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` are selectors for a model already configured in the workspace; they are not provider credentials. The step defaults to `openai` / `gpt-5.4-mini` / `llm`, verifies the selected model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilder.preflight.stableModel`. Do not pass model provider API keys through Cucumber or Playwright env vars. Provider credentials belong to the Dify environment seed/admin setup. If the selected provider/model is missing or inactive, the scenario must be blocked by preflight instead of trying to create or patch provider credentials during the test. -Use this stable OpenAI GPT-5 family selector unless a scenario explicitly needs a different model: +Override the default selector only when a scenario or environment explicitly needs a different stable model: ```bash E2E_STABLE_MODEL_PROVIDER=openai diff --git a/e2e/features/agent-v2/support/preflight/models.ts b/e2e/features/agent-v2/support/preflight/models.ts index 662008ee8e2935..330c61c9d482f6 100644 --- a/e2e/features/agent-v2/support/preflight/models.ts +++ b/e2e/features/agent-v2/support/preflight/models.ts @@ -11,6 +11,8 @@ const brokenChatModelProviderEnv = 'E2E_BROKEN_MODEL_PROVIDER' const brokenChatModelNameEnv = 'E2E_BROKEN_MODEL_NAME' const brokenChatModelTypeEnv = 'E2E_BROKEN_MODEL_TYPE' const activeModelStatus = 'active' +const defaultStableChatModelProvider = 'openai' +const defaultStableChatModelName = 'gpt-5.4-mini' const defaultStableChatModelType = 'llm' const defaultBrokenChatModelName = agentBuilderPreseededResources.brokenModel @@ -33,23 +35,10 @@ type ModelPreflightConfig } export function readAgentBuilderStableChatModelConfig(): ModelPreflightConfig { - const provider = process.env[stableChatModelProviderEnv]?.trim() - const name = process.env[stableChatModelNameEnv]?.trim() + const provider = process.env[stableChatModelProviderEnv]?.trim() || defaultStableChatModelProvider + const name = process.env[stableChatModelNameEnv]?.trim() || defaultStableChatModelName const type = process.env[stableChatModelTypeEnv]?.trim() || defaultStableChatModelType - const missing: string[] = [] - if (!provider) - missing.push(stableChatModelProviderEnv) - if (!name) - missing.push(stableChatModelNameEnv) - - if (!provider || !name) { - return { - ok: false, - reason: `${agentBuilderPreseededResources.stableChatModel} requires ${missing.join(', ')}.`, - } - } - return { ok: true, provider, From 37d1be40c976373d743abe4bf7d2e26d0cb123b1 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 23:51:35 +0800 Subject: [PATCH 114/185] docs(e2e): clarify agent skill seed contract --- e2e/features/agent-v2/AGENTS.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 0b3c9251f338ce..47a5aaa692ce3f 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -141,7 +141,9 @@ Use `the Agent Builder preseeded dataset "{name}" is indexed and ready` for know Use `the Agent Builder preseeded dataset "{name}" is indexing` for failure-recovery scenarios that require an indexing or queued knowledge base. It verifies at least one document is in `waiting`, `parsing`, `cleaning`, `splitting`, or `indexing`. -Use `the Agent Builder preseeded Agent "{agent}" includes drive skill "{skill}"` to verify that a fixed Agent has a drive-backed Skill attached. +`e2e-summary-skill` has two separate E2E contracts. The checked-in package under `e2e/fixtures/test-materials/e2e-summary-skill/SKILL.md` is used by scenario-owned Agents to verify that the Skill package can be uploaded to an Agent drive. Fixed display/configuration scenarios use `e2e-summary-skill` as a preseeded resource: the environment-owned `E2E New Agent Builder Full Config` Agent must already include that drive-backed Skill before the scenario starts. Do not mutate fixed preseeded Agents during a scenario to add missing Skills. + +Use `the Agent Builder preseeded Agent "{agent}" includes drive skill "{skill}"` to verify that a fixed Agent has a drive-backed Skill attached. If it is missing, return a blocked precondition owned by seed/product instead of uploading the Skill into the fixed Agent. Use `the Agent Builder preseeded Agent "{agent}" has Backend service API access with an API key` to verify that a fixed Agent has Backend service API enabled and at least one key. The API key step does not validate a human-readable key name because the Console API key response does not expose one. From 5ad75e9ed4b8950ec388d0242edfda7593cdd1f5 Mon Sep 17 00:00:00 2001 From: yyh Date: Wed, 1 Jul 2026 23:55:33 +0800 Subject: [PATCH 115/185] test(e2e): separate web app runtime from core publish suite --- e2e/features/agent-v2/AGENTS.md | 3 ++- e2e/features/agent-v2/publish.feature | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 47a5aaa692ce3f..881c56a866ae6b 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -17,7 +17,7 @@ Use API setup for prerequisite state, then use Playwright only for user-observab ## Tags - `@agent-v2` — required capability tag for all Agent v2 scenarios. -- `@core` — stable scenario expected to run in the regular Agent v2 suite when its explicit preconditions are met. +- `@core` — stable build-mode scenario expected to run in the regular Agent v2 suite when its explicit preconditions are met. Do not apply `@core` to Preview/Test Run or Web app runtime scenarios in the current slice. - `@infra` — infrastructure or readiness checks. - `@build` — Build mode and Build draft behavior. - `@files` — Files section upload, display, and fixture behavior. @@ -27,6 +27,7 @@ Use API setup for prerequisite state, then use Playwright only for user-observab - `@agent-edit` — saved Agent detail/configuration display surfaces. - `@publish` — publish and publish-bar state. - `@access-point` — Web app, Backend service API, and Workflow access surfaces. +- `@web-app-runtime` — published public Web app runtime behavior. Keep this outside `@core` until runtime coverage is explicitly in scope. - `@feature-gated` — product capability is optional. This tag alone does not skip execution; the scenario must include an explicit step that returns `skipped` with a blocked-precondition reason when the feature is unavailable. ## Step Organization diff --git a/e2e/features/agent-v2/publish.feature b/e2e/features/agent-v2/publish.feature index ab692647aaf222..7c7f10619d6755 100644 --- a/e2e/features/agent-v2/publish.feature +++ b/e2e/features/agent-v2/publish.feature @@ -1,5 +1,6 @@ -@agent-v2 @authenticated @publish @core +@agent-v2 @authenticated @publish Feature: Agent v2 publish + @core Scenario: Publish a configured Agent v2 draft Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -8,6 +9,7 @@ Feature: Agent v2 publish And I publish the Agent v2 draft Then the Agent v2 draft should be published and up to date + @core Scenario: Publish action follows unpublished changes Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -21,6 +23,7 @@ Feature: Agent v2 publish Then the Agent v2 configuration should be saved automatically And the Agent v2 publish action should be available for unpublished changes + @core Scenario: Published Agent v2 version remains isolated from draft edits Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -33,6 +36,7 @@ Feature: Agent v2 publish And the normal Agent v2 draft should use the updated E2E prompt And the active published Agent v2 version should still use the normal E2E prompt + @core Scenario: Restoring a published Agent v2 version shows the restored configuration in Builder Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available From 20d31dd1212aa9e958ab42183c226efa2de99881 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 00:02:26 +0800 Subject: [PATCH 116/185] test(e2e): cover web app draft isolation --- e2e/features/agent-v2/AGENTS.md | 4 +-- e2e/features/agent-v2/publish.feature | 17 ++++++++++++ .../agent-v2/access-point.steps.ts | 27 +++++++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 881c56a866ae6b..432867843d21ae 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -10,7 +10,7 @@ Agent v2 scenarios live under `features/agent-v2/` and use the `@agent-v2` capab The E2E web environment enables Agent v2 through `NEXT_PUBLIC_ENABLE_AGENT_V2=true` in `scripts/common.ts`, because `/roster` routes are guarded by that feature flag. -Preview/Test Run scenarios are not part of the current build-mode slice unless explicitly requested. Current Agent v2 coverage should prioritize Configure, Build draft, saved configuration display, publish state, Access Point, preflight, files, advanced settings, and other build-mode behavior. +Preview/Test Run scenarios are not part of the current build-mode slice unless explicitly requested. Current Agent v2 coverage should prioritize Configure, Build draft, saved configuration display, publish state, Access Point, preflight, files, advanced settings, and other build-mode behavior. Published Web app runtime is not Builder Preview; keep it as a separate `@web-app-runtime` slice because it exercises the public app surface and real model-backed responses after publish. Use API setup for prerequisite state, then use Playwright only for user-observable navigation, editing, and assertions. Do not make assertions pass by mirroring the current implementation blindly; if a failure exposes a product ambiguity, resource gap, or test-quality problem, identify the owner before changing the test. @@ -27,7 +27,7 @@ Use API setup for prerequisite state, then use Playwright only for user-observab - `@agent-edit` — saved Agent detail/configuration display surfaces. - `@publish` — publish and publish-bar state. - `@access-point` — Web app, Backend service API, and Workflow access surfaces. -- `@web-app-runtime` — published public Web app runtime behavior. Keep this outside `@core` until runtime coverage is explicitly in scope. +- `@web-app-runtime` — published public Web app runtime behavior. Use it for scenarios that open the public Web app and assert real chat responses. Access Point URL, launch, customization, and settings surfaces remain `@access-point` behavior unless they send messages through the public Web app. - `@feature-gated` — product capability is optional. This tag alone does not skip execution; the scenario must include an explicit step that returns `skipped` with a blocked-precondition reason when the feature is unavailable. ## Step Organization diff --git a/e2e/features/agent-v2/publish.feature b/e2e/features/agent-v2/publish.feature index 7c7f10619d6755..4f04d2771be062 100644 --- a/e2e/features/agent-v2/publish.feature +++ b/e2e/features/agent-v2/publish.feature @@ -58,6 +58,23 @@ Feature: Agent v2 publish And the normal Agent v2 draft should use the normal E2E prompt And the Agent v2 publish action should be available for unpublished changes + @web-app-runtime + Scenario: Published Web app remains isolated from unpublished Agent v2 draft edits + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API + When I open the Agent v2 configure page + And I publish the Agent v2 draft + Then the Agent v2 draft should be published and up to date + When Agent v2 Web app access has been enabled via API + And I fill the Agent v2 prompt editor with the updated E2E prompt + Then the Agent v2 configuration should be saved automatically + And the normal Agent v2 draft should use the updated E2E prompt + When I open the Agent v2 Web app URL + And I send an E2E message in the Agent v2 Web app + Then the Agent v2 Web app response should include the normal E2E marker + And the Agent v2 Web app response should not include the updated E2E marker + @web-app-runtime Scenario: Published Web app uses the latest Agent v2 published configuration Given I am signed in as the default E2E admin diff --git a/e2e/features/step-definitions/agent-v2/access-point.steps.ts b/e2e/features/step-definitions/agent-v2/access-point.steps.ts index 6ab05a22a91bb5..c60eecd4c1a1da 100644 --- a/e2e/features/step-definitions/agent-v2/access-point.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point.steps.ts @@ -252,6 +252,33 @@ Then( }, ) +Then( + 'the Agent v2 Web app response should include the normal E2E marker', + async function (this: DifyWorld) { + const webAppPage = this.agentBuilder.accessPoint.webAppPage + if (!webAppPage) + throw new Error('No Agent v2 Web app page was opened.') + + await expect(webAppPage.getByText(agentBuilderExpectedTokens.agentReply)) + .toBeVisible({ timeout: 120_000 }) + }, +) + +Then( + 'the Agent v2 Web app response should not include the updated E2E marker', + async function (this: DifyWorld) { + const webAppPage = this.agentBuilder.accessPoint.webAppPage + if (!webAppPage) + throw new Error('No Agent v2 Web app page was opened.') + + await expect(webAppPage.getByText(agentBuilderExpectedTokens.updatedAgentReply)) + .not + .toBeVisible() + await webAppPage.close() + this.agentBuilder.accessPoint.webAppPage = undefined + }, +) + When('I open Agent v2 Embedded configuration', async function (this: DifyWorld) { await getWebAppCard(this).getByRole('button', { name: 'Embedded' }).click() }) From 0f2fcd67411da60def7c6516ffe9d527c7889062 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 00:05:27 +0800 Subject: [PATCH 117/185] test(e2e): keep feature-gated scenarios outside core tags --- e2e/features/agent-v2/advanced-settings.feature | 7 ++++++- e2e/features/agent-v2/agent-edit.feature | 8 +++++++- e2e/features/agent-v2/build-draft.feature | 9 ++++++++- e2e/features/agent-v2/files.feature | 5 ++++- e2e/features/agent-v2/output-variables.feature | 5 ++++- 5 files changed, 29 insertions(+), 5 deletions(-) diff --git a/e2e/features/agent-v2/advanced-settings.feature b/e2e/features/agent-v2/advanced-settings.feature index d485e82e674fa5..7381bc1bdc4391 100644 --- a/e2e/features/agent-v2/advanced-settings.feature +++ b/e2e/features/agent-v2/advanced-settings.feature @@ -1,5 +1,6 @@ -@agent-v2 @authenticated @advanced-settings @core +@agent-v2 @authenticated @advanced-settings Feature: Agent v2 advanced settings + @core Scenario: Advanced Settings exposes supported configuration entries Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API @@ -8,6 +9,7 @@ Feature: Agent v2 advanced settings When I expand Agent v2 Advanced Settings Then I should see the supported Agent v2 Advanced Settings entries + @core Scenario: Plain environment variables are saved and restored Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API @@ -18,6 +20,7 @@ Feature: Agent v2 advanced settings When I refresh the current page Then I should see the plain Agent v2 environment variable in Advanced Settings + @core Scenario: Valid environment imports are saved and restored Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API @@ -28,6 +31,7 @@ Feature: Agent v2 advanced settings When I refresh the current page Then I should see the Agent v2 environment variables from the valid import in Advanced Settings + @core Scenario: Deleted environment variables are removed after refresh Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API @@ -41,6 +45,7 @@ Feature: Agent v2 advanced settings When I refresh the current page Then I should not see the deleted Agent v2 environment variable in Advanced Settings + @core Scenario: Invalid environment imports report skipped lines and keep existing variables Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API diff --git a/e2e/features/agent-v2/agent-edit.feature b/e2e/features/agent-v2/agent-edit.feature index b7789d69869a64..398b22e9dc2c9e 100644 --- a/e2e/features/agent-v2/agent-edit.feature +++ b/e2e/features/agent-v2/agent-edit.feature @@ -1,5 +1,6 @@ -@agent-v2 @authenticated @agent-edit @core +@agent-v2 @authenticated @agent-edit Feature: Agent v2 Agent Edit page + @core Scenario: Saved orchestration sections are visible on the Agent Edit page Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -8,6 +9,7 @@ Feature: Agent v2 Agent Edit page When I open the preseeded Agent v2 configure page for "E2E New Agent Builder Full Config" from the Agent Roster Then I should see the Agent v2 full-config fixture sections + @core Scenario: Duplicated Agent inherits configuration without changing the original Agent Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -21,6 +23,7 @@ Feature: Agent v2 Agent Edit page And the normal Agent v2 draft should use the updated E2E prompt And the preseeded Agent v2 "E2E New Agent Builder Full Config" should still use the normal E2E prompt + @core Scenario: Tool states are visible on the Agent Edit page Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" is available @@ -36,6 +39,7 @@ Feature: Agent v2 Agent Edit page When I open the preseeded Agent v2 configure page for "E2E New Agent Builder Tool States" from the Agent Roster Then Agent v2 Tool credential error state should be available + @core Scenario: File fixture entries are visible in the current flat Files list Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E Agent With File Tree" is available @@ -44,6 +48,7 @@ Feature: Agent v2 Agent Edit page When I open the preseeded Agent v2 configure page for "E2E Agent With File Tree" from the Agent Roster Then I should see the Agent v2 file fixture entries in the current flat Files list + @core Scenario: Dual Knowledge Retrieval settings are visible on the Agent Edit page Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E Agent With Dual Retrieval" is available @@ -51,6 +56,7 @@ Feature: Agent v2 Agent Edit page When I open the preseeded Agent v2 configure page for "E2E Agent With Dual Retrieval" from the Agent Roster Then I should see the Agent v2 dual retrieval fixture settings + @core Scenario: Agent Edit opens the same Agent in Agent Console Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available diff --git a/e2e/features/agent-v2/build-draft.feature b/e2e/features/agent-v2/build-draft.feature index a495ef2a8c0c77..d0130256e6ddc5 100644 --- a/e2e/features/agent-v2/build-draft.feature +++ b/e2e/features/agent-v2/build-draft.feature @@ -1,5 +1,6 @@ -@agent-v2 @authenticated @build @core +@agent-v2 @authenticated @build Feature: Agent v2 build draft + @core Scenario: Generating a Build draft leaves the normal Agent configuration unchanged Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -12,6 +13,7 @@ Feature: Agent v2 build draft And I should see the Agent v2 Build mode confirmation state And the normal Agent v2 draft should still use the normal E2E prompt + @core Scenario: Discarding a Build draft keeps the original Agent configuration Given I am signed in as the default E2E admin And an Agent v2 test agent has been created via API @@ -28,6 +30,7 @@ Feature: Agent v2 build draft Then I should see the normal E2E prompt in the Agent v2 prompt editor And the Agent v2 Build draft should no longer be active + @core Scenario: Discarding a Build draft does not apply supported configuration changes Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API @@ -53,6 +56,7 @@ Feature: Agent v2 build draft And the Agent v2 draft should not include the supported Build draft config And the Agent v2 Build draft should no longer be active + @core Scenario: Applying a pending Build draft updates the normal Agent configuration Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -70,6 +74,7 @@ Feature: Agent v2 build draft Then I should see the updated E2E prompt in the Agent v2 prompt editor And the Agent v2 Build draft should no longer be active + @core Scenario: Applying a Build draft updates supported configuration sections Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -96,6 +101,7 @@ Feature: Agent v2 build draft And I should see the supported E2E environment variable in Advanced Settings And the Agent v2 Build draft should no longer be active + @core Scenario: Applying a Build draft with an existing Skill keeps a single Skill entry Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -113,6 +119,7 @@ Feature: Agent v2 build draft Then I should see one e2e-summary-skill Skill in the Skills section And the Agent v2 draft should include one e2e-summary-skill Skill + @core Scenario: Pending Build draft remains protected after leaving Configure Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API diff --git a/e2e/features/agent-v2/files.feature b/e2e/features/agent-v2/files.feature index 3af8c7df7c3b48..fdc42259ebf00a 100644 --- a/e2e/features/agent-v2/files.feature +++ b/e2e/features/agent-v2/files.feature @@ -1,5 +1,6 @@ -@agent-v2 @authenticated @files @core +@agent-v2 @authenticated @files Feature: Agent v2 files + @core Scenario: Uploading a small file keeps it in the Agent configuration Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API @@ -11,6 +12,7 @@ Feature: Agent v2 files When I refresh the current page Then I should see the small Agent v2 file in the Files section + @core Scenario: Uploading an empty file keeps a zero-byte file in the Agent configuration Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API @@ -22,6 +24,7 @@ Feature: Agent v2 files When I refresh the current page Then I should see the empty Agent v2 file in the Files section + @core Scenario: Uploading a special-name file keeps the filename readable Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API diff --git a/e2e/features/agent-v2/output-variables.feature b/e2e/features/agent-v2/output-variables.feature index c6d989d5e2443b..a498c43f742edb 100644 --- a/e2e/features/agent-v2/output-variables.feature +++ b/e2e/features/agent-v2/output-variables.feature @@ -1,4 +1,4 @@ -@agent-v2 @authenticated @output-variables @core +@agent-v2 @authenticated @output-variables Feature: Agent v2 output variables @standalone-output-variables @feature-gated Scenario: Standalone Agent configure exposes Output Variables @@ -7,6 +7,7 @@ Feature: Agent v2 output variables When I open the Agent v2 configure page Then Agent v2 standalone Output Variables should be available + @core Scenario: Workflow Agent v2 output variables persist after refresh Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -23,6 +24,7 @@ Feature: Agent v2 output variables And I open the Agent v2 workflow node panel Then I should see the Agent v2 workflow node output variables + @core Scenario: Workflow Agent v2 nested object output variables persist after refresh Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -35,6 +37,7 @@ Feature: Agent v2 output variables And I open the Agent v2 workflow node panel Then I should see the Agent v2 workflow node nested object output variable + @core Scenario: Workflow Agent v2 prompt output reference stays synced when renamed Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available From 726821e80e8a721ff6411b72d0147706a3a59fc0 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 00:07:42 +0800 Subject: [PATCH 118/185] test(e2e): tag service api runtime separately --- e2e/features/agent-v2/AGENTS.md | 3 ++- e2e/features/agent-v2/access-point.feature | 11 +++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 432867843d21ae..a80ab5dc1e2572 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -17,7 +17,7 @@ Use API setup for prerequisite state, then use Playwright only for user-observab ## Tags - `@agent-v2` — required capability tag for all Agent v2 scenarios. -- `@core` — stable build-mode scenario expected to run in the regular Agent v2 suite when its explicit preconditions are met. Do not apply `@core` to Preview/Test Run or Web app runtime scenarios in the current slice. +- `@core` — stable non-runtime scenario expected to run in the regular Agent v2 suite when its explicit preconditions are met. Do not apply `@core` to Preview/Test Run, Web app chat runtime, or Backend service API chat runtime scenarios. - `@infra` — infrastructure or readiness checks. - `@build` — Build mode and Build draft behavior. - `@files` — Files section upload, display, and fixture behavior. @@ -28,6 +28,7 @@ Use API setup for prerequisite state, then use Playwright only for user-observab - `@publish` — publish and publish-bar state. - `@access-point` — Web app, Backend service API, and Workflow access surfaces. - `@web-app-runtime` — published public Web app runtime behavior. Use it for scenarios that open the public Web app and assert real chat responses. Access Point URL, launch, customization, and settings surfaces remain `@access-point` behavior unless they send messages through the public Web app. +- `@service-api-runtime` — Backend service API runtime behavior. Use it for scenarios that call the published service API and assert real chat responses. Endpoint display, copy, API key, and API reference surfaces remain `@access-point` behavior. - `@feature-gated` — product capability is optional. This tag alone does not skip execution; the scenario must include an explicit step that returns `skipped` with a blocked-precondition reason when the feature is unavailable. ## Step Organization diff --git a/e2e/features/agent-v2/access-point.feature b/e2e/features/agent-v2/access-point.feature index f213270e07f310..2ff5e2c439f563 100644 --- a/e2e/features/agent-v2/access-point.feature +++ b/e2e/features/agent-v2/access-point.feature @@ -1,5 +1,6 @@ -@agent-v2 @authenticated @access-point @core +@agent-v2 @authenticated @access-point Feature: Agent v2 Access Point + @core Scenario: Access Point shows the available Agent v2 access surfaces Given I am signed in as the default E2E admin And an Agent v2 test agent has been created via API @@ -7,7 +8,7 @@ Feature: Agent v2 Access Point And I switch to the Agent v2 Access Point section Then I should see the Agent v2 Access Point overview - @web-app-access + @core @web-app-access Scenario: Web app access actions open their public surfaces Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E Agent Published Web App" is available @@ -27,7 +28,7 @@ Feature: Agent v2 Access Point Then I should see the Agent v2 Web app settings dialog And the Agent v2 orchestration draft for "E2E Agent Published Web App" should be unchanged - @web-app-access + @core @web-app-access Scenario: Web app access can be disabled and restored Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E Agent Published Web App" is available @@ -45,7 +46,7 @@ Feature: Agent v2 Access Point When I refresh the current page Then Agent v2 Web app access should be in service - @workflow-reference + @core @workflow-reference Scenario: Workflow access shows the referencing workflow Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E Agent With Workflow Reference" is available @@ -56,6 +57,7 @@ Feature: Agent v2 Access Point When I open the Agent v2 Workflow access reference for "E2E Agent Reference Workflow" Then the Agent v2 Workflow access reference for "E2E Agent Reference Workflow" should open in Studio + @core Scenario: Backend service API supports endpoint copy, key creation, and API reference navigation Given I am signed in as the default E2E admin And an Agent v2 test agent has been created via API @@ -75,6 +77,7 @@ Feature: Agent v2 Access Point And I open the Agent v2 API Reference Then the Agent v2 API Reference should open in a new tab + @service-api-runtime Scenario: Backend service API can be disabled and restored Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available From 6f2bd98cc5037d2604944625a533e29e05147c37 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 00:10:20 +0800 Subject: [PATCH 119/185] docs(e2e): document agent v2 tag taxonomy --- e2e/features/agent-v2/AGENTS.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index a80ab5dc1e2572..87aa7c73eba88b 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -16,6 +16,12 @@ Use API setup for prerequisite state, then use Playwright only for user-observab ## Tags +Use tags in three layers: + +- Capability tags describe the product area: `@build`, `@files`, `@advanced-settings`, `@agent-edit`, `@publish`, `@access-point`, `@output-variables`, and similar tags. +- Execution-scope tags describe how the scenario should be selected: `@core`, `@infra`, `@web-app-runtime`, `@service-api-runtime`, `@preview`, and `@feature-gated`. +- Narrow fixture or sub-surface tags describe a specific dependency or slice: `@stable-model`, `@skill-fixture`, `@web-app-access`, `@workflow-reference`, `@files-limits`, and similar tags. + - `@agent-v2` — required capability tag for all Agent v2 scenarios. - `@core` — stable non-runtime scenario expected to run in the regular Agent v2 suite when its explicit preconditions are met. Do not apply `@core` to Preview/Test Run, Web app chat runtime, or Backend service API chat runtime scenarios. - `@infra` — infrastructure or readiness checks. @@ -31,6 +37,8 @@ Use API setup for prerequisite state, then use Playwright only for user-observab - `@service-api-runtime` — Backend service API runtime behavior. Use it for scenarios that call the published service API and assert real chat responses. Endpoint display, copy, API key, and API reference surfaces remain `@access-point` behavior. - `@feature-gated` — product capability is optional. This tag alone does not skip execution; the scenario must include an explicit step that returns `skipped` with a blocked-precondition reason when the feature is unavailable. +Use feature-level `@core` only when every scenario in the file is stable, non-runtime, and not feature-gated. If a feature file mixes stable scenarios with runtime, Preview, or feature-gated scenarios, put `@core` only on the stable scenarios. Keep runtime tags scenario-level so the regular core suite cannot inherit them accidentally. + ## Step Organization Keep Agent v2 step definitions grouped by user capability, not by DOM component or Cucumber keyword: From d9c0536e2d8754c5d9dbb872dac0aa3f10ba6c8b Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 00:14:38 +0800 Subject: [PATCH 120/185] docs(e2e): align cucumber playwright skill --- .agents/skills/e2e-cucumber-playwright/SKILL.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.agents/skills/e2e-cucumber-playwright/SKILL.md b/.agents/skills/e2e-cucumber-playwright/SKILL.md index dd7d204678c5c0..4596f9062ea716 100644 --- a/.agents/skills/e2e-cucumber-playwright/SKILL.md +++ b/.agents/skills/e2e-cucumber-playwright/SKILL.md @@ -5,7 +5,7 @@ description: Write, update, or review Dify end-to-end tests under `e2e/` that us # Dify E2E Cucumber + Playwright -Use this skill for Dify's repository-level E2E suite in `e2e/`. Use [`e2e/AGENTS.md`](../../../e2e/AGENTS.md) as the canonical guide for local architecture and conventions, then apply Playwright/Cucumber best practices only where they fit the current suite. +Use this skill for Dify's repository-level E2E suite in `e2e/`. Use [`e2e/AGENTS.md`](../../../e2e/AGENTS.md) as the canonical package guide for local architecture and conventions, then read any feature-scoped `AGENTS.md` that owns the target area. Apply Playwright/Cucumber best practices only where they fit the current suite. ## Scope @@ -25,6 +25,8 @@ Use this skill for Dify's repository-level E2E suite in `e2e/`. Use [`e2e/AGENTS 4. Read [`references/cucumber-best-practices.md`](references/cucumber-best-practices.md) only when scenario wording, step granularity, tags, or expression design are involved. 5. Re-check official Playwright or Cucumber docs with the available documentation tools before introducing a new framework pattern. +Keep this skill focused on Cucumber, Playwright, and package-level E2E guidance. Put feature-specific conventions in the owning feature's `AGENTS.md` instead of adding them here. + ## Local Rules - `e2e/` uses Cucumber for scenarios and Playwright as the browser layer. @@ -59,7 +61,7 @@ Use this skill for Dify's repository-level E2E suite in `e2e/`. Use [`e2e/AGENTS - Do not use `waitForTimeout`, manual polling, or raw visibility checks when a locator action or retrying assertion already expresses the behavior. 5. Validate narrowly. - Run the narrowest tagged scenario or flow that exercises the change. - - Run `pnpm -C e2e check`. + - Run `vpr lint --fix --quiet` from the repository root and `pnpm -C e2e type-check`. - Broaden verification only when the change affects hooks, tags, setup, or shared step semantics. ## Review Checklist From e90c03af57de9340d676a50413e22e8ec5d9b288 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 00:18:41 +0800 Subject: [PATCH 121/185] docs(e2e): document single runner constraint --- e2e/AGENTS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 599bd4021b75db..d2f62f308ecae8 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -30,6 +30,8 @@ pnpm -C e2e e2e:install `pnpm install` is resolved through the repository workspace and uses the shared root lockfile plus `pnpm-workspace.yaml`. +Run only one `pnpm -C e2e e2e*` process against a local workspace at a time. Separate runner processes share the frontend port, backend port, auth bootstrap state, and log paths; running them in parallel can create startup or authorization failures that are not scenario failures. + Use root lint plus the package type check as the default local verification step after editing E2E TypeScript, Cucumber support code, or feature glue: ```bash From c9b548fecc73ca5d99e6a958106b06c0c9a873d7 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 00:26:24 +0800 Subject: [PATCH 122/185] test(e2e): gate blocked agent v2 scenarios earlier --- e2e/features/agent-v2/agent-edit.feature | 1 + e2e/features/agent-v2/build-draft.feature | 1 + e2e/features/agent-v2/files.feature | 5 ++ .../agent-v2/output-variables.feature | 4 + .../agent-v2/agent-edit.steps.ts | 17 +++-- .../agent-v2/build-draft.steps.ts | 16 ++-- .../step-definitions/agent-v2/files.steps.ts | 67 +++++++++++------ .../agent-v2/output-variables.steps.ts | 73 +++++++++++-------- 8 files changed, 122 insertions(+), 62 deletions(-) diff --git a/e2e/features/agent-v2/agent-edit.feature b/e2e/features/agent-v2/agent-edit.feature index 398b22e9dc2c9e..2eb6d00f100760 100644 --- a/e2e/features/agent-v2/agent-edit.feature +++ b/e2e/features/agent-v2/agent-edit.feature @@ -34,6 +34,7 @@ Feature: Agent v2 Agent Edit page @tool-error-state @feature-gated Scenario: Tool credential error states are visible on the Agent Edit page Given I am signed in as the default E2E admin + And Agent v2 Tool credential error state is available And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" is available And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" includes the tool state fixture configuration When I open the preseeded Agent v2 configure page for "E2E New Agent Builder Tool States" from the Agent Roster diff --git a/e2e/features/agent-v2/build-draft.feature b/e2e/features/agent-v2/build-draft.feature index d0130256e6ddc5..5a2731b89128dd 100644 --- a/e2e/features/agent-v2/build-draft.feature +++ b/e2e/features/agent-v2/build-draft.feature @@ -137,6 +137,7 @@ Feature: Agent v2 build draft @build-tool-writeback @feature-gated Scenario: Applying a Build draft can add Dify Tools to the Agent configuration Given I am signed in as the default E2E admin + And Agent v2 Build chat Dify Tool writeback is available And a basic configured Agent v2 test agent has been created via API When I open the Agent v2 configure page Then Agent v2 Build chat Dify Tool writeback should be available diff --git a/e2e/features/agent-v2/files.feature b/e2e/features/agent-v2/files.feature index fdc42259ebf00a..6b4fb44851a80a 100644 --- a/e2e/features/agent-v2/files.feature +++ b/e2e/features/agent-v2/files.feature @@ -39,6 +39,7 @@ Feature: Agent v2 files @files-limits @feature-gated Scenario: Unsupported Agent v2 file formats show a clear rejection reason Given I am signed in as the default E2E admin + And Agent v2 unsupported file format rejection is available And a basic configured Agent v2 test agent has been created via API When I open the Agent v2 configure page Then Agent v2 unsupported file format rejection should be available @@ -46,6 +47,7 @@ Feature: Agent v2 files @files-limits @feature-gated Scenario: Oversized Agent v2 files show a clear rejection reason Given I am signed in as the default E2E admin + And Agent v2 oversized file rejection is available And a basic configured Agent v2 test agent has been created via API When I open the Agent v2 configure page Then Agent v2 oversized file rejection should be available @@ -53,6 +55,7 @@ Feature: Agent v2 files @files-limits @feature-gated Scenario: Agent v2 single-batch file count limits are enforced Given I am signed in as the default E2E admin + And Agent v2 single-batch file count limits are available And a basic configured Agent v2 test agent has been created via API When I open the Agent v2 configure page Then Agent v2 single-batch file count limits should be available @@ -60,6 +63,7 @@ Feature: Agent v2 files @files-limits @feature-gated Scenario: Agent v2 total file count limits are enforced Given I am signed in as the default E2E admin + And Agent v2 total file count limits are available And a basic configured Agent v2 test agent has been created via API When I open the Agent v2 configure page Then Agent v2 total file count limits should be available @@ -67,6 +71,7 @@ Feature: Agent v2 files @files-limits @feature-gated Scenario: Leaving during Agent v2 file upload keeps a recoverable state Given I am signed in as the default E2E admin + And Agent v2 in-progress file upload recovery is available And a basic configured Agent v2 test agent has been created via API When I open the Agent v2 configure page Then Agent v2 in-progress file upload recovery should be available diff --git a/e2e/features/agent-v2/output-variables.feature b/e2e/features/agent-v2/output-variables.feature index a498c43f742edb..aa351531ccb434 100644 --- a/e2e/features/agent-v2/output-variables.feature +++ b/e2e/features/agent-v2/output-variables.feature @@ -3,6 +3,7 @@ Feature: Agent v2 output variables @standalone-output-variables @feature-gated Scenario: Standalone Agent configure exposes Output Variables Given I am signed in as the default E2E admin + And Agent v2 standalone Output Variables are available And a basic configured Agent v2 test agent has been created via API When I open the Agent v2 configure page Then Agent v2 standalone Output Variables should be available @@ -55,6 +56,7 @@ Feature: Agent v2 output variables @output-reference-delete @feature-gated Scenario: Workflow Agent v2 prompt output reference deletion remains explicit Given I am signed in as the default E2E admin + And Agent v2 workflow task output reference deletion consistency is available And the Agent Builder stable chat model is available And a workflow app with an Agent v2 node has been created via API When I open the app from the app list @@ -65,6 +67,7 @@ Feature: Agent v2 output variables @output-retry-strategy @feature-gated Scenario: Workflow Agent v2 output retry strategy can be saved after refresh Given I am signed in as the default E2E admin + And Agent v2 workflow output retry strategy is available And the Agent Builder stable chat model is available And a workflow app with an Agent v2 node has been created via API When I open the app from the app list @@ -74,6 +77,7 @@ Feature: Agent v2 output variables @output-retry-validation @feature-gated Scenario: Workflow Agent v2 output retry count validation is enforced Given I am signed in as the default E2E admin + And Agent v2 workflow output retry count validation is available And the Agent Builder stable chat model is available And a workflow app with an Agent v2 node has been created via API When I open the app from the app list diff --git a/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts b/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts index e1fe3323495e0e..8ba3cab332f1e2 100644 --- a/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts +++ b/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts @@ -1,6 +1,6 @@ import type { PostAgentByAgentIdCopyResponse } from '@dify/contracts/api/console/agent/types.gen' import type { DifyWorld } from '../../support/world' -import { Then, When } from '@cucumber/cucumber' +import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { createE2EResourceName } from '../../../support/naming' import { @@ -231,18 +231,23 @@ Then('I should see the Agent v2 tool state fixture tools', async function (this: ) }) -Then('Agent v2 Tool credential error state should be available', async function (this: DifyWorld) { - const toolsSection = this.getPage().getByRole('region', { name: 'Tools' }) - - await expect(toolsSection).toBeVisible({ timeout: 30_000 }) +async function skipToolCredentialErrorState(world: DifyWorld) { return skipBlockedPrecondition( - this, + world, 'Agent v2 Tool credential error state is not covered: the current fixture only proves usable and not-authorized tool states.', { owner: 'seed/product', remediation: 'Define a stable invalid credential fixture and the expected user-visible error label before enabling this scenario.', }, ) +} + +Given('Agent v2 Tool credential error state is available', async function (this: DifyWorld) { + return skipToolCredentialErrorState(this) +}) + +Then('Agent v2 Tool credential error state should be available', async function (this: DifyWorld) { + return skipToolCredentialErrorState(this) }) Then('I should see the Agent v2 dual retrieval fixture settings', async function (this: DifyWorld) { diff --git a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts index 572cb4145b0b72..81c1d27f05eb42 100644 --- a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts +++ b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts @@ -153,19 +153,23 @@ When('I apply the Agent v2 Build draft', async function (this: DifyWorld) { await expect(page.getByText('Action succeeded')).toBeVisible() }) -Then('Agent v2 Build chat Dify Tool writeback should be available', async function (this: DifyWorld) { - const toolsSection = this.getPage().getByRole('region', { name: 'Tools' }) - - await expect(toolsSection).toBeVisible({ timeout: 30_000 }) - +async function skipBuildDraftToolWriteback(world: DifyWorld) { return skipBlockedPrecondition( - this, + world, 'Build draft Dify Tool writeback is not available: Build draft currently supports files, skills, and env only.', { owner: 'product', remediation: 'Define and implement Build draft Tool writeback before enabling this scenario.', }, ) +} + +Given('Agent v2 Build chat Dify Tool writeback is available', async function (this: DifyWorld) { + return skipBuildDraftToolWriteback(this) +}) + +Then('Agent v2 Build chat Dify Tool writeback should be available', async function (this: DifyWorld) { + return skipBuildDraftToolWriteback(this) }) Then('I should see the Agent v2 Build draft pending changes', async function (this: DifyWorld) { diff --git a/e2e/features/step-definitions/agent-v2/files.steps.ts b/e2e/features/step-definitions/agent-v2/files.steps.ts index d972b289c8cf33..e8780f5a032fb7 100644 --- a/e2e/features/step-definitions/agent-v2/files.steps.ts +++ b/e2e/features/step-definitions/agent-v2/files.steps.ts @@ -1,5 +1,5 @@ import type { DifyWorld } from '../../support/world' -import { Then, When } from '@cucumber/cucumber' +import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { skipBlockedPrecondition } from '../../agent-v2/support/preflight/common' import { agentBuilderFileTreeFixtureFileNames } from '../../agent-v2/support/test-materials' @@ -82,72 +82,97 @@ Then( }, ) -Then('Agent v2 unsupported file format rejection should be available', async function (this: DifyWorld) { - await expectFilesSectionVisible(this) - +async function skipUnsupportedFileFormatRejection(world: DifyWorld) { return skipBlockedPrecondition( - this, + world, 'Agent v2 unsupported file format rejection is not stable: default upload configuration allows arbitrary extensions unless UPLOAD_FILE_EXTENSION_BLACKLIST is seeded.', { owner: 'product/seed', remediation: 'Define Agent config file type restrictions or seed UPLOAD_FILE_EXTENSION_BLACKLIST before enabling this scenario.', }, ) +} + +Given('Agent v2 unsupported file format rejection is available', async function (this: DifyWorld) { + return skipUnsupportedFileFormatRejection(this) }) -Then('Agent v2 oversized file rejection should be available', async function (this: DifyWorld) { - await expectFilesSectionVisible(this) +Then('Agent v2 unsupported file format rejection should be available', async function (this: DifyWorld) { + return skipUnsupportedFileFormatRejection(this) +}) +async function skipOversizedFileRejection(world: DifyWorld) { return skipBlockedPrecondition( - this, + world, 'Agent v2 oversized file rejection lacks a clear user-visible reason: the current upload dialog collapses upload and commit failures into a generic failure toast.', { owner: 'product', remediation: 'Expose a stable user-visible file-size error before enabling this scenario.', }, ) +} + +Given('Agent v2 oversized file rejection is available', async function (this: DifyWorld) { + return skipOversizedFileRejection(this) }) -Then('Agent v2 single-batch file count limits should be available', async function (this: DifyWorld) { - await expectFilesSectionVisible(this) +Then('Agent v2 oversized file rejection should be available', async function (this: DifyWorld) { + return skipOversizedFileRejection(this) +}) +async function skipSingleBatchFileCountLimits(world: DifyWorld) { return skipBlockedPrecondition( - this, + world, 'Agent v2 single-batch file count limits are not reachable: the current Agent config file upload dialog accepts one file per upload.', { owner: 'product', remediation: 'Define multi-file upload behavior and its count-limit error before enabling this scenario.', }, ) +} + +Given('Agent v2 single-batch file count limits are available', async function (this: DifyWorld) { + return skipSingleBatchFileCountLimits(this) }) -Then('Agent v2 total file count limits should be available', async function (this: DifyWorld) { - await expectFilesSectionVisible(this) +Then('Agent v2 single-batch file count limits should be available', async function (this: DifyWorld) { + return skipSingleBatchFileCountLimits(this) +}) +async function skipTotalFileCountLimits(world: DifyWorld) { return skipBlockedPrecondition( - this, + world, 'Agent v2 total file count limits are not defined for Agent config files in the current product contract.', { owner: 'product', remediation: 'Define the Agent config file total-count limit and user-visible error before enabling this scenario.', }, ) +} + +Given('Agent v2 total file count limits are available', async function (this: DifyWorld) { + return skipTotalFileCountLimits(this) }) -Then('Agent v2 in-progress file upload recovery should be available', async function (this: DifyWorld) { - await expectFilesSectionVisible(this) +Then('Agent v2 total file count limits should be available', async function (this: DifyWorld) { + return skipTotalFileCountLimits(this) +}) +async function skipInProgressFileUploadRecovery(world: DifyWorld) { return skipBlockedPrecondition( - this, + world, 'Agent v2 in-progress file upload recovery is not stable: the current dialog has no deterministic slow-upload fixture or user-visible navigation guard contract.', { owner: 'product/test-infra', remediation: 'Define upload-in-progress navigation behavior and provide a deterministic slow upload fixture before enabling this scenario.', }, ) +} + +Given('Agent v2 in-progress file upload recovery is available', async function (this: DifyWorld) { + return skipInProgressFileUploadRecovery(this) }) -async function expectFilesSectionVisible(world: DifyWorld) { - await expect(world.getPage().getByRole('region', { name: 'Files' })) - .toBeVisible({ timeout: 30_000 }) -} +Then('Agent v2 in-progress file upload recovery should be available', async function (this: DifyWorld) { + return skipInProgressFileUploadRecovery(this) +}) diff --git a/e2e/features/step-definitions/agent-v2/output-variables.steps.ts b/e2e/features/step-definitions/agent-v2/output-variables.steps.ts index 6dde187d6d83c3..8161fdb714e216 100644 --- a/e2e/features/step-definitions/agent-v2/output-variables.steps.ts +++ b/e2e/features/step-definitions/agent-v2/output-variables.steps.ts @@ -1,70 +1,85 @@ import type { DifyWorld } from '../../support/world' -import { Then } from '@cucumber/cucumber' -import { expect } from '@playwright/test' +import { Given, Then } from '@cucumber/cucumber' import { skipBlockedPrecondition } from '../../agent-v2/support/preflight/common' -Then('Agent v2 standalone Output Variables should be available', async function (this: DifyWorld) { - const page = this.getPage() - - await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) - +async function skipStandaloneOutputVariables(world: DifyWorld) { return skipBlockedPrecondition( - this, + world, 'Standalone Agent Output Variables are not available: output variables currently belong to Workflow Agent v2 nodes.', { owner: 'product', remediation: 'Expose standalone Agent Output Variables or keep this scenario excluded until the product path exists.', }, ) -}) +} -Then('Agent v2 workflow output retry strategy should be available', async function (this: DifyWorld) { - const page = this.getPage() +Given('Agent v2 standalone Output Variables are available', async function (this: DifyWorld) { + return skipStandaloneOutputVariables(this) +}) - await expect(page.getByRole('button', { name: 'Output Variables' })).toBeVisible({ - timeout: 30_000, - }) +Then('Agent v2 standalone Output Variables should be available', async function (this: DifyWorld) { + return skipStandaloneOutputVariables(this) +}) +async function skipWorkflowOutputRetryStrategy(world: DifyWorld) { return skipBlockedPrecondition( - this, + world, 'Agent v2 workflow Output Variables retry strategy is not available in the current editor UI.', { owner: 'product', remediation: 'Expose user-visible retry strategy controls before enabling this scenario.', }, ) -}) +} -Then('Agent v2 workflow task output reference deletion consistency should be available', async function (this: DifyWorld) { - const page = this.getPage() +Given('Agent v2 workflow output retry strategy is available', async function (this: DifyWorld) { + return skipWorkflowOutputRetryStrategy(this) +}) - await expect(page.getByText('e2e_report.pdf', { exact: true })).toBeVisible({ - timeout: 30_000, - }) +Then('Agent v2 workflow output retry strategy should be available', async function (this: DifyWorld) { + return skipWorkflowOutputRetryStrategy(this) +}) +async function skipWorkflowTaskOutputReferenceDeletionConsistency(world: DifyWorld) { return skipBlockedPrecondition( - this, + world, 'Agent v2 workflow task output deletion consistency is not available: deleting an output from the list currently leaves the Prompt token without a stable user-visible invalid-reference state.', { owner: 'product', remediation: 'Define whether deletion should sync the Prompt token, block deletion, or expose an invalid-reference state before enabling this scenario.', }, ) -}) +} -Then('Agent v2 workflow output retry count validation should be available', async function (this: DifyWorld) { - const page = this.getPage() +Given( + 'Agent v2 workflow task output reference deletion consistency is available', + async function (this: DifyWorld) { + return skipWorkflowTaskOutputReferenceDeletionConsistency(this) + }, +) - await expect(page.getByRole('button', { name: 'Output Variables' })).toBeVisible({ - timeout: 30_000, - }) +Then( + 'Agent v2 workflow task output reference deletion consistency should be available', + async function (this: DifyWorld) { + return skipWorkflowTaskOutputReferenceDeletionConsistency(this) + }, +) +async function skipWorkflowOutputRetryCountValidation(world: DifyWorld) { return skipBlockedPrecondition( - this, + world, 'Agent v2 workflow Output Variables retry count validation is not reachable because retry strategy controls are not available in the current editor UI.', { owner: 'product', remediation: 'Expose retry count controls and validation states before enabling this scenario.', }, ) +} + +Given('Agent v2 workflow output retry count validation is available', async function (this: DifyWorld) { + return skipWorkflowOutputRetryCountValidation(this) +}) + +Then('Agent v2 workflow output retry count validation should be available', async function (this: DifyWorld) { + return skipWorkflowOutputRetryCountValidation(this) }) From af5ea20804f11625ff5db9114d183aa3012dd8ea Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 00:34:24 +0800 Subject: [PATCH 123/185] test(e2e): split agent access point steps --- e2e/features/agent-v2/AGENTS.md | 5 +- .../agent-v2/access-point-helpers.ts | 36 ++ .../access-point-service-api.steps.ts | 219 +++++++ .../agent-v2/access-point-web-app.steps.ts | 293 +++++++++ .../agent-v2/access-point-workflow.steps.ts | 70 ++ .../agent-v2/access-point.steps.ts | 607 +----------------- 6 files changed, 629 insertions(+), 601 deletions(-) create mode 100644 e2e/features/step-definitions/agent-v2/access-point-helpers.ts create mode 100644 e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts create mode 100644 e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts create mode 100644 e2e/features/step-definitions/agent-v2/access-point-workflow.steps.ts diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 87aa7c73eba88b..e79cd05926cf0b 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -51,7 +51,10 @@ Keep Agent v2 step definitions grouped by user capability, not by DOM component - `advanced-settings.steps.ts` — Env Editor, Content Moderation, and Advanced Settings behavior. - `agent-edit.steps.ts` — saved Agent detail display assertions. - `publish.steps.ts` — publish and publish-bar assertions. -- `access-point.steps.ts` — Access Point behavior. +- `access-point.steps.ts` — common Access Point navigation and overview. +- `access-point-web-app.steps.ts` — Web app access entrypoints and public Web app assertions. +- `access-point-service-api.steps.ts` — Backend service API entrypoints, keys, API reference, and service requests. +- `access-point-workflow.steps.ts` — Workflow access references. - `preflight.steps.ts` — explicit `Given` entrypoints for Agent Builder preflight resources. Cucumber step definitions are globally registered. Do not duplicate the same step text across files, even if one is written as `Given` and another as `Then`. diff --git a/e2e/features/step-definitions/agent-v2/access-point-helpers.ts b/e2e/features/step-definitions/agent-v2/access-point-helpers.ts new file mode 100644 index 00000000000000..0dc2f9ee2d6383 --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/access-point-helpers.ts @@ -0,0 +1,36 @@ +import type { DifyWorld } from '../../support/world' + +export const getCurrentAgentId = (world: DifyWorld) => { + const agentId = world.createdAgentIds.at(-1) + if (!agentId) + throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') + + return agentId +} + +export const getPreseededResource = ( + world: DifyWorld, + name: string, + kind: 'agent' | 'workflow', +) => { + const resource = world.agentBuilder.preflight.preseededResources[name] + if (!resource || resource.kind !== kind) { + throw new Error( + `Preseeded ${kind} "${name}" is not available. Run the matching preflight step first.`, + ) + } + + return resource +} + +export const getAccessRegion = (world: DifyWorld) => + world.getPage().getByRole('region', { name: 'Access Point' }) + +export const getWebAppCard = (world: DifyWorld) => + getAccessRegion(world).locator('article').filter({ hasText: 'Web app' }).first() + +export const getServiceApiCard = (world: DifyWorld) => + getAccessRegion(world).locator('article').filter({ hasText: 'Backend service API' }).first() + +export const getDialog = (world: DifyWorld, name: string | RegExp) => + world.getPage().getByRole('dialog', { name }) diff --git a/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts b/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts new file mode 100644 index 00000000000000..d54c43ae231715 --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts @@ -0,0 +1,219 @@ +import type { DifyWorld } from '../../support/world' +import { Given, Then, When } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { + createAgentApiKey, + publishAgent, + sendAgentServiceApiChatMessage, + setAgentApiAccess, +} from '../../agent-v2/support/agent' +import { agentBuilderExpectedTokens } from '../../agent-v2/support/agent-builder-resources' +import { getCurrentAgentId, getServiceApiCard } from './access-point-helpers' + +Given( + 'Agent v2 Backend service API access has been enabled via API', + async function (this: DifyWorld) { + const apiAccess = await setAgentApiAccess(getCurrentAgentId(this), true) + + this.agentBuilder.accessPoint.serviceApiBaseURL = apiAccess.service_api_base_url + }, +) + +Given('the Agent v2 draft has been published via API', async function (this: DifyWorld) { + await publishAgent(getCurrentAgentId(this)) +}) + +Given( + 'Agent v2 Backend service API access has been enabled with a key via API', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + const apiAccess = await setAgentApiAccess(agentId, true) + const apiKey = await createAgentApiKey(agentId) + + this.agentBuilder.accessPoint.serviceApiBaseURL = apiAccess.service_api_base_url + this.agentBuilder.accessPoint.generatedApiKey = apiKey.token + }, +) + +Then('I should see the Agent v2 Backend service API endpoint', async function (this: DifyWorld) { + const serviceApiCard = getServiceApiCard(this) + + if (!this.agentBuilder.accessPoint.serviceApiBaseURL) + throw new Error('No Agent v2 service API endpoint found. Enable Backend service API first.') + + await expect(serviceApiCard.getByRole('heading', { name: 'Backend service API' })).toBeVisible({ + timeout: 30_000, + }) + await expect(serviceApiCard.getByText('Service API Endpoint')).toBeVisible() + await expect(serviceApiCard.getByText(this.agentBuilder.accessPoint.serviceApiBaseURL)).toBeVisible() + await expect(serviceApiCard.getByLabel('Copy service API endpoint')).toBeEnabled() +}) + +When('I copy the Agent v2 Backend service API endpoint', async function (this: DifyWorld) { + await getServiceApiCard(this).getByLabel('Copy service API endpoint').click() +}) + +Then( + 'the Agent v2 Backend service API endpoint should show it was copied', + async function (this: DifyWorld) { + await expect(this.getPage().getByLabel('Copied')).toBeVisible() + }, +) + +When('I open Agent v2 API key management', async function (this: DifyWorld) { + await getServiceApiCard(this) + .getByRole('button', { name: /^API Key\b/ }) + .click() +}) + +Then('Agent v2 API keys should not expose a secret by default', async function (this: DifyWorld) { + const page = this.getPage() + const dialog = page.getByRole('dialog', { name: /API Secret key/i }) + + await expect(dialog).toBeVisible() + await expect(dialog.getByText('Secret Key', { exact: true })).toBeVisible() + await expect(dialog.getByText('CREATED', { exact: true })).toBeVisible() + await expect(dialog.getByText('LAST USED', { exact: true })).toBeVisible() + await expect(dialog.getByText('No data', { exact: true })).toBeVisible() + await expect(dialog.getByRole('button', { name: 'Create new Secret key' })).toBeVisible() + await expect(dialog.getByText(/^app-/)).not.toBeVisible() + await expect(page.getByRole('dialog', { name: 'Internal Server Error' })).not.toBeVisible() +}) + +When('I create a new Agent v2 API key', async function (this: DifyWorld) { + const dialog = this.getPage().getByRole('dialog', { name: /API Secret key/i }) + + await dialog.getByRole('button', { name: 'Create new Secret key' }).click() +}) + +Then('I should see the newly generated Agent v2 API key once', async function (this: DifyWorld) { + const generatedKeyDialog = this.getPage() + .getByRole('dialog', { name: /API Secret key/i }) + .last() + const generatedKey = generatedKeyDialog.getByText(/^app-/) + + await expect(generatedKeyDialog).toBeVisible() + await expect( + generatedKeyDialog.getByText('Keep this key in a secure and accessible place.'), + ).toBeVisible() + await expect(generatedKey).toBeVisible() + await expect(generatedKeyDialog.getByLabel('Copy')).toBeVisible() + + this.agentBuilder.accessPoint.generatedApiKey = (await generatedKey.textContent())?.trim() + if (!this.agentBuilder.accessPoint.generatedApiKey) + throw new Error('Generated Agent v2 API key was empty.') +}) + +When('I close the newly generated Agent v2 API key', async function (this: DifyWorld) { + const page = this.getPage() + const generatedKeyDialog = page.getByRole('dialog', { name: /API Secret key/i }).last() + + await generatedKeyDialog.getByRole('button', { name: 'OK' }).click() + await expect(page.getByText('Keep this key in a secure and accessible place.')).not.toBeVisible() +}) + +Then( + 'the Agent v2 API key list should not expose the full generated secret', + async function (this: DifyWorld) { + const fullSecret = this.agentBuilder.accessPoint.generatedApiKey + if (!fullSecret) + throw new Error('No generated Agent v2 API key found.') + + const apiKeyDialog = this.getPage().getByRole('dialog', { name: /API Secret key/i }) + + await expect(apiKeyDialog).toBeVisible() + await expect(apiKeyDialog.getByText(fullSecret, { exact: true })).not.toBeVisible() + await expect(apiKeyDialog.getByText(/^app-/)).not.toBeVisible() + await expect(apiKeyDialog.getByLabel('Copy')).toBeVisible() + }, +) + +When('I close Agent v2 API key management', async function (this: DifyWorld) { + const apiKeyDialog = this.getPage().getByRole('dialog', { name: /API Secret key/i }) + + await apiKeyDialog.getByLabel('Close').click() + await expect(apiKeyDialog).not.toBeVisible() +}) + +When('I open the Agent v2 API Reference', async function (this: DifyWorld) { + const page = this.getPage() + const apiReferenceLink = page.getByRole('link', { name: 'API Reference' }) + + await expect(apiReferenceLink).toBeVisible() + + const [apiReferencePage] = await Promise.all([ + page.waitForEvent('popup'), + apiReferenceLink.click(), + ]) + + this.agentBuilder.accessPoint.apiReferencePage = apiReferencePage +}) + +Then('the Agent v2 API Reference should open in a new tab', async function (this: DifyWorld) { + const apiReferencePage = this.agentBuilder.accessPoint.apiReferencePage + if (!apiReferencePage) + throw new Error('No Agent v2 API Reference page was opened.') + + await expect(apiReferencePage).toHaveURL(/developing-with-apis/) + await apiReferencePage.close() + this.agentBuilder.accessPoint.apiReferencePage = undefined +}) + +When('I disable Agent v2 Backend service API access', async function (this: DifyWorld) { + await getServiceApiCard(this).getByLabel('Toggle Backend service API access').click() +}) + +Then('Agent v2 Backend service API access should be out of service', async function (this: DifyWorld) { + const serviceApiCard = getServiceApiCard(this) + + await expect(serviceApiCard.getByText('Out of service')).toBeVisible({ timeout: 30_000 }) +}) + +When('I enable Agent v2 Backend service API access', async function (this: DifyWorld) { + await getServiceApiCard(this).getByLabel('Toggle Backend service API access').click() +}) + +Then('Agent v2 Backend service API access should be in service', async function (this: DifyWorld) { + const serviceApiCard = getServiceApiCard(this) + + await expect(serviceApiCard.getByText('In service')).toBeVisible({ timeout: 30_000 }) +}) + +When('I send the Agent v2 Backend service API minimal request', async function (this: DifyWorld) { + const serviceApiBaseURL = this.agentBuilder.accessPoint.serviceApiBaseURL + const apiKey = this.agentBuilder.accessPoint.generatedApiKey + if (!serviceApiBaseURL) + throw new Error('No Agent v2 service API endpoint found. Enable Backend service API first.') + if (!apiKey) + throw new Error('No Agent v2 API key found. Create a Backend service API key first.') + + this.agentBuilder.accessPoint.serviceApiResponse = await sendAgentServiceApiChatMessage({ + apiKey, + serviceApiBaseURL, + }) +}) + +Then( + 'the Agent v2 Backend service API request should be rejected while disabled', + async function (this: DifyWorld) { + const response = this.agentBuilder.accessPoint.serviceApiResponse + if (!response) + throw new Error('No Agent v2 Backend service API response was recorded.') + + expect(response.ok).toBe(false) + expect(response.status).toBe(403) + expect(JSON.stringify(response.body).toLowerCase()).toContain('disabled') + }, +) + +Then( + 'the Agent v2 Backend service API request should succeed with the normal E2E marker', + async function (this: DifyWorld) { + const response = this.agentBuilder.accessPoint.serviceApiResponse + if (!response) + throw new Error('No Agent v2 Backend service API response was recorded.') + + expect(response.ok).toBe(true) + expect(JSON.stringify(response.body)).toContain(agentBuilderExpectedTokens.agentReply) + }, +) diff --git a/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts b/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts new file mode 100644 index 00000000000000..4a0a007b6ff190 --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts @@ -0,0 +1,293 @@ +import type { DifyWorld } from '../../support/world' +import { Given, Then, When } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { + getAgentComposerDraft, + setAgentSiteAccessAndGetURL, +} from '../../agent-v2/support/agent' +import { agentBuilderExpectedTokens } from '../../agent-v2/support/agent-builder-resources' +import { + getCurrentAgentId, + getDialog, + getPreseededResource, + getWebAppCard, +} from './access-point-helpers' + +const closeDialog = async (world: DifyWorld, name: string | RegExp) => { + const dialog = getDialog(world, name) + + await dialog.getByLabel('Close').click() + await expect(dialog).not.toBeVisible() +} + +Given( + 'Agent v2 Web app access will be restored for {string}', + async function (this: DifyWorld, agentName: string) { + const agent = getPreseededResource(this, agentName, 'agent') + + this.registerCleanup(async () => { + await setAgentSiteAccessAndGetURL(agent.id, true) + }) + }, +) + +When( + 'Agent v2 Web app access has been enabled via API', + async function (this: DifyWorld) { + this.agentBuilder.accessPoint.webAppURL = await setAgentSiteAccessAndGetURL( + getCurrentAgentId(this), + true, + ) + }, +) + +Then('I should see the Agent v2 Web app access URL', async function (this: DifyWorld) { + const webAppCard = getWebAppCard(this) + + await expect(webAppCard.getByRole('heading', { name: 'Web app' })).toBeVisible() + await expect(webAppCard.getByText('Access URL')).toBeVisible() + await expect(webAppCard.getByLabel('Copy access URL')).toBeEnabled() + await expect(webAppCard.getByRole('link', { name: 'Launch' })).toBeVisible() +}) + +Then( + 'I record the Agent v2 orchestration draft for {string}', + async function (this: DifyWorld, agentName: string) { + const agent = getPreseededResource(this, agentName, 'agent') + const draft = await getAgentComposerDraft(agent.id) + + this.agentBuilder.accessPoint.composerDraftSnapshot = JSON.stringify(draft.agent_soul ?? {}) + }, +) + +When('I copy the Agent v2 Web app access URL', async function (this: DifyWorld) { + await getWebAppCard(this).getByLabel('Copy access URL').click() +}) + +Then('the Agent v2 Web app access URL should show it was copied', async function (this: DifyWorld) { + await expect(this.getPage().getByLabel('Copied')).toBeVisible() +}) + +When('I launch the Agent v2 Web app', async function (this: DifyWorld) { + const launchLink = getWebAppCard(this).getByRole('link', { name: 'Launch' }) + const href = await launchLink.getAttribute('href') + if (!href) + throw new Error('Agent v2 Web app Launch link does not expose an href.') + + const [webAppPage] = await Promise.all([ + this.getPage().waitForEvent('popup'), + launchLink.click(), + ]) + + this.agentBuilder.accessPoint.webAppURL = href + this.agentBuilder.accessPoint.webAppPage = webAppPage +}) + +When('I open the Agent v2 Web app URL', async function (this: DifyWorld) { + const webAppURL = this.agentBuilder.accessPoint.webAppURL + if (!webAppURL) + throw new Error('No Agent v2 Web app URL was recorded.') + if (!this.context) + throw new Error('Playwright browser context has not been initialized.') + + const webAppPage = await this.context.newPage() + await webAppPage.goto(webAppURL) + + this.agentBuilder.accessPoint.webAppPage = webAppPage +}) + +When('I send an E2E message in the Agent v2 Web app', async function (this: DifyWorld) { + const webAppPage = this.agentBuilder.accessPoint.webAppPage + if (!webAppPage) + throw new Error('No Agent v2 Web app page was opened.') + + const messageInput = webAppPage.getByRole('textbox').last() + await expect(messageInput).toBeEditable({ timeout: 30_000 }) + await messageInput.fill('Please reply with the test success marker.') + await messageInput.press('Enter') +}) + +Then('the Agent v2 Web app should open in a new tab', async function (this: DifyWorld) { + const webAppPage = this.agentBuilder.accessPoint.webAppPage + const webAppURL = this.agentBuilder.accessPoint.webAppURL + if (!webAppPage || !webAppURL) + throw new Error('No Agent v2 Web app page was opened.') + + await expect(webAppPage).toHaveURL(webAppURL) + await webAppPage.close() + this.agentBuilder.accessPoint.webAppPage = undefined + this.agentBuilder.accessPoint.webAppURL = undefined +}) + +Then( + 'the Agent v2 Web app response should include the updated E2E marker', + async function (this: DifyWorld) { + const webAppPage = this.agentBuilder.accessPoint.webAppPage + if (!webAppPage) + throw new Error('No Agent v2 Web app page was opened.') + + await expect(webAppPage.getByText(agentBuilderExpectedTokens.updatedAgentReply)) + .toBeVisible({ timeout: 120_000 }) + await webAppPage.close() + this.agentBuilder.accessPoint.webAppPage = undefined + }, +) + +Then( + 'the Agent v2 Web app response should include the normal E2E marker', + async function (this: DifyWorld) { + const webAppPage = this.agentBuilder.accessPoint.webAppPage + if (!webAppPage) + throw new Error('No Agent v2 Web app page was opened.') + + await expect(webAppPage.getByText(agentBuilderExpectedTokens.agentReply)) + .toBeVisible({ timeout: 120_000 }) + }, +) + +Then( + 'the Agent v2 Web app response should not include the updated E2E marker', + async function (this: DifyWorld) { + const webAppPage = this.agentBuilder.accessPoint.webAppPage + if (!webAppPage) + throw new Error('No Agent v2 Web app page was opened.') + + await expect(webAppPage.getByText(agentBuilderExpectedTokens.updatedAgentReply)) + .not + .toBeVisible() + await webAppPage.close() + this.agentBuilder.accessPoint.webAppPage = undefined + }, +) + +When('I open Agent v2 Embedded configuration', async function (this: DifyWorld) { + await getWebAppCard(this).getByRole('button', { name: 'Embedded' }).click() +}) + +Then('I should see the Agent v2 Embedded configuration dialog', async function (this: DifyWorld) { + const dialog = getDialog(this, 'Embed on website') + + await expect(dialog).toBeVisible() + await expect(dialog.getByText('Embed on website')).toBeVisible() + await expect(dialog.getByText(/iframe|script/i)).toBeVisible() + await closeDialog(this, 'Embed on website') +}) + +When('I open Agent v2 Web app customization', async function (this: DifyWorld) { + await getWebAppCard(this).getByRole('button', { name: 'Customize' }).click() +}) + +Then('I should see the Agent v2 Web app customization dialog', async function (this: DifyWorld) { + const dialog = getDialog(this, 'Customize AI web app') + + await expect(dialog).toBeVisible() + await expect(dialog.getByText('Customize AI web app')).toBeVisible() + await expect(dialog.getByText(/NEXT_PUBLIC_APP_ID|NEXT_PUBLIC_API_URL/)).toBeVisible() + await closeDialog(this, 'Customize AI web app') +}) + +When('I open Agent v2 Web app settings', async function (this: DifyWorld) { + await getWebAppCard(this).getByRole('button', { name: 'Settings' }).click() +}) + +Then('I should see the Agent v2 Web app settings dialog', async function (this: DifyWorld) { + const dialog = getDialog(this, 'Web App Settings') + + await expect(dialog).toBeVisible() + await expect(dialog.getByText('Web App Settings')).toBeVisible() + await expect(dialog.getByText('web app Name')).toBeVisible() + await expect(dialog.getByText('web app Description')).toBeVisible() + await closeDialog(this, 'Web App Settings') +}) + +Then( + 'the Agent v2 orchestration draft for {string} should be unchanged', + async function (this: DifyWorld, agentName: string) { + const snapshot = this.agentBuilder.accessPoint.composerDraftSnapshot + if (!snapshot) + throw new Error('No Agent v2 orchestration draft snapshot was recorded.') + + const agent = getPreseededResource(this, agentName, 'agent') + const draft = await getAgentComposerDraft(agent.id) + + expect(JSON.stringify(draft.agent_soul ?? {})).toBe(snapshot) + }, +) + +When('I disable Agent v2 Web app access', async function (this: DifyWorld) { + const webAppCard = getWebAppCard(this) + const launchLink = webAppCard.getByRole('link', { name: 'Launch' }) + const href = await launchLink.getAttribute('href') + if (!href) + throw new Error('Agent v2 Web app Launch link does not expose an href.') + + this.agentBuilder.accessPoint.webAppURL = href + + await webAppCard.getByLabel('Toggle Web app access').click() +}) + +Then('Agent v2 Web app access should be out of service', async function (this: DifyWorld) { + const webAppCard = getWebAppCard(this) + + await expect(webAppCard.getByText('Out of service')).toBeVisible() + await expect(webAppCard.getByRole('button', { name: 'Launch' })).toBeDisabled() +}) + +When('I open the disabled Agent v2 Web app URL', async function (this: DifyWorld) { + const webAppURL = this.agentBuilder.accessPoint.webAppURL + if (!webAppURL) + throw new Error('No Agent v2 Web app URL was recorded.') + if (!this.context) + throw new Error('Playwright browser context has not been initialized.') + + const webAppPage = await this.context.newPage() + await webAppPage.goto(webAppURL) + + this.agentBuilder.accessPoint.webAppPage = webAppPage +}) + +Then('the disabled Agent v2 Web app should show an unavailable state', async function (this: DifyWorld) { + const webAppPage = this.agentBuilder.accessPoint.webAppPage + if (!webAppPage) + throw new Error('No Agent v2 Web app page was opened.') + + await expect(webAppPage.getByText(/app is unavailable|site is disabled/i)).toBeVisible({ + timeout: 30_000, + }) + await webAppPage.close() + this.agentBuilder.accessPoint.webAppPage = undefined +}) + +When('I enable Agent v2 Web app access', async function (this: DifyWorld) { + await getWebAppCard(this).getByLabel('Toggle Web app access').click() +}) + +Then('Agent v2 Web app access should be in service', async function (this: DifyWorld) { + const webAppCard = getWebAppCard(this) + + await expect(webAppCard.getByText('In service')).toBeVisible() + await expect(webAppCard.getByRole('link', { name: 'Launch' })).toBeVisible() +}) + +When('I open the restored Agent v2 Web app URL', async function (this: DifyWorld) { + const webAppURL = this.agentBuilder.accessPoint.webAppURL + if (!webAppURL) + throw new Error('No Agent v2 Web app URL was recorded.') + if (!this.context) + throw new Error('Playwright browser context has not been initialized.') + + const webAppPage = await this.context.newPage() + await webAppPage.goto(webAppURL) + + this.agentBuilder.accessPoint.webAppPage = webAppPage +}) + +Then('the restored Agent v2 Web app should not show an unavailable state', async function (this: DifyWorld) { + const webAppPage = this.agentBuilder.accessPoint.webAppPage + if (!webAppPage) + throw new Error('No Agent v2 Web app page was opened.') + + await expect(webAppPage.getByText(/app is unavailable|site is disabled/i)).not.toBeVisible() + await webAppPage.close() + this.agentBuilder.accessPoint.webAppPage = undefined +}) diff --git a/e2e/features/step-definitions/agent-v2/access-point-workflow.steps.ts b/e2e/features/step-definitions/agent-v2/access-point-workflow.steps.ts new file mode 100644 index 00000000000000..e1f51d3b07ed72 --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/access-point-workflow.steps.ts @@ -0,0 +1,70 @@ +import type { DifyWorld } from '../../support/world' +import { Then, When } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { getAgentReferencingWorkflows } from '../../agent-v2/support/agent' +import { agentBuilderPreseededResources } from '../../agent-v2/support/agent-builder-resources' +import { getAccessRegion, getPreseededResource } from './access-point-helpers' + +Then( + 'I should see the Agent v2 Workflow access reference for {string}', + async function (this: DifyWorld, workflowName: string) { + const workflow = getPreseededResource(this, workflowName, 'workflow') + const agent = getPreseededResource( + this, + agentBuilderPreseededResources.workflowReferenceAgent, + 'agent', + ) + const references = await getAgentReferencingWorkflows(agent.id) + const reference = references.find(item => item.app_id === workflow.id || item.app_name === workflow.name) + if (!reference) + throw new Error(`Agent "${agent.name}" does not reference workflow "${workflow.name}".`) + + const accessRegion = getAccessRegion(this) + const workflowSection = accessRegion.getByRole('region', { name: 'Workflow access' }) + const row = workflowSection.getByRole('row').filter({ hasText: workflowName }) + const nodeCount = reference.node_ids?.length ?? 0 + + await expect(accessRegion.getByRole('columnheader', { name: 'Name' })).toBeVisible() + await expect(accessRegion.getByRole('columnheader', { name: 'Version' })).toBeVisible() + await expect(accessRegion.getByRole('columnheader', { name: 'Nodes' })).toBeVisible() + await expect(accessRegion.getByRole('columnheader', { name: 'Last updated' })).toBeVisible() + await expect(accessRegion.getByRole('columnheader', { name: 'Actions' })).toBeVisible() + await expect(row).toBeVisible({ timeout: 30_000 }) + await expect(row.getByText(reference.workflow_version, { exact: true })).toBeVisible() + await expect(row.getByText(new RegExp(`^${nodeCount} nodes?$`))).toBeVisible() + if (reference.app_updated_at == null) + await expect(row.getByText('N/A', { exact: true })).toBeVisible() + else + await expect(row.getByText('N/A', { exact: true })).not.toBeVisible() + await expect(row.getByRole('link', { name: `Open ${workflowName} in Studio` })).toBeVisible() + }, +) + +When( + 'I open the Agent v2 Workflow access reference for {string}', + async function (this: DifyWorld, workflowName: string) { + const workflowLink = this.getPage().getByRole('link', { name: `Open ${workflowName} in Studio` }) + + const [workflowPage] = await Promise.all([ + this.getPage().waitForEvent('popup'), + workflowLink.click(), + ]) + + this.agentBuilder.accessPoint.workflowReferencePage = workflowPage + }, +) + +Then( + 'the Agent v2 Workflow access reference for {string} should open in Studio', + async function (this: DifyWorld, workflowName: string) { + const workflowPage = this.agentBuilder.accessPoint.workflowReferencePage + if (!workflowPage) + throw new Error('No Agent v2 Workflow access reference page was opened.') + + const workflow = getPreseededResource(this, workflowName, 'workflow') + + await expect(workflowPage).toHaveURL(new RegExp(`/app/${workflow.id}/workflow(?:\\?.*)?$`)) + await workflowPage.close() + this.agentBuilder.accessPoint.workflowReferencePage = undefined + }, +) diff --git a/e2e/features/step-definitions/agent-v2/access-point.steps.ts b/e2e/features/step-definitions/agent-v2/access-point.steps.ts index c60eecd4c1a1da..c563379933f60c 100644 --- a/e2e/features/step-definitions/agent-v2/access-point.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point.steps.ts @@ -1,104 +1,12 @@ import type { DifyWorld } from '../../support/world' -import { Given, Then, When } from '@cucumber/cucumber' +import { Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' +import { getAgentAccessPath } from '../../agent-v2/support/agent' import { - createAgentApiKey, - getAgentAccessPath, - getAgentComposerDraft, - getAgentReferencingWorkflows, - publishAgent, - sendAgentServiceApiChatMessage, - setAgentApiAccess, - setAgentSiteAccessAndGetURL, -} from '../../agent-v2/support/agent' -import { - agentBuilderExpectedTokens, - agentBuilderPreseededResources, -} from '../../agent-v2/support/agent-builder-resources' - -const getCurrentAgentId = (world: DifyWorld) => { - const agentId = world.createdAgentIds.at(-1) - if (!agentId) - throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') - - return agentId -} - -const getPreseededResource = (world: DifyWorld, name: string, kind: 'agent' | 'workflow') => { - const resource = world.agentBuilder.preflight.preseededResources[name] - if (!resource || resource.kind !== kind) { - throw new Error( - `Preseeded ${kind} "${name}" is not available. Run the matching preflight step first.`, - ) - } - - return resource -} - -const getAccessRegion = (world: DifyWorld) => - world.getPage().getByRole('region', { name: 'Access Point' }) - -const getWebAppCard = (world: DifyWorld) => - getAccessRegion(world).locator('article').filter({ hasText: 'Web app' }).first() - -const getServiceApiCard = (world: DifyWorld) => - getAccessRegion(world).locator('article').filter({ hasText: 'Backend service API' }).first() - -const getDialog = (world: DifyWorld, name: string | RegExp) => - world.getPage().getByRole('dialog', { name }) - -const closeDialog = async (world: DifyWorld, name: string | RegExp) => { - const dialog = getDialog(world, name) - - await dialog.getByLabel('Close').click() - await expect(dialog).not.toBeVisible() -} - -Given( - 'Agent v2 Backend service API access has been enabled via API', - async function (this: DifyWorld) { - const apiAccess = await setAgentApiAccess(getCurrentAgentId(this), true) - - this.agentBuilder.accessPoint.serviceApiBaseURL = apiAccess.service_api_base_url - }, -) - -Given('the Agent v2 draft has been published via API', async function (this: DifyWorld) { - await publishAgent(getCurrentAgentId(this)) -}) - -Given( - 'Agent v2 Backend service API access has been enabled with a key via API', - async function (this: DifyWorld) { - const agentId = getCurrentAgentId(this) - const apiAccess = await setAgentApiAccess(agentId, true) - const apiKey = await createAgentApiKey(agentId) - - this.agentBuilder.accessPoint.serviceApiBaseURL = apiAccess.service_api_base_url - this.agentBuilder.accessPoint.generatedApiKey = apiKey.token - }, -) - -Given( - 'Agent v2 Web app access will be restored for {string}', - async function (this: DifyWorld, agentName: string) { - const agent = getPreseededResource(this, agentName, 'agent') - - this.registerCleanup(async () => { - await setAgentSiteAccessAndGetURL(agent.id, true) - }) - }, -) - -When( - 'Agent v2 Web app access has been enabled via API', - async function (this: DifyWorld) { - this.agentBuilder.accessPoint.webAppURL = await setAgentSiteAccessAndGetURL( - getCurrentAgentId(this), - true, - ) - }, -) + getAccessRegion, + getCurrentAgentId, + getPreseededResource, +} from './access-point-helpers' When('I open the Agent v2 Access Point page', async function (this: DifyWorld) { await this.getPage().goto(getAgentAccessPath(getCurrentAgentId(this))) @@ -131,8 +39,7 @@ When('I switch to the Agent v2 Access Point section', async function (this: Dify }) Then('I should see the Agent v2 Access Point overview', async function (this: DifyWorld) { - const page = this.getPage() - const accessRegion = page.getByRole('region', { name: 'Access Point' }) + const accessRegion = getAccessRegion(this) await expect(accessRegion).toBeVisible({ timeout: 30_000 }) await expect(accessRegion.getByRole('heading', { name: 'Access Point' })).toBeVisible() @@ -159,503 +66,3 @@ Then('I should see the Agent v2 Access Point overview', async function (this: Di await expect(accessRegion.getByRole('columnheader', { name: 'Actions' })).toBeVisible() await expect(accessRegion.getByText('No workflow references yet.')).toBeVisible() }) - -Then('I should see the Agent v2 Web app access URL', async function (this: DifyWorld) { - const webAppCard = getWebAppCard(this) - - await expect(webAppCard.getByRole('heading', { name: 'Web app' })).toBeVisible() - await expect(webAppCard.getByText('Access URL')).toBeVisible() - await expect(webAppCard.getByLabel('Copy access URL')).toBeEnabled() - await expect(webAppCard.getByRole('link', { name: 'Launch' })).toBeVisible() -}) - -Then( - 'I record the Agent v2 orchestration draft for {string}', - async function (this: DifyWorld, agentName: string) { - const agent = getPreseededResource(this, agentName, 'agent') - const draft = await getAgentComposerDraft(agent.id) - - this.agentBuilder.accessPoint.composerDraftSnapshot = JSON.stringify(draft.agent_soul ?? {}) - }, -) - -When('I copy the Agent v2 Web app access URL', async function (this: DifyWorld) { - await getWebAppCard(this).getByLabel('Copy access URL').click() -}) - -Then('the Agent v2 Web app access URL should show it was copied', async function (this: DifyWorld) { - await expect(this.getPage().getByLabel('Copied')).toBeVisible() -}) - -When('I launch the Agent v2 Web app', async function (this: DifyWorld) { - const launchLink = getWebAppCard(this).getByRole('link', { name: 'Launch' }) - const href = await launchLink.getAttribute('href') - if (!href) - throw new Error('Agent v2 Web app Launch link does not expose an href.') - - const [webAppPage] = await Promise.all([ - this.getPage().waitForEvent('popup'), - launchLink.click(), - ]) - - this.agentBuilder.accessPoint.webAppURL = href - this.agentBuilder.accessPoint.webAppPage = webAppPage -}) - -When('I open the Agent v2 Web app URL', async function (this: DifyWorld) { - const webAppURL = this.agentBuilder.accessPoint.webAppURL - if (!webAppURL) - throw new Error('No Agent v2 Web app URL was recorded.') - if (!this.context) - throw new Error('Playwright browser context has not been initialized.') - - const webAppPage = await this.context.newPage() - await webAppPage.goto(webAppURL) - - this.agentBuilder.accessPoint.webAppPage = webAppPage -}) - -When('I send an E2E message in the Agent v2 Web app', async function (this: DifyWorld) { - const webAppPage = this.agentBuilder.accessPoint.webAppPage - if (!webAppPage) - throw new Error('No Agent v2 Web app page was opened.') - - const messageInput = webAppPage.getByRole('textbox').last() - await expect(messageInput).toBeEditable({ timeout: 30_000 }) - await messageInput.fill('Please reply with the test success marker.') - await messageInput.press('Enter') -}) - -Then('the Agent v2 Web app should open in a new tab', async function (this: DifyWorld) { - const webAppPage = this.agentBuilder.accessPoint.webAppPage - const webAppURL = this.agentBuilder.accessPoint.webAppURL - if (!webAppPage || !webAppURL) - throw new Error('No Agent v2 Web app page was opened.') - - await expect(webAppPage).toHaveURL(webAppURL) - await webAppPage.close() - this.agentBuilder.accessPoint.webAppPage = undefined - this.agentBuilder.accessPoint.webAppURL = undefined -}) - -Then( - 'the Agent v2 Web app response should include the updated E2E marker', - async function (this: DifyWorld) { - const webAppPage = this.agentBuilder.accessPoint.webAppPage - if (!webAppPage) - throw new Error('No Agent v2 Web app page was opened.') - - await expect(webAppPage.getByText(agentBuilderExpectedTokens.updatedAgentReply)) - .toBeVisible({ timeout: 120_000 }) - await webAppPage.close() - this.agentBuilder.accessPoint.webAppPage = undefined - }, -) - -Then( - 'the Agent v2 Web app response should include the normal E2E marker', - async function (this: DifyWorld) { - const webAppPage = this.agentBuilder.accessPoint.webAppPage - if (!webAppPage) - throw new Error('No Agent v2 Web app page was opened.') - - await expect(webAppPage.getByText(agentBuilderExpectedTokens.agentReply)) - .toBeVisible({ timeout: 120_000 }) - }, -) - -Then( - 'the Agent v2 Web app response should not include the updated E2E marker', - async function (this: DifyWorld) { - const webAppPage = this.agentBuilder.accessPoint.webAppPage - if (!webAppPage) - throw new Error('No Agent v2 Web app page was opened.') - - await expect(webAppPage.getByText(agentBuilderExpectedTokens.updatedAgentReply)) - .not - .toBeVisible() - await webAppPage.close() - this.agentBuilder.accessPoint.webAppPage = undefined - }, -) - -When('I open Agent v2 Embedded configuration', async function (this: DifyWorld) { - await getWebAppCard(this).getByRole('button', { name: 'Embedded' }).click() -}) - -Then('I should see the Agent v2 Embedded configuration dialog', async function (this: DifyWorld) { - const dialog = getDialog(this, 'Embed on website') - - await expect(dialog).toBeVisible() - await expect(dialog.getByText('Embed on website')).toBeVisible() - await expect(dialog.getByText(/iframe|script/i)).toBeVisible() - await closeDialog(this, 'Embed on website') -}) - -When('I open Agent v2 Web app customization', async function (this: DifyWorld) { - await getWebAppCard(this).getByRole('button', { name: 'Customize' }).click() -}) - -Then('I should see the Agent v2 Web app customization dialog', async function (this: DifyWorld) { - const dialog = getDialog(this, 'Customize AI web app') - - await expect(dialog).toBeVisible() - await expect(dialog.getByText('Customize AI web app')).toBeVisible() - await expect(dialog.getByText(/NEXT_PUBLIC_APP_ID|NEXT_PUBLIC_API_URL/)).toBeVisible() - await closeDialog(this, 'Customize AI web app') -}) - -When('I open Agent v2 Web app settings', async function (this: DifyWorld) { - await getWebAppCard(this).getByRole('button', { name: 'Settings' }).click() -}) - -Then('I should see the Agent v2 Web app settings dialog', async function (this: DifyWorld) { - const dialog = getDialog(this, 'Web App Settings') - - await expect(dialog).toBeVisible() - await expect(dialog.getByText('Web App Settings')).toBeVisible() - await expect(dialog.getByText('web app Name')).toBeVisible() - await expect(dialog.getByText('web app Description')).toBeVisible() - await closeDialog(this, 'Web App Settings') -}) - -Then( - 'the Agent v2 orchestration draft for {string} should be unchanged', - async function (this: DifyWorld, agentName: string) { - const snapshot = this.agentBuilder.accessPoint.composerDraftSnapshot - if (!snapshot) - throw new Error('No Agent v2 orchestration draft snapshot was recorded.') - - const agent = getPreseededResource(this, agentName, 'agent') - const draft = await getAgentComposerDraft(agent.id) - - expect(JSON.stringify(draft.agent_soul ?? {})).toBe(snapshot) - }, -) - -When('I disable Agent v2 Web app access', async function (this: DifyWorld) { - const webAppCard = getWebAppCard(this) - const launchLink = webAppCard.getByRole('link', { name: 'Launch' }) - const href = await launchLink.getAttribute('href') - if (!href) - throw new Error('Agent v2 Web app Launch link does not expose an href.') - - this.agentBuilder.accessPoint.webAppURL = href - - await webAppCard.getByLabel('Toggle Web app access').click() -}) - -Then('Agent v2 Web app access should be out of service', async function (this: DifyWorld) { - const webAppCard = getWebAppCard(this) - - await expect(webAppCard.getByText('Out of service')).toBeVisible() - await expect(webAppCard.getByRole('button', { name: 'Launch' })).toBeDisabled() -}) - -When('I open the disabled Agent v2 Web app URL', async function (this: DifyWorld) { - const webAppURL = this.agentBuilder.accessPoint.webAppURL - if (!webAppURL) - throw new Error('No Agent v2 Web app URL was recorded.') - if (!this.context) - throw new Error('Playwright browser context has not been initialized.') - - const webAppPage = await this.context.newPage() - await webAppPage.goto(webAppURL) - - this.agentBuilder.accessPoint.webAppPage = webAppPage -}) - -Then('the disabled Agent v2 Web app should show an unavailable state', async function (this: DifyWorld) { - const webAppPage = this.agentBuilder.accessPoint.webAppPage - if (!webAppPage) - throw new Error('No Agent v2 Web app page was opened.') - - await expect(webAppPage.getByText(/app is unavailable|site is disabled/i)).toBeVisible({ - timeout: 30_000, - }) - await webAppPage.close() - this.agentBuilder.accessPoint.webAppPage = undefined -}) - -When('I enable Agent v2 Web app access', async function (this: DifyWorld) { - await getWebAppCard(this).getByLabel('Toggle Web app access').click() -}) - -Then('Agent v2 Web app access should be in service', async function (this: DifyWorld) { - const webAppCard = getWebAppCard(this) - - await expect(webAppCard.getByText('In service')).toBeVisible() - await expect(webAppCard.getByRole('link', { name: 'Launch' })).toBeVisible() -}) - -When('I open the restored Agent v2 Web app URL', async function (this: DifyWorld) { - const webAppURL = this.agentBuilder.accessPoint.webAppURL - if (!webAppURL) - throw new Error('No Agent v2 Web app URL was recorded.') - if (!this.context) - throw new Error('Playwright browser context has not been initialized.') - - const webAppPage = await this.context.newPage() - await webAppPage.goto(webAppURL) - - this.agentBuilder.accessPoint.webAppPage = webAppPage -}) - -Then('the restored Agent v2 Web app should not show an unavailable state', async function (this: DifyWorld) { - const webAppPage = this.agentBuilder.accessPoint.webAppPage - if (!webAppPage) - throw new Error('No Agent v2 Web app page was opened.') - - await expect(webAppPage.getByText(/app is unavailable|site is disabled/i)).not.toBeVisible() - await webAppPage.close() - this.agentBuilder.accessPoint.webAppPage = undefined -}) - -Then( - 'I should see the Agent v2 Workflow access reference for {string}', - async function (this: DifyWorld, workflowName: string) { - const page = this.getPage() - const workflow = getPreseededResource(this, workflowName, 'workflow') - const agent = getPreseededResource( - this, - agentBuilderPreseededResources.workflowReferenceAgent, - 'agent', - ) - const references = await getAgentReferencingWorkflows(agent.id) - const reference = references.find(item => item.app_id === workflow.id || item.app_name === workflow.name) - if (!reference) - throw new Error(`Agent "${agent.name}" does not reference workflow "${workflow.name}".`) - - const accessRegion = page.getByRole('region', { name: 'Access Point' }) - const workflowSection = accessRegion.getByRole('region', { name: 'Workflow access' }) - const row = workflowSection.getByRole('row').filter({ hasText: workflowName }) - const nodeCount = reference.node_ids?.length ?? 0 - - await expect(accessRegion.getByRole('columnheader', { name: 'Name' })).toBeVisible() - await expect(accessRegion.getByRole('columnheader', { name: 'Version' })).toBeVisible() - await expect(accessRegion.getByRole('columnheader', { name: 'Nodes' })).toBeVisible() - await expect(accessRegion.getByRole('columnheader', { name: 'Last updated' })).toBeVisible() - await expect(accessRegion.getByRole('columnheader', { name: 'Actions' })).toBeVisible() - await expect(row).toBeVisible({ timeout: 30_000 }) - await expect(row.getByText(reference.workflow_version, { exact: true })).toBeVisible() - await expect(row.getByText(new RegExp(`^${nodeCount} nodes?$`))).toBeVisible() - if (reference.app_updated_at == null) - await expect(row.getByText('N/A', { exact: true })).toBeVisible() - else - await expect(row.getByText('N/A', { exact: true })).not.toBeVisible() - await expect(row.getByRole('link', { name: `Open ${workflowName} in Studio` })).toBeVisible() - }, -) - -When( - 'I open the Agent v2 Workflow access reference for {string}', - async function (this: DifyWorld, workflowName: string) { - const page = this.getPage() - const workflowLink = page.getByRole('link', { name: `Open ${workflowName} in Studio` }) - - const [workflowPage] = await Promise.all([ - page.waitForEvent('popup'), - workflowLink.click(), - ]) - - this.agentBuilder.accessPoint.workflowReferencePage = workflowPage - }, -) - -Then( - 'the Agent v2 Workflow access reference for {string} should open in Studio', - async function (this: DifyWorld, workflowName: string) { - const workflowPage = this.agentBuilder.accessPoint.workflowReferencePage - if (!workflowPage) - throw new Error('No Agent v2 Workflow access reference page was opened.') - - const workflow = getPreseededResource(this, workflowName, 'workflow') - - await expect(workflowPage).toHaveURL(new RegExp(`/app/${workflow.id}/workflow(?:\\?.*)?$`)) - await workflowPage.close() - this.agentBuilder.accessPoint.workflowReferencePage = undefined - }, -) - -Then('I should see the Agent v2 Backend service API endpoint', async function (this: DifyWorld) { - const serviceApiCard = getServiceApiCard(this) - - if (!this.agentBuilder.accessPoint.serviceApiBaseURL) - throw new Error('No Agent v2 service API endpoint found. Enable Backend service API first.') - - await expect(serviceApiCard.getByRole('heading', { name: 'Backend service API' })).toBeVisible({ - timeout: 30_000, - }) - await expect(serviceApiCard.getByText('Service API Endpoint')).toBeVisible() - await expect(serviceApiCard.getByText(this.agentBuilder.accessPoint.serviceApiBaseURL)).toBeVisible() - await expect(serviceApiCard.getByLabel('Copy service API endpoint')).toBeEnabled() -}) - -When('I copy the Agent v2 Backend service API endpoint', async function (this: DifyWorld) { - await getServiceApiCard(this).getByLabel('Copy service API endpoint').click() -}) - -Then( - 'the Agent v2 Backend service API endpoint should show it was copied', - async function (this: DifyWorld) { - await expect(this.getPage().getByLabel('Copied')).toBeVisible() - }, -) - -When('I open Agent v2 API key management', async function (this: DifyWorld) { - await getServiceApiCard(this) - .getByRole('button', { name: /^API Key\b/ }) - .click() -}) - -Then('Agent v2 API keys should not expose a secret by default', async function (this: DifyWorld) { - const page = this.getPage() - const dialog = page.getByRole('dialog', { name: /API Secret key/i }) - - await expect(dialog).toBeVisible() - await expect(dialog.getByText('Secret Key', { exact: true })).toBeVisible() - await expect(dialog.getByText('CREATED', { exact: true })).toBeVisible() - await expect(dialog.getByText('LAST USED', { exact: true })).toBeVisible() - await expect(dialog.getByText('No data', { exact: true })).toBeVisible() - await expect(dialog.getByRole('button', { name: 'Create new Secret key' })).toBeVisible() - await expect(dialog.getByText(/^app-/)).not.toBeVisible() - await expect(page.getByRole('dialog', { name: 'Internal Server Error' })).not.toBeVisible() -}) - -When('I create a new Agent v2 API key', async function (this: DifyWorld) { - const dialog = this.getPage().getByRole('dialog', { name: /API Secret key/i }) - - await dialog.getByRole('button', { name: 'Create new Secret key' }).click() -}) - -Then('I should see the newly generated Agent v2 API key once', async function (this: DifyWorld) { - const generatedKeyDialog = this.getPage() - .getByRole('dialog', { name: /API Secret key/i }) - .last() - const generatedKey = generatedKeyDialog.getByText(/^app-/) - - await expect(generatedKeyDialog).toBeVisible() - await expect( - generatedKeyDialog.getByText('Keep this key in a secure and accessible place.'), - ).toBeVisible() - await expect(generatedKey).toBeVisible() - await expect(generatedKeyDialog.getByLabel('Copy')).toBeVisible() - - this.agentBuilder.accessPoint.generatedApiKey = (await generatedKey.textContent())?.trim() - if (!this.agentBuilder.accessPoint.generatedApiKey) - throw new Error('Generated Agent v2 API key was empty.') -}) - -When('I close the newly generated Agent v2 API key', async function (this: DifyWorld) { - const page = this.getPage() - const generatedKeyDialog = page.getByRole('dialog', { name: /API Secret key/i }).last() - - await generatedKeyDialog.getByRole('button', { name: 'OK' }).click() - await expect(page.getByText('Keep this key in a secure and accessible place.')).not.toBeVisible() -}) - -Then( - 'the Agent v2 API key list should not expose the full generated secret', - async function (this: DifyWorld) { - const fullSecret = this.agentBuilder.accessPoint.generatedApiKey - if (!fullSecret) - throw new Error('No generated Agent v2 API key found.') - - const apiKeyDialog = this.getPage().getByRole('dialog', { name: /API Secret key/i }) - - await expect(apiKeyDialog).toBeVisible() - await expect(apiKeyDialog.getByText(fullSecret, { exact: true })).not.toBeVisible() - await expect(apiKeyDialog.getByText(/^app-/)).not.toBeVisible() - await expect(apiKeyDialog.getByLabel('Copy')).toBeVisible() - }, -) - -When('I close Agent v2 API key management', async function (this: DifyWorld) { - const apiKeyDialog = this.getPage().getByRole('dialog', { name: /API Secret key/i }) - - await apiKeyDialog.getByLabel('Close').click() - await expect(apiKeyDialog).not.toBeVisible() -}) - -When('I open the Agent v2 API Reference', async function (this: DifyWorld) { - const page = this.getPage() - const apiReferenceLink = page.getByRole('link', { name: 'API Reference' }) - - await expect(apiReferenceLink).toBeVisible() - - const [apiReferencePage] = await Promise.all([ - page.waitForEvent('popup'), - apiReferenceLink.click(), - ]) - - this.agentBuilder.accessPoint.apiReferencePage = apiReferencePage -}) - -Then('the Agent v2 API Reference should open in a new tab', async function (this: DifyWorld) { - const apiReferencePage = this.agentBuilder.accessPoint.apiReferencePage - if (!apiReferencePage) - throw new Error('No Agent v2 API Reference page was opened.') - - await expect(apiReferencePage).toHaveURL(/developing-with-apis/) - await apiReferencePage.close() - this.agentBuilder.accessPoint.apiReferencePage = undefined -}) - -When('I disable Agent v2 Backend service API access', async function (this: DifyWorld) { - await getServiceApiCard(this).getByLabel('Toggle Backend service API access').click() -}) - -Then('Agent v2 Backend service API access should be out of service', async function (this: DifyWorld) { - const serviceApiCard = getServiceApiCard(this) - - await expect(serviceApiCard.getByText('Out of service')).toBeVisible({ timeout: 30_000 }) -}) - -When('I enable Agent v2 Backend service API access', async function (this: DifyWorld) { - await getServiceApiCard(this).getByLabel('Toggle Backend service API access').click() -}) - -Then('Agent v2 Backend service API access should be in service', async function (this: DifyWorld) { - const serviceApiCard = getServiceApiCard(this) - - await expect(serviceApiCard.getByText('In service')).toBeVisible({ timeout: 30_000 }) -}) - -When('I send the Agent v2 Backend service API minimal request', async function (this: DifyWorld) { - const serviceApiBaseURL = this.agentBuilder.accessPoint.serviceApiBaseURL - const apiKey = this.agentBuilder.accessPoint.generatedApiKey - if (!serviceApiBaseURL) - throw new Error('No Agent v2 service API endpoint found. Enable Backend service API first.') - if (!apiKey) - throw new Error('No Agent v2 API key found. Create a Backend service API key first.') - - this.agentBuilder.accessPoint.serviceApiResponse = await sendAgentServiceApiChatMessage({ - apiKey, - serviceApiBaseURL, - }) -}) - -Then( - 'the Agent v2 Backend service API request should be rejected while disabled', - async function (this: DifyWorld) { - const response = this.agentBuilder.accessPoint.serviceApiResponse - if (!response) - throw new Error('No Agent v2 Backend service API response was recorded.') - - expect(response.ok).toBe(false) - expect(response.status).toBe(403) - expect(JSON.stringify(response.body).toLowerCase()).toContain('disabled') - }, -) - -Then( - 'the Agent v2 Backend service API request should succeed with the normal E2E marker', - async function (this: DifyWorld) { - const response = this.agentBuilder.accessPoint.serviceApiResponse - if (!response) - throw new Error('No Agent v2 Backend service API response was recorded.') - - expect(response.ok).toBe(true) - expect(JSON.stringify(response.body)).toContain(agentBuilderExpectedTokens.agentReply) - }, -) From 5946b24d8e3edec15fb080e00f7eabc96799e6ed Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 00:41:04 +0800 Subject: [PATCH 124/185] test(e2e): clean uploaded agent drive skills --- e2e/features/step-definitions/agent-v2/configure.steps.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index d69e2aa6b47c34..3f1bfffe97f6e7 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -104,11 +104,15 @@ Given('the Agent v2 composer draft uses the normal E2E prompt', async function ( }) Given('the e2e-summary-skill Skill is available to the Agent v2 test agent', async function (this: DifyWorld) { - await uploadAgentDriveSkill({ - agentId: getCurrentAgentId(this), + const agentId = getCurrentAgentId(this) + const upload = await uploadAgentDriveSkill({ + agentId, fileName: agentBuilderTestMaterials.summarySkill, filePath: getAgentBuilderTestMaterialPath('summarySkill'), }) + this.createdAgentDriveFiles.push({ agentId, key: upload.skill.skill_md_key }) + if (upload.skill.archive_key) + this.createdAgentDriveFiles.push({ agentId, key: upload.skill.archive_key }) }) Then('the Agent v2 test agent should include drive skill {string}', async function (this: DifyWorld, skillName: string) { From b90422fecc3ab4a60269d44570575caafae018a4 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 00:46:14 +0800 Subject: [PATCH 125/185] test(e2e): split agent advanced settings steps --- e2e/features/agent-v2/AGENTS.md | 4 +- .../agent-v2/advanced-settings.steps.ts | 460 +----------------- .../agent-v2/content-moderation.steps.ts | 139 ++++++ .../agent-v2/env-editor.steps.ts | 310 ++++++++++++ 4 files changed, 454 insertions(+), 459 deletions(-) create mode 100644 e2e/features/step-definitions/agent-v2/content-moderation.steps.ts create mode 100644 e2e/features/step-definitions/agent-v2/env-editor.steps.ts diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index e79cd05926cf0b..5386b8fa3c91d3 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -48,7 +48,9 @@ Keep Agent v2 step definitions grouped by user capability, not by DOM component - `files.steps.ts` — Files upload, display, and fixture-list assertions. - `knowledge.steps.ts` — Knowledge Retrieval configuration persistence and reference cleanup. - `tools.steps.ts` — Tools selector, search, and configuration-boundary behavior. -- `advanced-settings.steps.ts` — Env Editor, Content Moderation, and Advanced Settings behavior. +- `advanced-settings.steps.ts` — common Advanced Settings shell and supported-entry assertions. +- `env-editor.steps.ts` — Env Editor add, import, delete, persistence, and restored-display behavior. +- `content-moderation.steps.ts` — Content Moderation availability, keyword settings, and feature-gated assertions. - `agent-edit.steps.ts` — saved Agent detail display assertions. - `publish.steps.ts` — publish and publish-bar assertions. - `access-point.steps.ts` — common Access Point navigation and overview. diff --git a/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts b/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts index 8f76ee64d9784e..0ac160dddf28e0 100644 --- a/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts +++ b/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts @@ -1,108 +1,7 @@ -import type { Locator } from '@playwright/test' import type { DifyWorld } from '../../support/world' import { Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' -import { getAgentComposerDraft } from '../../agent-v2/support/agent' -import { agentBuilderFixedInputs } from '../../agent-v2/support/agent-builder-resources' -import { skipBlockedPrecondition } from '../../agent-v2/support/preflight/common' -import { getAgentBuilderTestMaterialPath } from '../../agent-v2/support/test-materials' -import { - expectAgentEnvVariableHidden, - expectAgentEnvVariableVisible, - getAgentEnvVariables, - getAgentEnvVariableValue, - getCurrentAgentId, - getEnvVariableKey, - openAgentAdvancedSettings, -} from './configure-helpers' - -const getModerationSettingsDialog = (world: DifyWorld) => - world.getPage().getByRole('dialog').filter({ hasText: 'Content moderation settings' }) - -const getContentModerationRegion = (world: DifyWorld) => - world.getPage().getByRole('region', { name: 'Content moderation' }) - -const ensureSwitchChecked = async (switchLocator: Locator) => { - if (await switchLocator.getAttribute('aria-checked') !== 'true') - await switchLocator.click() -} - -When( - 'I add the plain Agent v2 environment variable from Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() - await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() - - await advancedSettings - .getByRole('textbox', { name: 'Key' }) - .fill(agentBuilderFixedInputs.envPlainKey) - await advancedSettings - .getByRole('textbox', { name: 'Value' }) - .fill(agentBuilderFixedInputs.envPlainValue) - await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() - }, -) - -When( - 'I import the invalid Agent v2 environment file from Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - const fileChooserPromise = page.waitForEvent('filechooser') - await advancedSettings.getByRole('button', { name: 'Import .env' }).click() - await (await fileChooserPromise).setFiles(getAgentBuilderTestMaterialPath('invalidEnv')) - }, -) - -When( - 'I add the secondary plain Agent v2 environment variable from Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await advancedSettings.getByRole('button', { name: 'Add environment variable' }).click() - await advancedSettings - .getByRole('textbox', { name: 'Key' }) - .last() - .fill(agentBuilderFixedInputs.envModeKey) - await advancedSettings - .getByRole('textbox', { name: 'Value' }) - .last() - .fill(agentBuilderFixedInputs.envModeValue) - await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(2) - }, -) - -When( - 'I delete the plain Agent v2 environment variable from Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await advancedSettings - .getByRole('button', { name: `Delete ${agentBuilderFixedInputs.envPlainKey}` }) - .click() - }, -) - -When( - 'I import the valid Agent v2 environment file from Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() - await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() - - const fileChooserPromise = page.waitForEvent('filechooser') - await advancedSettings.getByRole('button', { name: 'Import .env' }).click() - await (await fileChooserPromise).setFiles(getAgentBuilderTestMaterialPath('validEnv')) - }, -) +import { openAgentAdvancedSettings } from './configure-helpers' When('I expand Agent v2 Advanced Settings', async function (this: DifyWorld) { const page = this.getPage() @@ -112,45 +11,6 @@ When('I expand Agent v2 Advanced Settings', async function (this: DifyWorld) { await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() }) -When( - 'I configure Agent v2 Content Moderation keyword preset replies', - async function (this: DifyWorld) { - const page = this.getPage() - const contentModeration = getContentModerationRegion(this) - const enabledSwitch = contentModeration.getByRole('switch', { name: 'Content moderation' }) - - if (await enabledSwitch.getAttribute('aria-checked') === 'true') - await contentModeration.getByRole('button', { name: 'Settings' }).click() - else - await enabledSwitch.click() - - const dialog = getModerationSettingsDialog(this) - await expect(dialog).toBeVisible() - await dialog.getByRole('button', { name: 'Keywords' }).click() - await dialog - .getByRole('textbox', { name: 'Keywords' }) - .fill(agentBuilderFixedInputs.moderationKeyword) - - const inputModeration = dialog.getByRole('region', { name: 'Moderate INPUT Content' }) - const outputModeration = dialog.getByRole('region', { name: 'Moderate OUTPUT Content' }) - await ensureSwitchChecked(inputModeration.getByRole('switch', { name: 'Moderate INPUT Content' })) - - await dialog.getByRole('button', { name: 'Save' }).click() - await expect(page.getByText('Preset replies cannot be empty')).toBeVisible() - await expect(dialog).toBeVisible() - - await inputModeration - .getByRole('textbox', { name: 'Preset replies' }) - .fill(agentBuilderFixedInputs.inputModerationReply) - await ensureSwitchChecked(outputModeration.getByRole('switch', { name: 'Moderate OUTPUT Content' })) - await outputModeration - .getByRole('textbox', { name: 'Preset replies' }) - .fill(agentBuilderFixedInputs.outputModerationReply) - await dialog.getByRole('button', { name: 'Save' }).click() - await expect(dialog).not.toBeVisible() - }, -) - Then( 'Agent v2 Advanced Settings should describe supported entries while collapsed', async function (this: DifyWorld) { @@ -167,139 +27,10 @@ Then( }, ) -Then( - 'the plain Agent v2 environment variable should be saved in the Agent v2 draft', - async function (this: DifyWorld) { - const agentId = getCurrentAgentId(this) - - await expect - .poll(async () => { - const env = (await getAgentComposerDraft(agentId)).agent_soul?.env - const variable = env?.variables?.find(item => - getEnvVariableKey(item) === agentBuilderFixedInputs.envPlainKey, - ) - - return { - secretCount: env?.secret_refs?.length ?? 0, - value: variable?.value, - } - }, { - timeout: 30_000, - }) - .toEqual({ - secretCount: 0, - value: agentBuilderFixedInputs.envPlainValue, - }) - }, -) - -Then( - 'the Agent v2 environment variables for deletion should be saved in the Agent v2 draft', - async function (this: DifyWorld) { - const agentId = getCurrentAgentId(this) - - await expect - .poll(async () => { - const variables = await getAgentEnvVariables(agentId) - - return { - modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey), - plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), - } - }, { - timeout: 30_000, - }) - .toEqual({ - modeValue: agentBuilderFixedInputs.envModeValue, - plainValue: agentBuilderFixedInputs.envPlainValue, - }) - }, -) - -Then( - 'the plain Agent v2 environment variable should be removed from the Agent v2 draft', - async function (this: DifyWorld) { - const agentId = getCurrentAgentId(this) - - await expect - .poll(async () => { - const variables = await getAgentEnvVariables(agentId) - - return { - modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey), - plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), - } - }, { - timeout: 30_000, - }) - .toEqual({ - modeValue: agentBuilderFixedInputs.envModeValue, - plainValue: undefined, - }) - }, -) - -Then( - 'the valid Agent v2 environment import should be saved in the Agent v2 draft', - async function (this: DifyWorld) { - const agentId = getCurrentAgentId(this) - - await expect - .poll(async () => { - const variables = await getAgentEnvVariables(agentId) - - return { - modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey), - plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), - } - }, { - timeout: 30_000, - }) - .toEqual({ - modeValue: agentBuilderFixedInputs.envModeValue, - plainValue: agentBuilderFixedInputs.envPlainValue, - }) - }, -) - -Then( - 'the invalid Agent v2 environment import should report skipped lines', - async function (this: DifyWorld) { - await expect(this.getPage().getByText('2 invalid .env lines were skipped.')).toBeVisible() - }, -) - -Then( - 'the Agent v2 environment variables from the invalid import should be saved in the Agent v2 draft', - async function (this: DifyWorld) { - const agentId = getCurrentAgentId(this) - - await expect - .poll(async () => { - const variables = await getAgentEnvVariables(agentId) - - return { - importedValue: getAgentEnvVariableValue( - variables, - agentBuilderFixedInputs.envAfterInvalidImportKey, - ), - plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), - } - }, { - timeout: 30_000, - }) - .toEqual({ - importedValue: agentBuilderFixedInputs.envAfterInvalidImportValue, - plainValue: agentBuilderFixedInputs.envPlainValue, - }) - }, -) - Then( 'I should see the supported Agent v2 Advanced Settings entries', async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + const advancedSettings = await openAgentAdvancedSettings(this.getPage()) const envEditor = advancedSettings.getByRole('region', { name: 'Env Editor' }) await expect(envEditor).toBeVisible() @@ -311,190 +42,3 @@ Then( await expect(envEditor.getByText('Scope', { exact: true })).toBeVisible() }, ) - -Then('Agent v2 Content Moderation Settings should be available', async function (this: DifyWorld) { - const advancedSettings = this.getPage().getByRole('region', { name: 'Advanced Settings' }) - const contentModeration = advancedSettings.getByRole('region', { name: 'Content moderation' }) - - try { - await expect(contentModeration).toBeVisible({ timeout: 3_000 }) - } - catch { - return skipBlockedPrecondition( - this, - 'Agent v2 Content Moderation Settings is not available in this build.', - { - owner: 'product', - remediation: 'Enable ENABLE_AGENT_CONTENT_MODERATION or keep this scenario feature-gated.', - }, - ) - } -}) - -Then( - 'Agent v2 Content Moderation keyword preset replies should be saved in the Agent v2 draft', - async function (this: DifyWorld) { - const agentId = getCurrentAgentId(this) - - await expect - .poll(async () => { - const draft = await getAgentComposerDraft(agentId) - const appFeatures = draft.agent_soul?.app_features as Record | undefined - const moderation = appFeatures?.sensitive_word_avoidance as Record | undefined - const config = moderation?.config as Record | undefined - const inputsConfig = config?.inputs_config as Record | undefined - const outputsConfig = config?.outputs_config as Record | undefined - - return { - enabled: moderation?.enabled, - inputEnabled: inputsConfig?.enabled, - inputPreset: inputsConfig?.preset_response, - keywords: config?.keywords, - outputEnabled: outputsConfig?.enabled, - outputPreset: outputsConfig?.preset_response, - type: moderation?.type, - } - }, { - timeout: 30_000, - }) - .toEqual({ - enabled: true, - inputEnabled: true, - inputPreset: agentBuilderFixedInputs.inputModerationReply, - keywords: agentBuilderFixedInputs.moderationKeyword, - outputEnabled: true, - outputPreset: agentBuilderFixedInputs.outputModerationReply, - type: 'keywords', - }) - }, -) - -Then( - 'I should see the Agent v2 Content Moderation keyword preset replies in Advanced Settings', - async function (this: DifyWorld) { - const contentModeration = getContentModerationRegion(this) - - await expect(contentModeration).toContainText('Keywords') - await expect(contentModeration).toContainText('INPUT & OUTPUT') - await contentModeration.getByRole('button', { name: 'Settings' }).click() - - const dialog = getModerationSettingsDialog(this) - await expect(dialog).toBeVisible() - await expect(dialog.getByRole('textbox', { name: 'Keywords' })) - .toHaveValue(agentBuilderFixedInputs.moderationKeyword) - await expect(dialog.getByRole('region', { name: 'Moderate INPUT Content' }) - .getByRole('textbox', { name: 'Preset replies' })) - .toHaveValue(agentBuilderFixedInputs.inputModerationReply) - await expect(dialog.getByRole('region', { name: 'Moderate OUTPUT Content' }) - .getByRole('textbox', { name: 'Preset replies' })) - .toHaveValue(agentBuilderFixedInputs.outputModerationReply) - await dialog.getByRole('button', { name: 'Cancel' }).click() - await expect(dialog).not.toBeVisible() - }, -) - -Then( - 'I should see the Agent v2 environment variables from the valid import in Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() - await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() - await expect.poll( - async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => - inputs.map(input => (input as HTMLInputElement).value), - ), - { timeout: 30_000 }, - ).toEqual(expect.arrayContaining([ - agentBuilderFixedInputs.envPlainKey, - agentBuilderFixedInputs.envPlainValue, - agentBuilderFixedInputs.envModeKey, - agentBuilderFixedInputs.envModeValue, - ])) - await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(2) - }, -) - -Then( - 'I should not see the deleted Agent v2 environment variable in Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() - await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() - await expect.poll( - async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => - inputs.map(input => (input as HTMLInputElement).value), - ), - { timeout: 30_000 }, - ).toEqual(expect.arrayContaining([ - agentBuilderFixedInputs.envModeKey, - agentBuilderFixedInputs.envModeValue, - ])) - await expect.poll( - async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => - inputs.map(input => (input as HTMLInputElement).value), - ), - { timeout: 30_000 }, - ).not.toContain(agentBuilderFixedInputs.envPlainKey) - await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(1) - }, -) - -Then( - 'I should see the plain Agent v2 environment variable in Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = await openAgentAdvancedSettings(page) - - await expect(advancedSettings.getByRole('textbox', { name: 'Key' })) - .toHaveValue(agentBuilderFixedInputs.envPlainKey) - await expect(advancedSettings.getByRole('textbox', { name: 'Value' })) - .toHaveValue(agentBuilderFixedInputs.envPlainValue) - await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() - await expect(page.getByRole('button', { name: /^Build$/i })).toBeVisible() - }, -) - -Then( - 'I should see the supported E2E environment variable in Advanced Settings', - async function (this: DifyWorld) { - await expectAgentEnvVariableVisible( - this, - agentBuilderFixedInputs.envPlainKey, - agentBuilderFixedInputs.envPlainValue, - ) - }, -) - -Then( - 'I should not see the supported E2E environment variable in Advanced Settings', - async function (this: DifyWorld) { - await expectAgentEnvVariableHidden(this, agentBuilderFixedInputs.envPlainKey) - }, -) - -Then( - 'I should see the Agent v2 environment variables from the invalid import in Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() - await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() - await expect.poll( - async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => - inputs.map(input => (input as HTMLInputElement).value), - ), - { timeout: 30_000 }, - ).toEqual(expect.arrayContaining([ - agentBuilderFixedInputs.envPlainKey, - agentBuilderFixedInputs.envPlainValue, - agentBuilderFixedInputs.envAfterInvalidImportKey, - agentBuilderFixedInputs.envAfterInvalidImportValue, - ])) - await expect(page.getByRole('button', { name: /^Build$/i })).toBeVisible() - }, -) diff --git a/e2e/features/step-definitions/agent-v2/content-moderation.steps.ts b/e2e/features/step-definitions/agent-v2/content-moderation.steps.ts new file mode 100644 index 00000000000000..2987e030d85c0f --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/content-moderation.steps.ts @@ -0,0 +1,139 @@ +import type { Locator } from '@playwright/test' +import type { DifyWorld } from '../../support/world' +import { Then, When } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { getAgentComposerDraft } from '../../agent-v2/support/agent' +import { agentBuilderFixedInputs } from '../../agent-v2/support/agent-builder-resources' +import { skipBlockedPrecondition } from '../../agent-v2/support/preflight/common' +import { getCurrentAgentId } from './configure-helpers' + +const getModerationSettingsDialog = (world: DifyWorld) => + world.getPage().getByRole('dialog').filter({ hasText: 'Content moderation settings' }) + +const getContentModerationRegion = (world: DifyWorld) => + world.getPage().getByRole('region', { name: 'Content moderation' }) + +const ensureSwitchChecked = async (switchLocator: Locator) => { + if (await switchLocator.getAttribute('aria-checked') !== 'true') + await switchLocator.click() +} + +When( + 'I configure Agent v2 Content Moderation keyword preset replies', + async function (this: DifyWorld) { + const page = this.getPage() + const contentModeration = getContentModerationRegion(this) + const enabledSwitch = contentModeration.getByRole('switch', { name: 'Content moderation' }) + + if (await enabledSwitch.getAttribute('aria-checked') === 'true') + await contentModeration.getByRole('button', { name: 'Settings' }).click() + else + await enabledSwitch.click() + + const dialog = getModerationSettingsDialog(this) + await expect(dialog).toBeVisible() + await dialog.getByRole('button', { name: 'Keywords' }).click() + await dialog + .getByRole('textbox', { name: 'Keywords' }) + .fill(agentBuilderFixedInputs.moderationKeyword) + + const inputModeration = dialog.getByRole('region', { name: 'Moderate INPUT Content' }) + const outputModeration = dialog.getByRole('region', { name: 'Moderate OUTPUT Content' }) + await ensureSwitchChecked(inputModeration.getByRole('switch', { name: 'Moderate INPUT Content' })) + + await dialog.getByRole('button', { name: 'Save' }).click() + await expect(page.getByText('Preset replies cannot be empty')).toBeVisible() + await expect(dialog).toBeVisible() + + await inputModeration + .getByRole('textbox', { name: 'Preset replies' }) + .fill(agentBuilderFixedInputs.inputModerationReply) + await ensureSwitchChecked(outputModeration.getByRole('switch', { name: 'Moderate OUTPUT Content' })) + await outputModeration + .getByRole('textbox', { name: 'Preset replies' }) + .fill(agentBuilderFixedInputs.outputModerationReply) + await dialog.getByRole('button', { name: 'Save' }).click() + await expect(dialog).not.toBeVisible() + }, +) + +Then('Agent v2 Content Moderation Settings should be available', async function (this: DifyWorld) { + const advancedSettings = this.getPage().getByRole('region', { name: 'Advanced Settings' }) + const contentModeration = advancedSettings.getByRole('region', { name: 'Content moderation' }) + + try { + await expect(contentModeration).toBeVisible({ timeout: 3_000 }) + } + catch { + return skipBlockedPrecondition( + this, + 'Agent v2 Content Moderation Settings is not available in this build.', + { + owner: 'product', + remediation: 'Enable ENABLE_AGENT_CONTENT_MODERATION or keep this scenario feature-gated.', + }, + ) + } +}) + +Then( + 'Agent v2 Content Moderation keyword preset replies should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect + .poll(async () => { + const draft = await getAgentComposerDraft(agentId) + const appFeatures = draft.agent_soul?.app_features as Record | undefined + const moderation = appFeatures?.sensitive_word_avoidance as Record | undefined + const config = moderation?.config as Record | undefined + const inputsConfig = config?.inputs_config as Record | undefined + const outputsConfig = config?.outputs_config as Record | undefined + + return { + enabled: moderation?.enabled, + inputEnabled: inputsConfig?.enabled, + inputPreset: inputsConfig?.preset_response, + keywords: config?.keywords, + outputEnabled: outputsConfig?.enabled, + outputPreset: outputsConfig?.preset_response, + type: moderation?.type, + } + }, { + timeout: 30_000, + }) + .toEqual({ + enabled: true, + inputEnabled: true, + inputPreset: agentBuilderFixedInputs.inputModerationReply, + keywords: agentBuilderFixedInputs.moderationKeyword, + outputEnabled: true, + outputPreset: agentBuilderFixedInputs.outputModerationReply, + type: 'keywords', + }) + }, +) + +Then( + 'I should see the Agent v2 Content Moderation keyword preset replies in Advanced Settings', + async function (this: DifyWorld) { + const contentModeration = getContentModerationRegion(this) + + await expect(contentModeration).toContainText('Keywords') + await expect(contentModeration).toContainText('INPUT & OUTPUT') + await contentModeration.getByRole('button', { name: 'Settings' }).click() + + const dialog = getModerationSettingsDialog(this) + await expect(dialog).toBeVisible() + await expect(dialog.getByRole('textbox', { name: 'Keywords' })) + .toHaveValue(agentBuilderFixedInputs.moderationKeyword) + await expect(dialog.getByRole('region', { name: 'Moderate INPUT Content' }) + .getByRole('textbox', { name: 'Preset replies' })) + .toHaveValue(agentBuilderFixedInputs.inputModerationReply) + await expect(dialog.getByRole('region', { name: 'Moderate OUTPUT Content' }) + .getByRole('textbox', { name: 'Preset replies' })) + .toHaveValue(agentBuilderFixedInputs.outputModerationReply) + await dialog.getByRole('button', { name: 'Cancel' }).click() + await expect(dialog).not.toBeVisible() + }, +) diff --git a/e2e/features/step-definitions/agent-v2/env-editor.steps.ts b/e2e/features/step-definitions/agent-v2/env-editor.steps.ts new file mode 100644 index 00000000000000..6a18bfc93e9cf2 --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/env-editor.steps.ts @@ -0,0 +1,310 @@ +import type { DifyWorld } from '../../support/world' +import { Then, When } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { getAgentComposerDraft } from '../../agent-v2/support/agent' +import { agentBuilderFixedInputs } from '../../agent-v2/support/agent-builder-resources' +import { getAgentBuilderTestMaterialPath } from '../../agent-v2/support/test-materials' +import { + expectAgentEnvVariableHidden, + expectAgentEnvVariableVisible, + getAgentEnvVariables, + getAgentEnvVariableValue, + getCurrentAgentId, + getEnvVariableKey, + openAgentAdvancedSettings, +} from './configure-helpers' + +When( + 'I add the plain Agent v2 environment variable from Advanced Settings', + async function (this: DifyWorld) { + const advancedSettings = await openAgentAdvancedSettings(this.getPage()) + + await advancedSettings + .getByRole('textbox', { name: 'Key' }) + .fill(agentBuilderFixedInputs.envPlainKey) + await advancedSettings + .getByRole('textbox', { name: 'Value' }) + .fill(agentBuilderFixedInputs.envPlainValue) + await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() + }, +) + +When( + 'I import the invalid Agent v2 environment file from Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + const fileChooserPromise = page.waitForEvent('filechooser') + await advancedSettings.getByRole('button', { name: 'Import .env' }).click() + await (await fileChooserPromise).setFiles(getAgentBuilderTestMaterialPath('invalidEnv')) + }, +) + +When( + 'I add the secondary plain Agent v2 environment variable from Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await advancedSettings.getByRole('button', { name: 'Add environment variable' }).click() + await advancedSettings + .getByRole('textbox', { name: 'Key' }) + .last() + .fill(agentBuilderFixedInputs.envModeKey) + await advancedSettings + .getByRole('textbox', { name: 'Value' }) + .last() + .fill(agentBuilderFixedInputs.envModeValue) + await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(2) + }, +) + +When( + 'I delete the plain Agent v2 environment variable from Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await advancedSettings + .getByRole('button', { name: `Delete ${agentBuilderFixedInputs.envPlainKey}` }) + .click() + }, +) + +When( + 'I import the valid Agent v2 environment file from Advanced Settings', + async function (this: DifyWorld) { + const advancedSettings = await openAgentAdvancedSettings(this.getPage()) + + const fileChooserPromise = this.getPage().waitForEvent('filechooser') + await advancedSettings.getByRole('button', { name: 'Import .env' }).click() + await (await fileChooserPromise).setFiles(getAgentBuilderTestMaterialPath('validEnv')) + }, +) + +Then( + 'the plain Agent v2 environment variable should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect + .poll(async () => { + const env = (await getAgentComposerDraft(agentId)).agent_soul?.env + const variable = env?.variables?.find(item => + getEnvVariableKey(item) === agentBuilderFixedInputs.envPlainKey, + ) + + return { + secretCount: env?.secret_refs?.length ?? 0, + value: variable?.value, + } + }, { + timeout: 30_000, + }) + .toEqual({ + secretCount: 0, + value: agentBuilderFixedInputs.envPlainValue, + }) + }, +) + +Then( + 'the Agent v2 environment variables for deletion should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect + .poll(async () => { + const variables = await getAgentEnvVariables(agentId) + + return { + modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey), + plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), + } + }, { + timeout: 30_000, + }) + .toEqual({ + modeValue: agentBuilderFixedInputs.envModeValue, + plainValue: agentBuilderFixedInputs.envPlainValue, + }) + }, +) + +Then( + 'the plain Agent v2 environment variable should be removed from the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect + .poll(async () => { + const variables = await getAgentEnvVariables(agentId) + + return { + modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey), + plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), + } + }, { + timeout: 30_000, + }) + .toEqual({ + modeValue: agentBuilderFixedInputs.envModeValue, + plainValue: undefined, + }) + }, +) + +Then( + 'the valid Agent v2 environment import should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect + .poll(async () => { + const variables = await getAgentEnvVariables(agentId) + + return { + modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey), + plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), + } + }, { + timeout: 30_000, + }) + .toEqual({ + modeValue: agentBuilderFixedInputs.envModeValue, + plainValue: agentBuilderFixedInputs.envPlainValue, + }) + }, +) + +Then( + 'the invalid Agent v2 environment import should report skipped lines', + async function (this: DifyWorld) { + await expect(this.getPage().getByText('2 invalid .env lines were skipped.')).toBeVisible() + }, +) + +Then( + 'the Agent v2 environment variables from the invalid import should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect + .poll(async () => { + const variables = await getAgentEnvVariables(agentId) + + return { + importedValue: getAgentEnvVariableValue( + variables, + agentBuilderFixedInputs.envAfterInvalidImportKey, + ), + plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), + } + }, { + timeout: 30_000, + }) + .toEqual({ + importedValue: agentBuilderFixedInputs.envAfterInvalidImportValue, + plainValue: agentBuilderFixedInputs.envPlainValue, + }) + }, +) + +Then( + 'I should see the Agent v2 environment variables from the valid import in Advanced Settings', + async function (this: DifyWorld) { + const advancedSettings = await openAgentAdvancedSettings(this.getPage()) + + await expect.poll( + async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => + inputs.map(input => (input as HTMLInputElement).value), + ), + { timeout: 30_000 }, + ).toEqual(expect.arrayContaining([ + agentBuilderFixedInputs.envPlainKey, + agentBuilderFixedInputs.envPlainValue, + agentBuilderFixedInputs.envModeKey, + agentBuilderFixedInputs.envModeValue, + ])) + await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(2) + }, +) + +Then( + 'I should not see the deleted Agent v2 environment variable in Advanced Settings', + async function (this: DifyWorld) { + const advancedSettings = await openAgentAdvancedSettings(this.getPage()) + + await expect.poll( + async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => + inputs.map(input => (input as HTMLInputElement).value), + ), + { timeout: 30_000 }, + ).toEqual(expect.arrayContaining([ + agentBuilderFixedInputs.envModeKey, + agentBuilderFixedInputs.envModeValue, + ])) + await expect.poll( + async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => + inputs.map(input => (input as HTMLInputElement).value), + ), + { timeout: 30_000 }, + ).not.toContain(agentBuilderFixedInputs.envPlainKey) + await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(1) + }, +) + +Then( + 'I should see the plain Agent v2 environment variable in Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = await openAgentAdvancedSettings(page) + + await expect(advancedSettings.getByRole('textbox', { name: 'Key' })) + .toHaveValue(agentBuilderFixedInputs.envPlainKey) + await expect(advancedSettings.getByRole('textbox', { name: 'Value' })) + .toHaveValue(agentBuilderFixedInputs.envPlainValue) + await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() + await expect(page.getByRole('button', { name: /^Build$/i })).toBeVisible() + }, +) + +Then( + 'I should see the supported E2E environment variable in Advanced Settings', + async function (this: DifyWorld) { + await expectAgentEnvVariableVisible( + this, + agentBuilderFixedInputs.envPlainKey, + agentBuilderFixedInputs.envPlainValue, + ) + }, +) + +Then( + 'I should not see the supported E2E environment variable in Advanced Settings', + async function (this: DifyWorld) { + await expectAgentEnvVariableHidden(this, agentBuilderFixedInputs.envPlainKey) + }, +) + +Then( + 'I should see the Agent v2 environment variables from the invalid import in Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = await openAgentAdvancedSettings(page) + + await expect.poll( + async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => + inputs.map(input => (input as HTMLInputElement).value), + ), + { timeout: 30_000 }, + ).toEqual(expect.arrayContaining([ + agentBuilderFixedInputs.envPlainKey, + agentBuilderFixedInputs.envPlainValue, + agentBuilderFixedInputs.envAfterInvalidImportKey, + agentBuilderFixedInputs.envAfterInvalidImportValue, + ])) + await expect(page.getByRole('button', { name: /^Build$/i })).toBeVisible() + }, +) From f1160999719e423ad126df78161e1b9a0e6450f1 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 00:49:15 +0800 Subject: [PATCH 126/185] test(e2e): gate build unavailable resource recovery --- e2e/features/agent-v2/AGENTS.md | 1 + e2e/features/agent-v2/build-draft.feature | 9 +++++++++ .../agent-v2/build-draft.steps.ts | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 5386b8fa3c91d3..35a642625bf02e 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -26,6 +26,7 @@ Use tags in three layers: - `@core` — stable non-runtime scenario expected to run in the regular Agent v2 suite when its explicit preconditions are met. Do not apply `@core` to Preview/Test Run, Web app chat runtime, or Backend service API chat runtime scenarios. - `@infra` — infrastructure or readiness checks. - `@build` — Build mode and Build draft behavior. +- `@build-unavailable-resources` — feature-gated Build chat recovery when the user requests unavailable Skills or Tools. - `@files` — Files section upload, display, and fixture behavior. - `@files-limits` — feature-gated file format, size, count, and in-progress upload limit behavior. - `@knowledge` — Knowledge Retrieval configuration display, persistence, and reference cleanup. diff --git a/e2e/features/agent-v2/build-draft.feature b/e2e/features/agent-v2/build-draft.feature index 5a2731b89128dd..dc4b8353c6ff26 100644 --- a/e2e/features/agent-v2/build-draft.feature +++ b/e2e/features/agent-v2/build-draft.feature @@ -141,3 +141,12 @@ Feature: Agent v2 build draft And a basic configured Agent v2 test agent has been created via API When I open the Agent v2 configure page Then Agent v2 Build chat Dify Tool writeback should be available + + @build-unavailable-resources @feature-gated + Scenario: Build chat reports unavailable Skill or Tool requests clearly + Given I am signed in as the default E2E admin + And Agent v2 Build chat unavailable Skill and Tool recovery is available + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API + When I open the Agent v2 configure page + Then Agent v2 Build chat unavailable Skill and Tool recovery should be available diff --git a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts index 81c1d27f05eb42..a646a92f33f105 100644 --- a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts +++ b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts @@ -172,6 +172,25 @@ Then('Agent v2 Build chat Dify Tool writeback should be available', async functi return skipBuildDraftToolWriteback(this) }) +async function skipBuildDraftUnavailableResourceRecovery(world: DifyWorld) { + return skipBlockedPrecondition( + world, + 'Build chat unavailable Skill/Tool recovery is not covered: the product needs a stable user-visible failure state and deterministic request fixture before this can be automated.', + { + owner: 'product/seed', + remediation: 'Define the unavailable-resource UX contract, then seed a stable model-backed prompt that requests a missing Skill and Tool without mutating the saved Agent config.', + }, + ) +} + +Given('Agent v2 Build chat unavailable Skill and Tool recovery is available', async function (this: DifyWorld) { + return skipBuildDraftUnavailableResourceRecovery(this) +}) + +Then('Agent v2 Build chat unavailable Skill and Tool recovery should be available', async function (this: DifyWorld) { + return skipBuildDraftUnavailableResourceRecovery(this) +}) + Then('I should see the Agent v2 Build draft pending changes', async function (this: DifyWorld) { const page = this.getPage() From 46abc349986e61d4722c64dff2d9f439cd0fb9cf Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 00:55:26 +0800 Subject: [PATCH 127/185] test(e2e): cover agent roster creation --- e2e/features/agent-v2/AGENTS.md | 2 + e2e/features/agent-v2/roster-create.feature | 7 +++ .../agent-v2/agent-roster.steps.ts | 48 +++++++++++++++++++ 3 files changed, 57 insertions(+) create mode 100644 e2e/features/agent-v2/roster-create.feature create mode 100644 e2e/features/step-definitions/agent-v2/agent-roster.steps.ts diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 35a642625bf02e..f25b7bb6c255c4 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -31,6 +31,7 @@ Use tags in three layers: - `@files-limits` — feature-gated file format, size, count, and in-progress upload limit behavior. - `@knowledge` — Knowledge Retrieval configuration display, persistence, and reference cleanup. - `@advanced-settings` — Env Editor, Content Moderation, and related Advanced Settings behavior. +- `@agent-create` — Agent Roster creation and initial Configure navigation. - `@agent-edit` — saved Agent detail/configuration display surfaces. - `@publish` — publish and publish-bar state. - `@access-point` — Web app, Backend service API, and Workflow access surfaces. @@ -52,6 +53,7 @@ Keep Agent v2 step definitions grouped by user capability, not by DOM component - `advanced-settings.steps.ts` — common Advanced Settings shell and supported-entry assertions. - `env-editor.steps.ts` — Env Editor add, import, delete, persistence, and restored-display behavior. - `content-moderation.steps.ts` — Content Moderation availability, keyword settings, and feature-gated assertions. +- `agent-roster.steps.ts` — Agent Roster creation and Roster-level user actions. - `agent-edit.steps.ts` — saved Agent detail display assertions. - `publish.steps.ts` — publish and publish-bar assertions. - `access-point.steps.ts` — common Access Point navigation and overview. diff --git a/e2e/features/agent-v2/roster-create.feature b/e2e/features/agent-v2/roster-create.feature new file mode 100644 index 00000000000000..8757be00554897 --- /dev/null +++ b/e2e/features/agent-v2/roster-create.feature @@ -0,0 +1,7 @@ +@agent-v2 @authenticated @agent-create @core +Feature: Agent v2 Roster creation + Scenario: Create an Agent from the Agent Roster + Given I am signed in as the default E2E admin + When I create an Agent v2 test agent from the Agent Roster + Then the created Agent v2 should open in Configure + And I should see the Agent v2 configure workspace diff --git a/e2e/features/step-definitions/agent-v2/agent-roster.steps.ts b/e2e/features/step-definitions/agent-v2/agent-roster.steps.ts new file mode 100644 index 00000000000000..99e69c1ba13009 --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/agent-roster.steps.ts @@ -0,0 +1,48 @@ +import type { AgentAppDetailWithSite } from '@dify/contracts/api/console/agent/types.gen' +import type { DifyWorld } from '../../support/world' +import { Then, When } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { createE2EResourceName } from '../../../support/naming' + +When('I create an Agent v2 test agent from the Agent Roster', async function (this: DifyWorld) { + const page = this.getPage() + const agentName = createE2EResourceName('Agent', 'roster-ui') + const agentRole = 'E2E roster-created assistant' + const agentDescription = 'Created by Dify E2E through the Agent Roster UI.' + + await page.goto('/roster') + await page.getByRole('button', { name: 'Create agent' }).click() + + const dialog = page.getByRole('dialog', { name: 'Create agent' }) + await expect(dialog).toBeVisible() + await dialog.getByRole('textbox', { name: 'Name' }).fill(agentName) + await dialog.getByRole('textbox', { name: 'Role' }).fill(agentRole) + await dialog.getByRole('textbox', { name: 'Description' }).fill(agentDescription) + + const createResponsePromise = page.waitForResponse(response => ( + response.request().method() === 'POST' + && new URL(response.url()).pathname.endsWith('/console/api/agent') + )) + + await dialog.getByRole('button', { name: 'Create' }).click() + const createResponse = await createResponsePromise + expect(createResponse.ok()).toBe(true) + + const createdAgent = await createResponse.json() as AgentAppDetailWithSite + this.createdAgentIds.push(createdAgent.id) + this.lastCreatedAgentName = createdAgent.name + this.lastCreatedAgentRole = createdAgent.role ?? undefined +}) + +Then('the created Agent v2 should open in Configure', async function (this: DifyWorld) { + const agentId = this.createdAgentIds.at(-1) + if (!agentId) + throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') + + await expect(this.getPage()).toHaveURL( + new RegExp(`/roster/agent/${agentId}/configure(?:\\?.*)?$`), + { timeout: 30_000 }, + ) + await expect(this.getPage().getByRole('heading', { name: 'Configure' })) + .toBeVisible({ timeout: 30_000 }) +}) From c83442424b549a7b8d3a82483c38f535f91a2a1e Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 00:55:55 +0800 Subject: [PATCH 128/185] docs(e2e): align lint command --- e2e/AGENTS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index d2f62f308ecae8..6b7b10de1950aa 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -35,7 +35,7 @@ Run only one `pnpm -C e2e e2e*` process against a local workspace at a time. Sep Use root lint plus the package type check as the default local verification step after editing E2E TypeScript, Cucumber support code, or feature glue: ```bash -vpr lint --fix +vpr lint --fix --quiet pnpm -C e2e type-check ``` @@ -180,7 +180,7 @@ open cucumber-report/report.html 1. Add step definitions under `features/step-definitions//` 1. Reuse existing steps from `common/` and other definition files before writing new ones 1. Run with `pnpm -C e2e e2e -- --tags @your-tag` to verify -1. Run `vpr lint --fix` from the repository root and `pnpm -C e2e type-check` before committing +1. Run `vpr lint --fix --quiet` from the repository root and `pnpm -C e2e type-check` before committing ### Feature file conventions From 0272ffbe9db2464a3fae7a6465ae832bf6d2dacb Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 01:04:58 +0800 Subject: [PATCH 129/185] test(e2e): split agent access support --- e2e/features/agent-v2/AGENTS.md | 2 +- e2e/features/agent-v2/support/access-point.ts | 104 ++++++++++++++ e2e/features/agent-v2/support/agent.ts | 129 +----------------- .../access-point-service-api.steps.ts | 4 +- .../agent-v2/access-point-web-app.steps.ts | 6 +- 5 files changed, 110 insertions(+), 135 deletions(-) create mode 100644 e2e/features/agent-v2/support/access-point.ts diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index f25b7bb6c255c4..1091da41c77736 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -87,7 +87,7 @@ Use the existing namespace shape: - `world.agentBuilder.workflow.agentConsolePage` - `world.agentBuilder.workflow.outputVariables` -Use `features/agent-v2/support/agent.ts` for Agent v2 API fixtures. It owns roster-shaped Agent IDs, configure/access route helpers, composer draft sync, build-draft helpers, publish, API access toggles, Agent drive file cleanup, and Agent cleanup. Store created roster Agent IDs in `DifyWorld.createdAgentIds`; the shared `After` hook deletes them after each scenario. +Use `features/agent-v2/support/agent.ts` for Agent v2 API fixtures. It owns roster-shaped Agent IDs, configure/access route helpers, composer draft sync, build-draft helpers, publish, Agent drive file cleanup, and Agent cleanup. Use `features/agent-v2/support/access-point.ts` for Web app access, Backend service API access, API keys, and service API request helpers. Store created roster Agent IDs in `DifyWorld.createdAgentIds`; the shared `After` hook deletes them after each scenario. Use `DifyWorld.createdAgentDriveFiles` for Agent drive files committed during a scenario and `DifyWorld.createdBuiltinToolCredentials` for built-in tool credentials created during a scenario. The shared `After` hook deletes Agent drive files first so cleanup also works for scenarios that upload into a preseeded Agent. diff --git a/e2e/features/agent-v2/support/access-point.ts b/e2e/features/agent-v2/support/access-point.ts new file mode 100644 index 00000000000000..d8449a673c670f --- /dev/null +++ b/e2e/features/agent-v2/support/access-point.ts @@ -0,0 +1,104 @@ +import type { + AgentApiAccessResponse, + ApiKeyItem, +} from '@dify/contracts/api/console/agent/types.gen' +import type { + ChatRequestPayloadWithUser, + PostChatMessagesResponse, +} from '@dify/contracts/api/service/types.gen' +import { request } from '@playwright/test' +import { createApiContext, expectApiResponseOK, setAppSiteEnabled } from '../../../support/api' +import { getTestAgent } from './agent' + +export type AgentServiceApiChatResult = { + body: PostChatMessagesResponse | unknown + ok: boolean + status: number +} + +export async function setAgentSiteAccessAndGetURL( + agentId: string, + enabled: boolean, +): Promise { + const agent = await getTestAgent(agentId) + const appId = agent.app_id ?? agent.backing_app_id + if (!appId) + throw new Error(`Agent v2 ${agentId} does not expose a backing app ID.`) + + const appDetail = await setAppSiteEnabled(appId, enabled) + const token = agent.site?.access_token ?? agent.site?.code ?? appDetail.site.access_token + const baseURL = agent.site?.app_base_url ?? appDetail.site.app_base_url + + return `${baseURL.replace(/\/$/, '')}/agent/${token}` +} + +export async function setAgentApiAccess( + agentId: string, + enabled: boolean, +): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.post(`/console/api/agent/${agentId}/api-enable`, { + data: { enable_api: enabled }, + }) + await expectApiResponseOK( + response, + `${enabled ? 'Enable' : 'Disable'} Agent v2 API access for ${agentId}`, + ) + return (await response.json()) as AgentApiAccessResponse + } + finally { + await ctx.dispose() + } +} + +export async function createAgentApiKey(agentId: string): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.post(`/console/api/agent/${agentId}/api-keys`) + await expectApiResponseOK(response, `Create Agent v2 API key for ${agentId}`) + return (await response.json()) as ApiKeyItem + } + finally { + await ctx.dispose() + } +} + +export async function sendAgentServiceApiChatMessage({ + apiKey, + query = 'Please reply with the test success marker.', + serviceApiBaseURL, +}: { + apiKey: string + query?: string + serviceApiBaseURL: string +}): Promise { + const ctx = await request.newContext() + const body = { + inputs: {}, + query, + response_mode: 'blocking', + user: 'e2e-agent-access-point', + } satisfies ChatRequestPayloadWithUser + + try { + const response = await ctx.post(`${serviceApiBaseURL.replace(/\/$/, '')}/chat-messages`, { + data: body, + headers: { + Authorization: `Bearer ${apiKey}`, + }, + }) + const responseBody = await response.json().catch(async () => ({ + message: await response.text().catch(() => ''), + })) + + return { + body: responseBody as PostChatMessagesResponse | unknown, + ok: response.ok(), + status: response.status(), + } + } + finally { + await ctx.dispose() + } +} diff --git a/e2e/features/agent-v2/support/agent.ts b/e2e/features/agent-v2/support/agent.ts index 80509fa7f1edac..2effc34b22b46a 100644 --- a/e2e/features/agent-v2/support/agent.ts +++ b/e2e/features/agent-v2/support/agent.ts @@ -1,5 +1,4 @@ import type { - AgentApiAccessResponse, AgentAppComposerResponse, AgentAppDetailWithSite, AgentBuildDraftResponse, @@ -15,17 +14,11 @@ import type { AgentReferencingWorkflowsResponse, AgentSkillUploadResponse, AgentSoulConfig, - ApiKeyItem, } from '@dify/contracts/api/console/agent/types.gen' -import type { - ChatRequestPayloadWithUser, - PostChatMessagesResponse, -} from '@dify/contracts/api/service/types.gen' import { Buffer } from 'node:buffer' import { readFile } from 'node:fs/promises' import path from 'node:path' -import { request } from '@playwright/test' -import { createApiContext, expectApiResponseOK, setAppSiteEnabled } from '../../../support/api' +import { createApiContext, expectApiResponseOK } from '../../../support/api' import { assertE2EResourceName, createE2EResourceName } from '../../../support/naming' export type AgentSeed = Pick< @@ -64,12 +57,6 @@ export type CreateTestAgentOptions = { role?: string } -export type AgentServiceApiChatResult = { - body: PostChatMessagesResponse | unknown - ok: boolean - status: number -} - export const defaultAgentSoulConfig: AgentSoulConfig = { prompt: { system_prompt: 'You are a Dify Agent E2E test assistant.', @@ -583,120 +570,6 @@ export async function publishAgent(agentId: string, versionNote = 'E2E publish') } } -export async function enableAgentSiteAndGetURL(agentId: string): Promise { - return setAgentSiteAccessAndGetURL(agentId, true) -} - -export async function setAgentSiteAccessAndGetURL( - agentId: string, - enabled: boolean, -): Promise { - const agent = await getTestAgent(agentId) - const appId = agent.app_id ?? agent.backing_app_id - if (!appId) - throw new Error(`Agent v2 ${agentId} does not expose a backing app ID.`) - - const appDetail = await setAppSiteEnabled(appId, enabled) - const token = agent.site?.access_token ?? agent.site?.code ?? appDetail.site.access_token - const baseURL = agent.site?.app_base_url ?? appDetail.site.app_base_url - - return `${baseURL.replace(/\/$/, '')}/agent/${token}` -} - -export async function getAgentApiAccess(agentId: string): Promise { - const ctx = await createApiContext() - try { - const response = await ctx.get(`/console/api/agent/${agentId}/api-access`) - await expectApiResponseOK(response, `Get Agent v2 API access for ${agentId}`) - return (await response.json()) as AgentApiAccessResponse - } - finally { - await ctx.dispose() - } -} - -export async function setAgentApiAccess( - agentId: string, - enabled: boolean, -): Promise { - const ctx = await createApiContext() - try { - const response = await ctx.post(`/console/api/agent/${agentId}/api-enable`, { - data: { enable_api: enabled }, - }) - await expectApiResponseOK( - response, - `${enabled ? 'Enable' : 'Disable'} Agent v2 API access for ${agentId}`, - ) - return (await response.json()) as AgentApiAccessResponse - } - finally { - await ctx.dispose() - } -} - -export async function createAgentApiKey(agentId: string): Promise { - const ctx = await createApiContext() - try { - const response = await ctx.post(`/console/api/agent/${agentId}/api-keys`) - await expectApiResponseOK(response, `Create Agent v2 API key for ${agentId}`) - return (await response.json()) as ApiKeyItem - } - finally { - await ctx.dispose() - } -} - -export async function deleteAgentApiKey(agentId: string, apiKeyId: string): Promise { - const ctx = await createApiContext() - try { - const response = await ctx.delete(`/console/api/agent/${agentId}/api-keys/${apiKeyId}`) - await expectApiResponseOK(response, `Delete Agent v2 API key ${apiKeyId} for ${agentId}`) - } - finally { - await ctx.dispose() - } -} - -export async function sendAgentServiceApiChatMessage({ - apiKey, - query = 'Please reply with the test success marker.', - serviceApiBaseURL, -}: { - apiKey: string - query?: string - serviceApiBaseURL: string -}): Promise { - const ctx = await request.newContext() - const body = { - inputs: {}, - query, - response_mode: 'blocking', - user: 'e2e-agent-access-point', - } satisfies ChatRequestPayloadWithUser - - try { - const response = await ctx.post(`${serviceApiBaseURL.replace(/\/$/, '')}/chat-messages`, { - data: body, - headers: { - Authorization: `Bearer ${apiKey}`, - }, - }) - const responseBody = await response.json().catch(async () => ({ - message: await response.text().catch(() => ''), - })) - - return { - body: responseBody as PostChatMessagesResponse | unknown, - ok: response.ok(), - status: response.status(), - } - } - finally { - await ctx.dispose() - } -} - export async function deleteAgentConfigFile(agentId: string, name: string): Promise { const ctx = await createApiContext() try { diff --git a/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts b/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts index d54c43ae231715..ef0f2fe858f3e8 100644 --- a/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts @@ -3,10 +3,10 @@ import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { createAgentApiKey, - publishAgent, sendAgentServiceApiChatMessage, setAgentApiAccess, -} from '../../agent-v2/support/agent' +} from '../../agent-v2/support/access-point' +import { publishAgent } from '../../agent-v2/support/agent' import { agentBuilderExpectedTokens } from '../../agent-v2/support/agent-builder-resources' import { getCurrentAgentId, getServiceApiCard } from './access-point-helpers' diff --git a/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts b/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts index 4a0a007b6ff190..28691447ecd479 100644 --- a/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts @@ -1,10 +1,8 @@ import type { DifyWorld } from '../../support/world' import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' -import { - getAgentComposerDraft, - setAgentSiteAccessAndGetURL, -} from '../../agent-v2/support/agent' +import { setAgentSiteAccessAndGetURL } from '../../agent-v2/support/access-point' +import { getAgentComposerDraft } from '../../agent-v2/support/agent' import { agentBuilderExpectedTokens } from '../../agent-v2/support/agent-builder-resources' import { getCurrentAgentId, From e05048efeec28324eb8c874521bfb92e390aa977 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 01:10:15 +0800 Subject: [PATCH 130/185] test(e2e): align stable model selector --- e2e/features/agent-v2/AGENTS.md | 4 ++-- e2e/features/agent-v2/support/preflight/models.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 1091da41c77736..1773df0ffdec47 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -137,7 +137,7 @@ Agent Builder resource checks live under `features/agent-v2/support/preflight/`. `preflight.steps.ts` should remain the explicit `Given` entrypoint. Do not move preflight into hidden hooks. -Use `the Agent Builder stable chat model is available` before scenarios that need a real Agent Soul model configuration. This includes true runtime scenarios and build-mode workflows whose backend API requires model config, such as Agent workflow nodes. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` are selectors for a model already configured in the workspace; they are not provider credentials. The step defaults to `openai` / `gpt-5.4-mini` / `llm`, verifies the selected model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilder.preflight.stableModel`. +Use `the Agent Builder stable chat model is available` before scenarios that need a real Agent Soul model configuration. This includes true runtime scenarios and build-mode workflows whose backend API requires model config, such as Agent workflow nodes. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` are selectors for a model already configured in the workspace; they are not provider credentials. The step defaults to `openai` / `gpt-5` / `llm`, verifies the selected model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilder.preflight.stableModel`. Do not pass model provider API keys through Cucumber or Playwright env vars. Provider credentials belong to the Dify environment seed/admin setup. If the selected provider/model is missing or inactive, the scenario must be blocked by preflight instead of trying to create or patch provider credentials during the test. @@ -145,7 +145,7 @@ Override the default selector only when a scenario or environment explicitly nee ```bash E2E_STABLE_MODEL_PROVIDER=openai -E2E_STABLE_MODEL_NAME=gpt-5.4-mini +E2E_STABLE_MODEL_NAME=gpt-5 E2E_STABLE_MODEL_TYPE=llm ``` diff --git a/e2e/features/agent-v2/support/preflight/models.ts b/e2e/features/agent-v2/support/preflight/models.ts index 330c61c9d482f6..2171ffdf9d4c8e 100644 --- a/e2e/features/agent-v2/support/preflight/models.ts +++ b/e2e/features/agent-v2/support/preflight/models.ts @@ -12,7 +12,7 @@ const brokenChatModelNameEnv = 'E2E_BROKEN_MODEL_NAME' const brokenChatModelTypeEnv = 'E2E_BROKEN_MODEL_TYPE' const activeModelStatus = 'active' const defaultStableChatModelProvider = 'openai' -const defaultStableChatModelName = 'gpt-5.4-mini' +const defaultStableChatModelName = 'gpt-5' const defaultStableChatModelType = 'llm' const defaultBrokenChatModelName = agentBuilderPreseededResources.brokenModel From 2f8665a067b6ecf2c78660e2d4a46a1e07a00710 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 01:15:18 +0800 Subject: [PATCH 131/185] test(e2e): cover agent model selection persistence --- .../agent-v2/configure-persistence.feature | 16 ++++++++++++++++ .../agent-v2/configure.steps.ts | 17 +++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/e2e/features/agent-v2/configure-persistence.feature b/e2e/features/agent-v2/configure-persistence.feature index 23c9182370a1f4..7a177dba8e0e8d 100644 --- a/e2e/features/agent-v2/configure-persistence.feature +++ b/e2e/features/agent-v2/configure-persistence.feature @@ -1,5 +1,21 @@ @agent-v2 @authenticated @core Feature: Agent v2 configure persistence + @configure-persistence + Scenario: Selecting a stable model in Configure persists after refresh + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And an Agent v2 test agent has been created via API + When I open the Agent v2 configure page + And I select the stable E2E model in the Agent v2 model selector + And I fill the Agent v2 prompt editor with the normal E2E prompt + Then the Agent v2 configuration should be saved automatically + And the Agent v2 draft should use the stable E2E model + And the normal Agent v2 draft should use the normal E2E prompt + When I refresh the current page + Then I should see the stable E2E model in the Agent v2 model selector + And I should see the normal E2E prompt in the Agent v2 prompt editor + And the Agent v2 draft should use the stable E2E model + @configure-persistence Scenario: Persisted Agent v2 instructions remain visible after refresh Given I am signed in as the default E2E admin diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 3f1bfffe97f6e7..4e48fc46a4881d 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -54,6 +54,12 @@ async function fillAgentPromptEditor(page: Page, prompt: string) { await getPromptEditor(page).fill(prompt) } +async function selectAgentModel(page: Page, modelName: string) { + await page.getByRole('combobox', { name: 'Configure model' }).click() + await page.getByLabel('Search model').fill(modelName) + await page.getByRole('option', { name: modelName }).click() +} + async function expectAgentComposerPrompt(agentId: string, prompt: string) { await expect.poll( async () => (await getAgentComposerDraft(agentId)).agent_soul?.prompt?.system_prompt, @@ -125,6 +131,17 @@ When('I open the Agent v2 configure page', async function (this: DifyWorld) { await this.getPage().goto(getAgentConfigurePath(getCurrentAgentId(this))) }) +When( + 'I select the stable E2E model in the Agent v2 model selector', + async function (this: DifyWorld) { + const stableModel = this.agentBuilder.preflight.stableModel + if (!stableModel) + throw new Error('Stable chat model preflight must run before selecting the Agent model.') + + await selectAgentModel(this.getPage(), stableModel.name) + }, +) + When('I switch to the Agent v2 Configure section', async function (this: DifyWorld) { const page = this.getPage() const agentId = getCurrentAgentId(this) From 104ce27cf80e4555fc85f52498eb9a45db196c12 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 01:32:38 +0800 Subject: [PATCH 132/185] test(web): cover embedded web app dialog close --- .../overview/embedded/__tests__/index.spec.tsx | 14 ++++++++++++++ .../__tests__/access-surface-cards.spec.tsx | 17 +++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/web/app/components/app/overview/embedded/__tests__/index.spec.tsx b/web/app/components/app/overview/embedded/__tests__/index.spec.tsx index a6e391cb0e5e4b..8c661829a97fad 100644 --- a/web/app/components/app/overview/embedded/__tests__/index.spec.tsx +++ b/web/app/components/app/overview/embedded/__tests__/index.spec.tsx @@ -142,6 +142,20 @@ describe('Embedded', () => { ) }) + it('calls onClose when the close button is clicked', async () => { + const onClose = vi.fn() + + await act(async () => { + render() + }) + + await act(async () => { + fireEvent.click(screen.getByRole('button', { name: 'Close' })) + }) + + expect(onClose).toHaveBeenCalledTimes(1) + }) + it('keeps hidden inputs collapsed by default and updates iframe and script content when values change', async () => { render( { }) }) + it('should close the embedded dialog from the close button', async () => { + const user = userEvent.setup() + + renderWithQueryClient( + , + ) + + await user.click(screen.getByRole('button', { name: 'agentV2.agentDetail.access.webApp.actions.embedded' })) + const dialog = await screen.findByRole('dialog', { name: 'appOverview.overview.appInfo.embedded.title' }) + + await user.click(within(dialog).getByRole('button', { name: 'Close' })) + + await waitFor(() => { + expect(screen.queryByRole('dialog', { name: 'appOverview.overview.appInfo.embedded.title' })).not.toBeInTheDocument() + }) + }) + it('should save settings through the backing app id and update the agent detail cache', async () => { const user = userEvent.setup() const agent = createAgent() From d90575ec7384f328858eddce9f39243d861b4e9f Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 01:32:44 +0800 Subject: [PATCH 133/185] test(e2e): isolate agent web app access scenarios --- e2e/features/agent-v2/access-point.feature | 55 ++++++++++++++----- .../agent-v2/access-point-web-app.steps.ts | 40 +++----------- 2 files changed, 50 insertions(+), 45 deletions(-) diff --git a/e2e/features/agent-v2/access-point.feature b/e2e/features/agent-v2/access-point.feature index 2ff5e2c439f563..13f079a41537dd 100644 --- a/e2e/features/agent-v2/access-point.feature +++ b/e2e/features/agent-v2/access-point.feature @@ -9,32 +9,61 @@ Feature: Agent v2 Access Point Then I should see the Agent v2 Access Point overview @core @web-app-access - Scenario: Web app access actions open their public surfaces + Scenario: Web app access URL can be copied and launched Given I am signed in as the default E2E admin - And the Agent Builder preseeded Agent "E2E Agent Published Web App" is available - And the Agent Builder preseeded Agent "E2E Agent Published Web App" has published Web app access - When I open the preseeded Agent v2 Access Point page for "E2E Agent Published Web App" from the Agent Roster + And a basic configured Agent v2 test agent has been created via API + And Agent v2 Web app access has been enabled via API + When I open the Agent v2 configure page from the Agent Roster + And I switch to the Agent v2 Access Point section Then I should see the Agent v2 Web app access URL - And I record the Agent v2 orchestration draft for "E2E Agent Published Web App" + And I record the current Agent v2 orchestration draft When I copy the Agent v2 Web app access URL Then the Agent v2 Web app access URL should show it was copied When I launch the Agent v2 Web app Then the Agent v2 Web app should open in a new tab - When I open Agent v2 Embedded configuration + And the current Agent v2 orchestration draft should be unchanged + + @core @web-app-access + Scenario: Web app Embedded configuration opens from Access Point + Given I am signed in as the default E2E admin + And a basic configured Agent v2 test agent has been created via API + And Agent v2 Web app access has been enabled via API + When I open the Agent v2 configure page from the Agent Roster + And I switch to the Agent v2 Access Point section + And I open Agent v2 Embedded configuration Then I should see the Agent v2 Embedded configuration dialog - When I open Agent v2 Web app customization + + @core @web-app-access + Scenario: Web app customization opens from Access Point + Given I am signed in as the default E2E admin + And a basic configured Agent v2 test agent has been created via API + And Agent v2 Web app access has been enabled via API + When I open the Agent v2 configure page from the Agent Roster + And I switch to the Agent v2 Access Point section + And I open Agent v2 Web app customization Then I should see the Agent v2 Web app customization dialog - When I open Agent v2 Web app settings + + @core @web-app-access + Scenario: Web app settings open from Access Point without changing orchestration + Given I am signed in as the default E2E admin + And a basic configured Agent v2 test agent has been created via API + And Agent v2 Web app access has been enabled via API + When I open the Agent v2 configure page from the Agent Roster + And I switch to the Agent v2 Access Point section + And I record the current Agent v2 orchestration draft + And I open Agent v2 Web app settings Then I should see the Agent v2 Web app settings dialog - And the Agent v2 orchestration draft for "E2E Agent Published Web App" should be unchanged + And the current Agent v2 orchestration draft should be unchanged @core @web-app-access Scenario: Web app access can be disabled and restored Given I am signed in as the default E2E admin - And the Agent Builder preseeded Agent "E2E Agent Published Web App" is available - And the Agent Builder preseeded Agent "E2E Agent Published Web App" has published Web app access - And Agent v2 Web app access will be restored for "E2E Agent Published Web App" - When I open the preseeded Agent v2 Access Point page for "E2E Agent Published Web App" from the Agent Roster + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API + And the Agent v2 draft has been published via API + And Agent v2 Web app access has been enabled via API + When I open the Agent v2 configure page from the Agent Roster + And I switch to the Agent v2 Access Point section And I disable Agent v2 Web app access Then Agent v2 Web app access should be out of service When I open the disabled Agent v2 Web app URL diff --git a/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts b/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts index 28691447ecd479..85dc25a9cf7a01 100644 --- a/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts @@ -1,5 +1,5 @@ import type { DifyWorld } from '../../support/world' -import { Given, Then, When } from '@cucumber/cucumber' +import { Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { setAgentSiteAccessAndGetURL } from '../../agent-v2/support/access-point' import { getAgentComposerDraft } from '../../agent-v2/support/agent' @@ -7,28 +7,9 @@ import { agentBuilderExpectedTokens } from '../../agent-v2/support/agent-builder import { getCurrentAgentId, getDialog, - getPreseededResource, getWebAppCard, } from './access-point-helpers' -const closeDialog = async (world: DifyWorld, name: string | RegExp) => { - const dialog = getDialog(world, name) - - await dialog.getByLabel('Close').click() - await expect(dialog).not.toBeVisible() -} - -Given( - 'Agent v2 Web app access will be restored for {string}', - async function (this: DifyWorld, agentName: string) { - const agent = getPreseededResource(this, agentName, 'agent') - - this.registerCleanup(async () => { - await setAgentSiteAccessAndGetURL(agent.id, true) - }) - }, -) - When( 'Agent v2 Web app access has been enabled via API', async function (this: DifyWorld) { @@ -49,10 +30,9 @@ Then('I should see the Agent v2 Web app access URL', async function (this: DifyW }) Then( - 'I record the Agent v2 orchestration draft for {string}', - async function (this: DifyWorld, agentName: string) { - const agent = getPreseededResource(this, agentName, 'agent') - const draft = await getAgentComposerDraft(agent.id) + 'I record the current Agent v2 orchestration draft', + async function (this: DifyWorld) { + const draft = await getAgentComposerDraft(getCurrentAgentId(this)) this.agentBuilder.accessPoint.composerDraftSnapshot = JSON.stringify(draft.agent_soul ?? {}) }, @@ -168,7 +148,6 @@ Then('I should see the Agent v2 Embedded configuration dialog', async function ( await expect(dialog).toBeVisible() await expect(dialog.getByText('Embed on website')).toBeVisible() await expect(dialog.getByText(/iframe|script/i)).toBeVisible() - await closeDialog(this, 'Embed on website') }) When('I open Agent v2 Web app customization', async function (this: DifyWorld) { @@ -181,7 +160,6 @@ Then('I should see the Agent v2 Web app customization dialog', async function (t await expect(dialog).toBeVisible() await expect(dialog.getByText('Customize AI web app')).toBeVisible() await expect(dialog.getByText(/NEXT_PUBLIC_APP_ID|NEXT_PUBLIC_API_URL/)).toBeVisible() - await closeDialog(this, 'Customize AI web app') }) When('I open Agent v2 Web app settings', async function (this: DifyWorld) { @@ -192,21 +170,19 @@ Then('I should see the Agent v2 Web app settings dialog', async function (this: const dialog = getDialog(this, 'Web App Settings') await expect(dialog).toBeVisible() - await expect(dialog.getByText('Web App Settings')).toBeVisible() + await expect(dialog.getByRole('heading', { name: 'Web App Settings' })).toBeVisible() await expect(dialog.getByText('web app Name')).toBeVisible() await expect(dialog.getByText('web app Description')).toBeVisible() - await closeDialog(this, 'Web App Settings') }) Then( - 'the Agent v2 orchestration draft for {string} should be unchanged', - async function (this: DifyWorld, agentName: string) { + 'the current Agent v2 orchestration draft should be unchanged', + async function (this: DifyWorld) { const snapshot = this.agentBuilder.accessPoint.composerDraftSnapshot if (!snapshot) throw new Error('No Agent v2 orchestration draft snapshot was recorded.') - const agent = getPreseededResource(this, agentName, 'agent') - const draft = await getAgentComposerDraft(agent.id) + const draft = await getAgentComposerDraft(getCurrentAgentId(this)) expect(JSON.stringify(draft.agent_soul ?? {})).toBe(snapshot) }, From fe99a35f3d179acac3b11bb45af71f187a61d0b5 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 01:37:57 +0800 Subject: [PATCH 134/185] docs(e2e): clarify preflight seed ownership --- e2e/AGENTS.md | 4 ++++ e2e/features/agent-v2/AGENTS.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 6b7b10de1950aa..69d3f259e06aa6 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -300,12 +300,16 @@ Use `fixtures/test-materials/` for checked-in files that scenarios upload, previ Use scoped feature support for scenarios that require optional external resources such as a model provider, plugin/tool credential, knowledge base seed, or fixed app. Prefer an explicit `Given` step that returns a skipped result with a clear blocked-precondition reason over hidden setup in hooks. +Treat preflight checks as read-only readiness checks. A preflight step may query the environment, record typed state on `DifyWorld`, attach a blocked-precondition reason, or return `skipped`; it must not create, repair, publish, reconfigure, or mutate shared seed resources. Long-lived resources belong to the environment seed/setup process and need an explicit owner outside individual scenarios. + Keep package-level support limited to broadly reusable primitives such as API clients, naming, fixture path resolution, and cleanup helpers. Feature-specific seed contracts and preflight checks belong under the owning feature's support folder. Use generated API contracts for Console/Web/Service API request, response, and payload shapes. Import the concrete type directly from `@dify/contracts/.../types.gen` when it exists, and do not hand-write duplicate response shapes or wrap generated types in local aliases just to preserve an older helper name. Keep local E2E types only for scenario state, fixture registries, helper input options, preflight resource state, and intentionally narrowed test view models that are not complete API responses. Use typed cleanup fields on `DifyWorld` for resource types created by scenarios, and use `DifyWorld.registerCleanup(...)` when a scenario creates any resource type that is not covered by typed cleanup fields. Cleanup callbacks run after typed cleanup queues, even when the scenario fails. +Scenario-owned setup may create disposable apps, Agents, files, credentials, drafts, or access toggles when the scenario owns their lifecycle and cleanup. Do not use scenario setup to silently fix or complete a shared preseeded resource; if a fixed resource is missing or drifted, report it as blocked and route it to the seed owner. + Feature-specific seed contracts, resource readiness rules, tags, and scenario ownership can be documented in one scoped `AGENTS.md` at the feature root when a module becomes large enough to need it. Do not add deeper `AGENTS.md` files unless the nested module becomes independently owned. ## Reusing existing steps diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 1773df0ffdec47..ff3af8be369846 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -101,6 +101,8 @@ Use `the Agent v2 configuration should be saved automatically` after UI edits th API setup is acceptable for creating scenario-owned Agents, enabling Backend service API, writing composer drafts, seeding Build drafts, and preparing fixed state. The scenario must still assert user-visible behavior or a real persisted product contract through the public Console API. Do not assert only that a setup API call succeeded. +Do not use scenario API setup to repair an environment-owned Agent Builder seed. If a scenario depends on a fixed Agent, dataset, workflow, Skill, Tool, credential, published Web app, or active model, use the matching preflight step to verify it and block when it is missing or drifted. Create or mutate resources only when they are scenario-owned and registered for cleanup. + ## API Contract Types Agent v2 support helpers consume Console API contracts from `@dify/contracts/api/console/.../types.gen`. When a generated request, response, or payload type exists, import and use that exact type name at the helper boundary. Do not keep an old local response type name as an alias for the generated type. @@ -137,6 +139,8 @@ Agent Builder resource checks live under `features/agent-v2/support/preflight/`. `preflight.steps.ts` should remain the explicit `Given` entrypoint. Do not move preflight into hidden hooks. +Agent Builder preflight is read-only. It checks long-lived seed resources and records their IDs or normalized metadata for later steps, but it must not create missing resources, toggle fixed access settings, upload missing Skills/files, publish fixed Agents, or patch model/provider credentials. Seed creation and repair belong to the environment setup process, not to Cucumber scenarios. + Use `the Agent Builder stable chat model is available` before scenarios that need a real Agent Soul model configuration. This includes true runtime scenarios and build-mode workflows whose backend API requires model config, such as Agent workflow nodes. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` are selectors for a model already configured in the workspace; they are not provider credentials. The step defaults to `openai` / `gpt-5` / `llm`, verifies the selected model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilder.preflight.stableModel`. Do not pass model provider API keys through Cucumber or Playwright env vars. Provider credentials belong to the Dify environment seed/admin setup. If the selected provider/model is missing or inactive, the scenario must be blocked by preflight instead of trying to create or patch provider credentials during the test. From fb7bfd3787b5ea61b27ba835ce4993a115b11bb0 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 01:49:35 +0800 Subject: [PATCH 135/185] test(e2e): cover agent knowledge retrieval settings --- e2e/features/agent-v2/knowledge.feature | 22 +++ .../agent-v2/knowledge.steps.ts | 168 +++++++++++++++++- 2 files changed, 187 insertions(+), 3 deletions(-) diff --git a/e2e/features/agent-v2/knowledge.feature b/e2e/features/agent-v2/knowledge.feature index 4462dc6ef26419..0d60cf6e7f406c 100644 --- a/e2e/features/agent-v2/knowledge.feature +++ b/e2e/features/agent-v2/knowledge.feature @@ -1,5 +1,27 @@ @agent-v2 @authenticated @knowledge @core Feature: Agent v2 Knowledge Retrieval + Scenario: Agent decide Knowledge Retrieval settings are saved and restored + Given I am signed in as the default E2E admin + And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready + And a basic configured Agent v2 test agent has been created via API + When I open the Agent v2 configure page + And I add the Agent Builder knowledge base as an Agent decide Knowledge Retrieval + Then the Agent v2 Agent decide Knowledge Retrieval should be saved in the Agent v2 draft + And the Agent v2 configuration should be saved automatically + When I refresh the current page + Then I should see the Agent v2 Agent decide Knowledge Retrieval settings + + Scenario: Custom query Knowledge Retrieval settings are saved and restored + Given I am signed in as the default E2E admin + And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready + And a basic configured Agent v2 test agent has been created via API + When I open the Agent v2 configure page + And I add the Agent Builder knowledge base as a Custom query Knowledge Retrieval + Then the Agent v2 Custom query Knowledge Retrieval should be saved in the Agent v2 draft + And the Agent v2 configuration should be saved automatically + When I refresh the current page + Then I should see the Agent v2 Custom query Knowledge Retrieval settings + Scenario: Removing Knowledge Retrieval clears the saved dataset reference Given I am signed in as the default E2E admin And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready diff --git a/e2e/features/step-definitions/agent-v2/knowledge.steps.ts b/e2e/features/step-definitions/agent-v2/knowledge.steps.ts index 81d1ba29c327bb..7682ce6956c82d 100644 --- a/e2e/features/step-definitions/agent-v2/knowledge.steps.ts +++ b/e2e/features/step-definitions/agent-v2/knowledge.steps.ts @@ -1,3 +1,4 @@ +import type { Locator } from '@playwright/test' import type { DifyWorld } from '../../support/world' import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' @@ -7,7 +8,7 @@ import { getAgentComposerDraft, normalAgentSoulConfig, } from '../../agent-v2/support/agent' -import { agentBuilderPreseededResources } from '../../agent-v2/support/agent-builder-resources' +import { agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../agent-v2/support/agent-builder-resources' import { asArray, asRecord } from '../../agent-v2/support/preflight/common' import { getCurrentAgentId } from './configure-helpers' @@ -27,6 +28,96 @@ const getPreseededKnowledgeBase = (world: DifyWorld) => { const getKnowledgeSection = (world: DifyWorld) => world.getPage().getByRole('region', { name: 'Knowledge Retrieval' }) +const getKnowledgeSets = async (agentId: string) => { + const draft = await getAgentComposerDraft(agentId) + + return asArray(asRecord(draft.agent_soul?.knowledge).sets) +} + +const openNewKnowledgeRetrievalDialog = async (world: DifyWorld) => { + const knowledgeSection = getKnowledgeSection(world) + + await expect(knowledgeSection).toBeVisible({ timeout: 30_000 }) + await knowledgeSection.getByRole('button', { name: 'Add knowledge retrieval' }).click() + + const dialog = world.getPage().getByRole('dialog', { + name: 'Knowledge Retrieval · Agent decide', + }) + await expect(dialog).toBeVisible() + + return dialog +} + +const selectPreseededKnowledgeBase = async ( + world: DifyWorld, + dialog: Locator, +) => { + const knowledgeBase = getPreseededKnowledgeBase(world) + const page = world.getPage() + + await dialog.getByRole('button', { name: 'Add Knowledge' }).click() + + const selectorDialog = page.getByRole('dialog', { name: 'Select reference Knowledge' }) + await expect(selectorDialog).toBeVisible() + await selectorDialog.getByRole('button').filter({ hasText: knowledgeBase.name }).click() + await selectorDialog.getByRole('button', { name: 'Add' }).click() +} + +const openKnowledgeRetrievalSettings = async (world: DifyWorld, name: string) => { + const knowledgeSection = getKnowledgeSection(world) + + await expect(knowledgeSection.getByText(name, { exact: true })) + .toBeVisible({ timeout: 30_000 }) + await knowledgeSection.getByText(name, { exact: true }).hover() + await knowledgeSection.getByRole('button', { + exact: true, + name: `Edit ${name}`, + }).click() + + const dialog = world.getPage().getByRole('dialog', { + name: 'Knowledge Retrieval · Agent decide', + }) + await expect(dialog).toBeVisible() + + return dialog +} + +const expectKnowledgeRetrievalDraft = async ( + world: DifyWorld, + expected: { + mode: 'generated_query' | 'user_query' + value?: string + }, +) => { + const agentId = getCurrentAgentId(world) + const knowledgeBase = getPreseededKnowledgeBase(world) + + await expect.poll( + async () => { + const knowledgeSets = await getKnowledgeSets(agentId) + const knowledgeSet = asRecord(knowledgeSets[0]) + const datasets = asArray(knowledgeSet.datasets) + const query = asRecord(knowledgeSet.query) + const retrieval = asRecord(knowledgeSet.retrieval) + + return { + datasetNames: datasets.map(dataset => asRecord(dataset).name), + mode: query.mode, + name: knowledgeSet.name, + retrievalMode: retrieval.mode, + value: query.value, + } + }, + { timeout: 30_000 }, + ).toEqual({ + datasetNames: expect.arrayContaining([knowledgeBase.name]), + mode: expected.mode, + name: 'Retrieval 1', + retrievalMode: 'multiple', + value: expected.value, + }) +} + Given( 'a knowledge-backed Agent v2 test agent has been created via API', async function (this: DifyWorld) { @@ -47,6 +138,33 @@ Given( }, ) +When( + 'I add the Agent Builder knowledge base as an Agent decide Knowledge Retrieval', + async function (this: DifyWorld) { + const dialog = await openNewKnowledgeRetrievalDialog(this) + + await expect(dialog.getByRole('radio', { name: 'Agent decide' })).toBeChecked() + await selectPreseededKnowledgeBase(this, dialog) + await expect(dialog.getByText(getPreseededKnowledgeBase(this).name, { exact: true })) + .toBeVisible() + }, +) + +When( + 'I add the Agent Builder knowledge base as a Custom query Knowledge Retrieval', + async function (this: DifyWorld) { + const dialog = await openNewKnowledgeRetrievalDialog(this) + + await dialog.getByRole('radio', { name: 'Custom query' }).click() + await dialog + .getByRole('textbox', { name: 'Custom query text' }) + .fill(agentBuilderFixedInputs.customKnowledgeQuery) + await selectPreseededKnowledgeBase(this, dialog) + await expect(dialog.getByText(getPreseededKnowledgeBase(this).name, { exact: true })) + .toBeVisible() + }, +) + Then('I should see the Agent v2 Knowledge Retrieval {string}', async function (this: DifyWorld, name: string) { const knowledgeSection = getKnowledgeSection(this) @@ -54,6 +172,51 @@ Then('I should see the Agent v2 Knowledge Retrieval {string}', async function (t await expect(knowledgeSection.getByText(name, { exact: true })).toBeVisible() }) +Then( + 'the Agent v2 Agent decide Knowledge Retrieval should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + await expectKnowledgeRetrievalDraft(this, { + mode: 'generated_query', + }) + }, +) + +Then( + 'the Agent v2 Custom query Knowledge Retrieval should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + await expectKnowledgeRetrievalDraft(this, { + mode: 'user_query', + value: agentBuilderFixedInputs.customKnowledgeQuery, + }) + }, +) + +Then( + 'I should see the Agent v2 Agent decide Knowledge Retrieval settings', + async function (this: DifyWorld) { + const dialog = await openKnowledgeRetrievalSettings(this, 'Retrieval 1') + + await expect(dialog.getByRole('radio', { name: 'Agent decide' })).toBeChecked() + await expect(dialog.getByText(getPreseededKnowledgeBase(this).name, { exact: true })) + .toBeVisible() + await expect(dialog.getByRole('button', { name: 'Disabled' })).toBeVisible() + }, +) + +Then( + 'I should see the Agent v2 Custom query Knowledge Retrieval settings', + async function (this: DifyWorld) { + const dialog = await openKnowledgeRetrievalSettings(this, 'Retrieval 1') + + await expect(dialog.getByRole('radio', { name: 'Custom query' })).toBeChecked() + await expect(dialog.getByRole('textbox', { name: 'Custom query text' })) + .toHaveValue(agentBuilderFixedInputs.customKnowledgeQuery) + await expect(dialog.getByText(getPreseededKnowledgeBase(this).name, { exact: true })) + .toBeVisible() + await expect(dialog.getByRole('button', { name: 'Disabled' })).toBeVisible() + }, +) + When('I remove the Agent v2 Knowledge Retrieval {string}', async function (this: DifyWorld, name: string) { const knowledgeSection = getKnowledgeSection(this) @@ -73,8 +236,7 @@ Then( await expect.poll( async () => { - const draft = await getAgentComposerDraft(agentId) - const knowledgeSets = asArray(asRecord(draft.agent_soul?.knowledge).sets) + const knowledgeSets = await getKnowledgeSets(agentId) return knowledgeSets.some(set => asArray(asRecord(set).datasets).some((dataset) => { const record = asRecord(dataset) From badad4535a04db690d9fc2749221b3100acf1cf4 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 02:01:55 +0800 Subject: [PATCH 136/185] fix(web): make tool action rows accessible --- .../block-selector/tool/__tests__/action-item.spec.tsx | 5 +++-- .../workflow/block-selector/tool/action-item.tsx | 8 +++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/web/app/components/workflow/block-selector/tool/__tests__/action-item.spec.tsx b/web/app/components/workflow/block-selector/tool/__tests__/action-item.spec.tsx index 0424c46fd504ea..341f0d2346fa01 100644 --- a/web/app/components/workflow/block-selector/tool/__tests__/action-item.spec.tsx +++ b/web/app/components/workflow/block-selector/tool/__tests__/action-item.spec.tsx @@ -45,7 +45,7 @@ describe('ToolActionItem', () => { />, ) - await user.click(screen.getByText('Search Tool')) + await user.click(screen.getByRole('button', { name: 'Search Tool' })) expect(onSelect).toHaveBeenCalledWith(BlockEnum.Tool, expect.objectContaining({ provider_id: 'provider-1', @@ -75,8 +75,9 @@ describe('ToolActionItem', () => { ) expect(screen.getByText('tools.addToolModal.added')).toBeInTheDocument() + expect(screen.getByRole('button', { name: /Search Tool/ })).toBeDisabled() - await user.click(screen.getByText('Search Tool')) + await user.click(screen.getByRole('button', { name: /Search Tool/ })) expect(onSelect).not.toHaveBeenCalled() expect(mockTrackEvent).not.toHaveBeenCalled() diff --git a/web/app/components/workflow/block-selector/tool/action-item.tsx b/web/app/components/workflow/block-selector/tool/action-item.tsx index 153bc75ce56bc6..0d28b9a1a4c4aa 100644 --- a/web/app/components/workflow/block-selector/tool/action-item.tsx +++ b/web/app/components/workflow/block-selector/tool/action-item.tsx @@ -69,9 +69,11 @@ const ToolItem: FC = ({ }, [theme, normalizedIcon, normalizedIconDark]) const row = ( -
{ if (disabled) return @@ -111,7 +113,7 @@ const ToolItem: FC = ({ {isAdded && (
{t('addToolModal.added', { ns: 'tools' })}
)} -
+ ) return ( From a684ae7ace58d1f03a251f2ef65bc22774dcfdee Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 02:08:07 +0800 Subject: [PATCH 137/185] test(e2e): cover agent tool configuration persistence --- e2e/features/agent-v2/tools.feature | 11 +++ .../step-definitions/agent-v2/tools.steps.ts | 86 ++++++++++++++++++- 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/e2e/features/agent-v2/tools.feature b/e2e/features/agent-v2/tools.feature index 7b1cc73f9c7eaa..4039ef90c76911 100644 --- a/e2e/features/agent-v2/tools.feature +++ b/e2e/features/agent-v2/tools.feature @@ -1,5 +1,16 @@ @agent-v2 @authenticated @tools @core Feature: Agent v2 tools + Scenario: JSON Replace tool is saved after adding it from the Tools selector + Given I am signed in as the default E2E admin + And the Agent Builder preseeded tool "JSON Process / JSON Replace" is available + And a basic configured Agent v2 test agent has been created via API + When I open the Agent v2 configure page + And I add the Agent Builder JSON Replace tool from the Tools selector + Then the Agent v2 JSON Replace tool should be saved in the Agent v2 draft + And the Agent v2 configuration should be saved automatically + When I refresh the current page + Then I should see the Agent v2 JSON Replace tool in the Tools section + Scenario: Tool selector shows an empty state for a missing tool search Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API diff --git a/e2e/features/step-definitions/agent-v2/tools.steps.ts b/e2e/features/step-definitions/agent-v2/tools.steps.ts index 02e56d23a6fbe7..2189bc823b9fbc 100644 --- a/e2e/features/step-definitions/agent-v2/tools.steps.ts +++ b/e2e/features/step-definitions/agent-v2/tools.steps.ts @@ -1,7 +1,11 @@ import type { DifyWorld } from '../../support/world' import { Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' -import { agentBuilderFixedInputs } from '../../agent-v2/support/agent-builder-resources' +import { getAgentComposerDraft } from '../../agent-v2/support/agent' +import { agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../agent-v2/support/agent-builder-resources' +import { asArray, asRecord } from '../../agent-v2/support/preflight/common' +import { hasToolEntry, splitToolDisplayName } from '../../agent-v2/support/preflight/tools' +import { expectProviderToolActionVisible, getCurrentAgentId } from './configure-helpers' const getToolsSection = (world: DifyWorld) => world.getPage().getByRole('region', { name: 'Tools' }) @@ -9,6 +13,68 @@ const getToolsSection = (world: DifyWorld) => const getToolSelectorSearch = (world: DifyWorld) => world.getPage().getByRole('textbox', { name: 'Search integrations...' }) +const getPreseededJsonReplaceTool = (world: DifyWorld) => { + const resource = world.agentBuilder.preflight.preseededResources[ + agentBuilderPreseededResources.jsonReplaceTool + ] + if (!resource || resource.kind !== 'tool') { + throw new Error( + `Preseeded tool "${agentBuilderPreseededResources.jsonReplaceTool}" is not available. Run the matching preflight step first.`, + ) + } + + const parsedDisplayName = splitToolDisplayName(resource.name) + const parsedToolId = splitToolDisplayName(resource.id) + if (!parsedDisplayName.ok) + throw new Error(parsedDisplayName.reason) + if (!parsedToolId.ok) + throw new Error(parsedToolId.reason) + + return { + providerDisplayName: parsedDisplayName.providerName, + providerName: parsedToolId.providerName, + toolDisplayName: parsedDisplayName.toolName, + toolName: parsedToolId.toolName, + } +} + +const expectJsonReplaceToolDraft = async (world: DifyWorld) => { + const agentId = getCurrentAgentId(world) + const tool = getPreseededJsonReplaceTool(world) + + await expect.poll( + async () => { + const draft = await getAgentComposerDraft(agentId) + const tools = asArray(asRecord(draft.agent_soul?.tools).dify_tools) + + return hasToolEntry(tools, tool) + }, + { timeout: 30_000 }, + ).toBe(true) +} + +When( + 'I add the Agent Builder JSON Replace tool from the Tools selector', + async function (this: DifyWorld) { + const page = this.getPage() + const toolsSection = getToolsSection(this) + + await expect(toolsSection).toBeVisible({ timeout: 30_000 }) + await toolsSection.getByRole('button', { name: 'Add tool' }).click() + await page.getByRole('button', { name: /^Tool\b/ }).click() + + const search = getToolSelectorSearch(this) + await expect(search).toBeVisible() + await search.fill('JSON Replace') + + await page.getByRole('button', { exact: true, name: 'JSON Replace' }).click() + await expectProviderToolActionVisible( + toolsSection, + agentBuilderPreseededResources.jsonReplaceTool, + ) + }, +) + When( 'I search for the missing Agent v2 tool from the Tools selector', async function (this: DifyWorld) { @@ -30,6 +96,24 @@ When('I clear the Agent v2 tool selector search', async function (this: DifyWorl await search.fill('') }) +Then( + 'the Agent v2 JSON Replace tool should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + await expectJsonReplaceToolDraft(this) + }, +) + +Then( + 'I should see the Agent v2 JSON Replace tool in the Tools section', + async function (this: DifyWorld) { + await expectProviderToolActionVisible( + getToolsSection(this), + agentBuilderPreseededResources.jsonReplaceTool, + ) + await expectJsonReplaceToolDraft(this) + }, +) + Then('I should see the Agent v2 tool selector empty state', async function (this: DifyWorld) { const page = this.getPage() From 1cd94be6165a2a2b8964dd66d67ca79308fbdcbd Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 02:13:12 +0800 Subject: [PATCH 138/185] test(e2e): split agent service api access scenarios --- e2e/features/agent-v2/access-point.feature | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/e2e/features/agent-v2/access-point.feature b/e2e/features/agent-v2/access-point.feature index 13f079a41537dd..926342f35eadea 100644 --- a/e2e/features/agent-v2/access-point.feature +++ b/e2e/features/agent-v2/access-point.feature @@ -87,7 +87,7 @@ Feature: Agent v2 Access Point Then the Agent v2 Workflow access reference for "E2E Agent Reference Workflow" should open in Studio @core - Scenario: Backend service API supports endpoint copy, key creation, and API reference navigation + Scenario: Backend service API endpoint can be copied Given I am signed in as the default E2E admin And an Agent v2 test agent has been created via API And Agent v2 Backend service API access has been enabled via API @@ -96,13 +96,28 @@ Feature: Agent v2 Access Point Then I should see the Agent v2 Backend service API endpoint When I copy the Agent v2 Backend service API endpoint Then the Agent v2 Backend service API endpoint should show it was copied - When I open Agent v2 API key management + + @core + Scenario: Backend service API keys are managed without exposing existing secrets + Given I am signed in as the default E2E admin + And an Agent v2 test agent has been created via API + And Agent v2 Backend service API access has been enabled via API + When I open the Agent v2 configure page from the Agent Roster + And I switch to the Agent v2 Access Point section + And I open Agent v2 API key management Then Agent v2 API keys should not expose a secret by default When I create a new Agent v2 API key Then I should see the newly generated Agent v2 API key once When I close the newly generated Agent v2 API key Then the Agent v2 API key list should not expose the full generated secret - When I close Agent v2 API key management + + @core + Scenario: Backend service API Reference opens from Access Point + Given I am signed in as the default E2E admin + And an Agent v2 test agent has been created via API + And Agent v2 Backend service API access has been enabled via API + When I open the Agent v2 configure page from the Agent Roster + And I switch to the Agent v2 Access Point section And I open the Agent v2 API Reference Then the Agent v2 API Reference should open in a new tab From 25ef41f03757064d0d165a20cc70d140927f7fe8 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Wed, 1 Jul 2026 18:18:32 +0000 Subject: [PATCH 139/185] [autofix.ci] apply automated fixes --- eslint-suppressions.json | 8 -------- 1 file changed, 8 deletions(-) diff --git a/eslint-suppressions.json b/eslint-suppressions.json index 73a52c346f0d98..634757ebd8c3a3 100644 --- a/eslint-suppressions.json +++ b/eslint-suppressions.json @@ -4637,14 +4637,6 @@ "count": 1 } }, - "web/app/components/workflow/block-selector/tool/action-item.tsx": { - "jsx-a11y/click-events-have-key-events": { - "count": 1 - }, - "jsx-a11y/no-static-element-interactions": { - "count": 1 - } - }, "web/app/components/workflow/block-selector/tool/tool.tsx": { "jsx-a11y/click-events-have-key-events": { "count": 2 From 2929066bfdef4f1880067ae41c0718ef98c58b40 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 02:22:21 +0800 Subject: [PATCH 140/185] test(e2e): split agent builder support helpers --- e2e/features/agent-v2/AGENTS.md | 4 +- .../agent-v2/support/agent-build-draft.ts | 51 ++ e2e/features/agent-v2/support/agent-drive.ts | 291 ++++++++++++ e2e/features/agent-v2/support/agent-soul.ts | 105 +++++ e2e/features/agent-v2/support/agent.ts | 437 +----------------- .../agent-v2/agent-edit.steps.ts | 2 +- .../agent-v2/build-draft.steps.ts | 14 +- .../agent-v2/configure-helpers.ts | 10 +- .../agent-v2/configure.steps.ts | 15 +- .../agent-v2/knowledge.steps.ts | 6 +- .../agent-v2/publish.steps.ts | 3 +- .../agent-v2/workflow-node.steps.ts | 4 +- e2e/features/support/hooks.ts | 3 +- 13 files changed, 481 insertions(+), 464 deletions(-) create mode 100644 e2e/features/agent-v2/support/agent-build-draft.ts create mode 100644 e2e/features/agent-v2/support/agent-drive.ts create mode 100644 e2e/features/agent-v2/support/agent-soul.ts diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index ff3af8be369846..94b92fb944fc43 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -87,7 +87,7 @@ Use the existing namespace shape: - `world.agentBuilder.workflow.agentConsolePage` - `world.agentBuilder.workflow.outputVariables` -Use `features/agent-v2/support/agent.ts` for Agent v2 API fixtures. It owns roster-shaped Agent IDs, configure/access route helpers, composer draft sync, build-draft helpers, publish, Agent drive file cleanup, and Agent cleanup. Use `features/agent-v2/support/access-point.ts` for Web app access, Backend service API access, API keys, and service API request helpers. Store created roster Agent IDs in `DifyWorld.createdAgentIds`; the shared `After` hook deletes them after each scenario. +Use `features/agent-v2/support/agent.ts` for Agent v2 core API fixtures. It owns roster-shaped Agent IDs, configure/access route helpers, composer draft sync, version details, workflow references, publish, and Agent cleanup. Use `features/agent-v2/support/agent-soul.ts` for reusable Agent Soul fixture configuration, prompts, and model/dataset config builders. Use `features/agent-v2/support/agent-build-draft.ts` for Build draft checkout/save/discard API helpers. Use `features/agent-v2/support/agent-drive.ts` for Agent drive/config file and Skill upload plus cleanup helpers. Use `features/agent-v2/support/access-point.ts` for Web app access, Backend service API access, API keys, and service API request helpers. Store created roster Agent IDs in `DifyWorld.createdAgentIds`; the shared `After` hook deletes them after each scenario. Use `DifyWorld.createdAgentDriveFiles` for Agent drive files committed during a scenario and `DifyWorld.createdBuiltinToolCredentials` for built-in tool credentials created during a scenario. The shared `After` hook deletes Agent drive files first so cleanup also works for scenarios that upload into a preseeded Agent. @@ -95,7 +95,7 @@ Use `DifyWorld.createdAgentDriveFiles` for Agent drive files committed during a Use `a basic configured Agent v2 test agent has been created via API` when a scenario only needs a created Agent with a composer draft. Do not use that basic shell for runtime, model, tool, skill, knowledge, environment variable, moderation, or output-variable coverage until those resources have explicit seed helpers and readiness checks. -Use `a runnable Agent v2 test agent has been created via API` after `the Agent Builder stable chat model is available` when a scenario needs a real model-backed Agent. The step writes the preflight model into the Agent Soul model config through `features/agent-v2/support/agent.ts` with deterministic E2E model settings; do not duplicate provider/model payload construction in individual steps. +Use `a runnable Agent v2 test agent has been created via API` after `the Agent Builder stable chat model is available` when a scenario needs a real model-backed Agent. The step writes the preflight model into the Agent Soul model config through `features/agent-v2/support/agent-soul.ts` with deterministic E2E model settings; do not duplicate provider/model payload construction in individual steps. Use `the Agent v2 configuration should be saved automatically` after UI edits that rely on Configure autosave. It waits for the user-visible publish bar saved state; do not replace it with network-idle waits or internal store checks. diff --git a/e2e/features/agent-v2/support/agent-build-draft.ts b/e2e/features/agent-v2/support/agent-build-draft.ts new file mode 100644 index 00000000000000..68227e39d5fab2 --- /dev/null +++ b/e2e/features/agent-v2/support/agent-build-draft.ts @@ -0,0 +1,51 @@ +import type { + AgentBuildDraftResponse, + AgentSoulConfig, +} from '@dify/contracts/api/console/agent/types.gen' +import { createApiContext, expectApiResponseOK } from '../../../support/api' + +export async function checkoutAgentBuildDraft(agentId: string): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.post(`/console/api/agent/${agentId}/build-draft/checkout`, { + data: { force: true }, + }) + await expectApiResponseOK(response, `Checkout Agent v2 build draft for ${agentId}`) + return (await response.json()) as AgentBuildDraftResponse + } + finally { + await ctx.dispose() + } +} + +export async function saveAgentBuildDraft( + agentId: string, + agentSoul: AgentSoulConfig, +): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.put(`/console/api/agent/${agentId}/build-draft`, { + data: { + agent_soul: agentSoul, + save_strategy: 'save_to_current_version', + variant: 'agent_app', + }, + }) + await expectApiResponseOK(response, `Save Agent v2 build draft for ${agentId}`) + return (await response.json()) as AgentBuildDraftResponse + } + finally { + await ctx.dispose() + } +} + +export async function discardAgentBuildDraft(agentId: string): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.delete(`/console/api/agent/${agentId}/build-draft`) + await expectApiResponseOK(response, `Discard Agent v2 build draft for ${agentId}`) + } + finally { + await ctx.dispose() + } +} diff --git a/e2e/features/agent-v2/support/agent-drive.ts b/e2e/features/agent-v2/support/agent-drive.ts new file mode 100644 index 00000000000000..87dfed2098e8bb --- /dev/null +++ b/e2e/features/agent-v2/support/agent-drive.ts @@ -0,0 +1,291 @@ +import type { + AgentConfigFileRefConfig, + AgentConfigFileUploadResponse, + AgentConfigSkillRefConfig, + AgentConfigSkillUploadResponse, + AgentDriveSkillItemResponse, + AgentDriveSkillListResponse, + AgentSkillUploadResponse, +} from '@dify/contracts/api/console/agent/types.gen' +import { Buffer } from 'node:buffer' +import { readFile } from 'node:fs/promises' +import path from 'node:path' +import { createApiContext, expectApiResponseOK } from '../../../support/api' + +export type UploadedConsoleFile = { + id: string + mime_type?: string | null + name: string + size?: number | null +} + +const crc32Table = new Uint32Array(256) +for (let i = 0; i < crc32Table.length; i++) { + let c = i + for (let k = 0; k < 8; k++) + c = c & 1 ? 0xEDB88320 ^ (c >>> 1) : c >>> 1 + crc32Table[i] = c >>> 0 +} + +const crc32 = (buffer: Buffer) => { + let crc = 0xFFFFFFFF + for (const byte of buffer) + crc = crc32Table[(crc ^ byte) & 0xFF]! ^ (crc >>> 8) + return (crc ^ 0xFFFFFFFF) >>> 0 +} + +const createSingleFileZip = ({ + content, + entryName, +}: { + content: Buffer + entryName: string +}) => { + const entryNameBuffer = Buffer.from(entryName) + const checksum = crc32(content) + const localHeader = Buffer.alloc(30) + localHeader.writeUInt32LE(0x04034B50, 0) + localHeader.writeUInt16LE(20, 4) + localHeader.writeUInt16LE(0, 6) + localHeader.writeUInt16LE(0, 8) + localHeader.writeUInt16LE(0, 10) + localHeader.writeUInt16LE(0, 12) + localHeader.writeUInt32LE(checksum, 14) + localHeader.writeUInt32LE(content.length, 18) + localHeader.writeUInt32LE(content.length, 22) + localHeader.writeUInt16LE(entryNameBuffer.length, 26) + localHeader.writeUInt16LE(0, 28) + + const centralDirectoryOffset = localHeader.length + entryNameBuffer.length + content.length + const centralDirectoryHeader = Buffer.alloc(46) + centralDirectoryHeader.writeUInt32LE(0x02014B50, 0) + centralDirectoryHeader.writeUInt16LE(20, 4) + centralDirectoryHeader.writeUInt16LE(20, 6) + centralDirectoryHeader.writeUInt16LE(0, 8) + centralDirectoryHeader.writeUInt16LE(0, 10) + centralDirectoryHeader.writeUInt16LE(0, 12) + centralDirectoryHeader.writeUInt16LE(0, 14) + centralDirectoryHeader.writeUInt32LE(checksum, 16) + centralDirectoryHeader.writeUInt32LE(content.length, 20) + centralDirectoryHeader.writeUInt32LE(content.length, 24) + centralDirectoryHeader.writeUInt16LE(entryNameBuffer.length, 28) + centralDirectoryHeader.writeUInt16LE(0, 30) + centralDirectoryHeader.writeUInt16LE(0, 32) + centralDirectoryHeader.writeUInt16LE(0, 34) + centralDirectoryHeader.writeUInt16LE(0, 36) + centralDirectoryHeader.writeUInt32LE(0, 38) + centralDirectoryHeader.writeUInt32LE(0, 42) + + const centralDirectorySize = centralDirectoryHeader.length + entryNameBuffer.length + const endOfCentralDirectory = Buffer.alloc(22) + endOfCentralDirectory.writeUInt32LE(0x06054B50, 0) + endOfCentralDirectory.writeUInt16LE(0, 4) + endOfCentralDirectory.writeUInt16LE(0, 6) + endOfCentralDirectory.writeUInt16LE(1, 8) + endOfCentralDirectory.writeUInt16LE(1, 10) + endOfCentralDirectory.writeUInt32LE(centralDirectorySize, 12) + endOfCentralDirectory.writeUInt32LE(centralDirectoryOffset, 16) + endOfCentralDirectory.writeUInt16LE(0, 20) + + return Buffer.concat([ + localHeader, + entryNameBuffer, + content, + centralDirectoryHeader, + entryNameBuffer, + endOfCentralDirectory, + ]) +} + +const toSkillArchiveUpload = async ({ + fileName, + filePath, +}: { + fileName: string + filePath: string +}) => { + if (fileName.endsWith('.zip') || fileName.endsWith('.skill')) { + return { + buffer: await readFile(filePath), + name: path.basename(fileName), + } + } + const sourceDirName = path.basename(path.dirname(fileName)) + const archiveBaseName = sourceDirName && sourceDirName !== '.' + ? sourceDirName + : path.basename(fileName, path.extname(fileName)) + + return { + buffer: createSingleFileZip({ + content: await readFile(filePath), + entryName: 'SKILL.md', + }), + name: `${archiveBaseName}.skill`, + } +} + +export async function uploadAgentDriveSkill({ + agentId, + fileName, + filePath, +}: { + agentId: string + fileName: string + filePath: string +}): Promise { + const ctx = await createApiContext() + try { + const upload = await toSkillArchiveUpload({ fileName, filePath }) + const response = await ctx.post(`/console/api/agent/${agentId}/skills/upload`, { + multipart: { + file: { + buffer: upload.buffer, + mimeType: 'application/zip', + name: upload.name, + }, + }, + }) + await expectApiResponseOK(response, `Upload Agent v2 drive skill ${fileName} for ${agentId}`) + return (await response.json()) as AgentSkillUploadResponse + } + finally { + await ctx.dispose() + } +} + +export async function uploadAgentConfigFileToDraft({ + agentId, + fileName, + filePath, +}: { + agentId: string + fileName: string + filePath: string +}): Promise { + const ctx = await createApiContext() + try { + const uploadResponse = await ctx.post('/console/api/files/upload', { + multipart: { + file: { + buffer: await readFile(filePath), + mimeType: 'text/plain', + name: fileName, + }, + }, + }) + await expectApiResponseOK(uploadResponse, `Upload Agent v2 config source file ${fileName}`) + const uploadedFile = (await uploadResponse.json()) as UploadedConsoleFile + + const commitResponse = await ctx.post(`/console/api/agent/${agentId}/config/files`, { + data: { + upload_file_id: uploadedFile.id, + }, + }) + await expectApiResponseOK(commitResponse, `Commit Agent v2 config file ${fileName} for ${agentId}`) + const body = (await commitResponse.json()) as AgentConfigFileUploadResponse + const file = body.file + if (!file.file_id) + throw new Error(`Agent v2 config file ${fileName} did not return a file_id.`) + + return { + file_id: file.file_id, + file_kind: 'upload_file', + hash: file.hash, + mime_type: file.mime_type, + name: file.name, + size: file.size, + } + } + finally { + await ctx.dispose() + } +} + +export async function uploadAgentConfigSkillToDraft({ + agentId, + fileName, + filePath, +}: { + agentId: string + fileName: string + filePath: string +}): Promise { + const ctx = await createApiContext() + try { + const upload = await toSkillArchiveUpload({ fileName, filePath }) + const response = await ctx.post(`/console/api/agent/${agentId}/config/skills/upload`, { + multipart: { + file: { + buffer: upload.buffer, + mimeType: 'application/zip', + name: upload.name, + }, + }, + }) + await expectApiResponseOK(response, `Upload Agent v2 config skill ${fileName} for ${agentId}`) + const body = (await response.json()) as AgentConfigSkillUploadResponse + const skill = body.skill + if (!skill.file_id) + throw new Error(`Agent v2 config skill ${fileName} did not return a file_id.`) + + return { + description: skill.description, + file_id: skill.file_id, + file_kind: 'tool_file', + hash: skill.hash, + mime_type: skill.mime_type, + name: skill.name, + size: skill.size, + } + } + finally { + await ctx.dispose() + } +} + +export async function getAgentDriveSkills(agentId: string): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agentId}/drive/skills`) + await expectApiResponseOK(response, `Get Agent v2 drive skills for ${agentId}`) + const body = (await response.json()) as AgentDriveSkillListResponse + return body.items ?? [] + } + finally { + await ctx.dispose() + } +} + +export async function deleteAgentConfigFile(agentId: string, name: string): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.delete(`/console/api/agent/${agentId}/config/files/${encodeURIComponent(name)}`) + await expectApiResponseOK(response, `Delete Agent v2 config file ${name} for ${agentId}`) + } + finally { + await ctx.dispose() + } +} + +export async function deleteAgentConfigSkill(agentId: string, name: string): Promise { + const ctx = await createApiContext() + try { + const response = await ctx.delete(`/console/api/agent/${agentId}/config/skills/${encodeURIComponent(name)}`) + await expectApiResponseOK(response, `Delete Agent v2 config skill ${name} for ${agentId}`) + } + finally { + await ctx.dispose() + } +} + +export async function deleteAgentDriveFile(agentId: string, key: string): Promise { + const ctx = await createApiContext() + try { + const searchParams = new URLSearchParams({ key }) + const response = await ctx.delete(`/console/api/agent/${agentId}/files?${searchParams}`) + await expectApiResponseOK(response, `Delete Agent v2 drive file ${key} for ${agentId}`) + } + finally { + await ctx.dispose() + } +} diff --git a/e2e/features/agent-v2/support/agent-soul.ts b/e2e/features/agent-v2/support/agent-soul.ts new file mode 100644 index 00000000000000..1cdf59037e4a01 --- /dev/null +++ b/e2e/features/agent-v2/support/agent-soul.ts @@ -0,0 +1,105 @@ +import type { + AgentKnowledgeDatasetConfig, + AgentSoulConfig, +} from '@dify/contracts/api/console/agent/types.gen' + +export type AgentComposerEnvVariable = NonNullable< + NonNullable['variables'] +>[number] + +export type AgentModelSelection = { + name: string + provider: string +} + +export const defaultAgentSoulConfig: AgentSoulConfig = { + prompt: { + system_prompt: 'You are a Dify Agent E2E test assistant.', + }, +} + +export const normalAgentPrompt + = 'You are a Dify Agent E2E test assistant. Reply briefly to every user message, and always include AGENT_E2E_PASS in your response.' + +export const updatedAgentPrompt + = 'You are a Dify Agent E2E test assistant. Every response must start with E2E_AGENT_UPDATED.' + +export const concurrentFirstAgentPrompt + = 'You are a Dify Agent E2E concurrent edit assistant. Always include E2E_CONCURRENT_FIRST in saved instructions.' + +export const concurrentSecondAgentPrompt + = 'You are a Dify Agent E2E concurrent edit assistant. Always include E2E_CONCURRENT_SECOND in saved instructions.' + +export const normalAgentSoulConfig: AgentSoulConfig = { + prompt: { + system_prompt: normalAgentPrompt, + }, +} + +export const updatedAgentSoulConfig: AgentSoulConfig = { + prompt: { + system_prompt: updatedAgentPrompt, + }, +} + +const getAgentModelPluginId = (provider: string) => { + const [organization, pluginName] = provider.split('/').filter(Boolean) + + if (organization && pluginName) + return `${organization}/${pluginName}` + + return provider ? `langgenius/${provider}` : '' +} + +const getExistingModelConfig = (agentSoul: AgentSoulConfig) => { + const model = agentSoul.model + + if (model && typeof model === 'object' && !Array.isArray(model)) + return model as Record + + return {} +} + +export function createAgentSoulConfigWithModel( + agentSoul: AgentSoulConfig, + model: AgentModelSelection, +): AgentSoulConfig { + return { + ...agentSoul, + model: { + ...getExistingModelConfig(agentSoul), + plugin_id: getAgentModelPluginId(model.provider), + model_provider: model.provider, + model: model.name, + model_settings: { + temperature: 0, + max_tokens: 512, + }, + }, + } +} + +export function createAgentSoulConfigWithKnowledgeDataset( + agentSoul: AgentSoulConfig, + dataset: AgentKnowledgeDatasetConfig, +): AgentSoulConfig { + return { + ...agentSoul, + knowledge: { + sets: [ + { + datasets: [dataset], + id: 'e2e-knowledge-retrieval', + name: 'Retrieval 1', + query: { + mode: 'generated_query', + }, + retrieval: { + mode: 'multiple', + top_k: 4, + }, + }, + ], + }, + } +} diff --git a/e2e/features/agent-v2/support/agent.ts b/e2e/features/agent-v2/support/agent.ts index 2effc34b22b46a..b8a7881b100765 100644 --- a/e2e/features/agent-v2/support/agent.ts +++ b/e2e/features/agent-v2/support/agent.ts @@ -1,25 +1,14 @@ import type { AgentAppComposerResponse, AgentAppDetailWithSite, - AgentBuildDraftResponse, - AgentConfigFileRefConfig, - AgentConfigFileUploadResponse, - AgentConfigSkillRefConfig, - AgentConfigSkillUploadResponse, AgentConfigSnapshotDetailResponse, - AgentDriveSkillItemResponse, - AgentDriveSkillListResponse, - AgentKnowledgeDatasetConfig, AgentReferencingWorkflowResponse, AgentReferencingWorkflowsResponse, - AgentSkillUploadResponse, AgentSoulConfig, } from '@dify/contracts/api/console/agent/types.gen' -import { Buffer } from 'node:buffer' -import { readFile } from 'node:fs/promises' -import path from 'node:path' import { createApiContext, expectApiResponseOK } from '../../../support/api' import { assertE2EResourceName, createE2EResourceName } from '../../../support/naming' +import { defaultAgentSoulConfig, normalAgentSoulConfig } from './agent-soul' export type AgentSeed = Pick< AgentAppDetailWithSite, @@ -36,227 +25,15 @@ export type AgentSeed = Pick< active_config_snapshot_id?: string | null } -export type AgentComposerEnvVariable = NonNullable< - NonNullable['variables'] ->[number] -export type AgentModelSelection = { - name: string - provider: string -} - -export type UploadedConsoleFile = { - id: string - mime_type?: string | null - name: string - size?: number | null -} - export type CreateTestAgentOptions = { description?: string name?: string role?: string } -export const defaultAgentSoulConfig: AgentSoulConfig = { - prompt: { - system_prompt: 'You are a Dify Agent E2E test assistant.', - }, -} - -export const normalAgentPrompt - = 'You are a Dify Agent E2E test assistant. Reply briefly to every user message, and always include AGENT_E2E_PASS in your response.' - -export const updatedAgentPrompt - = 'You are a Dify Agent E2E test assistant. Every response must start with E2E_AGENT_UPDATED.' - -export const concurrentFirstAgentPrompt - = 'You are a Dify Agent E2E concurrent edit assistant. Always include E2E_CONCURRENT_FIRST in saved instructions.' - -export const concurrentSecondAgentPrompt - = 'You are a Dify Agent E2E concurrent edit assistant. Always include E2E_CONCURRENT_SECOND in saved instructions.' - -export const normalAgentSoulConfig: AgentSoulConfig = { - prompt: { - system_prompt: normalAgentPrompt, - }, -} - -export const updatedAgentSoulConfig: AgentSoulConfig = { - prompt: { - system_prompt: updatedAgentPrompt, - }, -} - export const getAgentConfigurePath = (agentId: string) => `/roster/agent/${agentId}/configure` export const getAgentAccessPath = (agentId: string) => `/roster/agent/${agentId}/access` -const getAgentModelPluginId = (provider: string) => { - const [organization, pluginName] = provider.split('/').filter(Boolean) - - if (organization && pluginName) - return `${organization}/${pluginName}` - - return provider ? `langgenius/${provider}` : '' -} - -const getExistingModelConfig = (agentSoul: AgentSoulConfig) => { - const model = agentSoul.model - - if (model && typeof model === 'object' && !Array.isArray(model)) - return model as Record - - return {} -} - -const crc32Table = new Uint32Array(256) -for (let i = 0; i < crc32Table.length; i++) { - let c = i - for (let k = 0; k < 8; k++) - c = c & 1 ? 0xEDB88320 ^ (c >>> 1) : c >>> 1 - crc32Table[i] = c >>> 0 -} - -const crc32 = (buffer: Buffer) => { - let crc = 0xFFFFFFFF - for (const byte of buffer) - crc = crc32Table[(crc ^ byte) & 0xFF]! ^ (crc >>> 8) - return (crc ^ 0xFFFFFFFF) >>> 0 -} - -const createSingleFileZip = ({ - content, - entryName, -}: { - content: Buffer - entryName: string -}) => { - const entryNameBuffer = Buffer.from(entryName) - const checksum = crc32(content) - const localHeader = Buffer.alloc(30) - localHeader.writeUInt32LE(0x04034B50, 0) - localHeader.writeUInt16LE(20, 4) - localHeader.writeUInt16LE(0, 6) - localHeader.writeUInt16LE(0, 8) - localHeader.writeUInt16LE(0, 10) - localHeader.writeUInt16LE(0, 12) - localHeader.writeUInt32LE(checksum, 14) - localHeader.writeUInt32LE(content.length, 18) - localHeader.writeUInt32LE(content.length, 22) - localHeader.writeUInt16LE(entryNameBuffer.length, 26) - localHeader.writeUInt16LE(0, 28) - - const centralDirectoryOffset = localHeader.length + entryNameBuffer.length + content.length - const centralDirectoryHeader = Buffer.alloc(46) - centralDirectoryHeader.writeUInt32LE(0x02014B50, 0) - centralDirectoryHeader.writeUInt16LE(20, 4) - centralDirectoryHeader.writeUInt16LE(20, 6) - centralDirectoryHeader.writeUInt16LE(0, 8) - centralDirectoryHeader.writeUInt16LE(0, 10) - centralDirectoryHeader.writeUInt16LE(0, 12) - centralDirectoryHeader.writeUInt16LE(0, 14) - centralDirectoryHeader.writeUInt32LE(checksum, 16) - centralDirectoryHeader.writeUInt32LE(content.length, 20) - centralDirectoryHeader.writeUInt32LE(content.length, 24) - centralDirectoryHeader.writeUInt16LE(entryNameBuffer.length, 28) - centralDirectoryHeader.writeUInt16LE(0, 30) - centralDirectoryHeader.writeUInt16LE(0, 32) - centralDirectoryHeader.writeUInt16LE(0, 34) - centralDirectoryHeader.writeUInt16LE(0, 36) - centralDirectoryHeader.writeUInt32LE(0, 38) - centralDirectoryHeader.writeUInt32LE(0, 42) - - const centralDirectorySize = centralDirectoryHeader.length + entryNameBuffer.length - const endOfCentralDirectory = Buffer.alloc(22) - endOfCentralDirectory.writeUInt32LE(0x06054B50, 0) - endOfCentralDirectory.writeUInt16LE(0, 4) - endOfCentralDirectory.writeUInt16LE(0, 6) - endOfCentralDirectory.writeUInt16LE(1, 8) - endOfCentralDirectory.writeUInt16LE(1, 10) - endOfCentralDirectory.writeUInt32LE(centralDirectorySize, 12) - endOfCentralDirectory.writeUInt32LE(centralDirectoryOffset, 16) - endOfCentralDirectory.writeUInt16LE(0, 20) - - return Buffer.concat([ - localHeader, - entryNameBuffer, - content, - centralDirectoryHeader, - entryNameBuffer, - endOfCentralDirectory, - ]) -} - -const toSkillArchiveUpload = async ({ - fileName, - filePath, -}: { - fileName: string - filePath: string -}) => { - if (fileName.endsWith('.zip') || fileName.endsWith('.skill')) { - return { - buffer: await readFile(filePath), - name: path.basename(fileName), - } - } - const sourceDirName = path.basename(path.dirname(fileName)) - const archiveBaseName = sourceDirName && sourceDirName !== '.' - ? sourceDirName - : path.basename(fileName, path.extname(fileName)) - - return { - buffer: createSingleFileZip({ - content: await readFile(filePath), - entryName: 'SKILL.md', - }), - name: `${archiveBaseName}.skill`, - } -} - -export function createAgentSoulConfigWithModel( - agentSoul: AgentSoulConfig, - model: AgentModelSelection, -): AgentSoulConfig { - return { - ...agentSoul, - model: { - ...getExistingModelConfig(agentSoul), - plugin_id: getAgentModelPluginId(model.provider), - model_provider: model.provider, - model: model.name, - model_settings: { - temperature: 0, - max_tokens: 512, - }, - }, - } -} - -export function createAgentSoulConfigWithKnowledgeDataset( - agentSoul: AgentSoulConfig, - dataset: AgentKnowledgeDatasetConfig, -): AgentSoulConfig { - return { - ...agentSoul, - knowledge: { - sets: [ - { - datasets: [dataset], - id: 'e2e-knowledge-retrieval', - name: 'Retrieval 1', - query: { - mode: 'generated_query', - }, - retrieval: { - mode: 'multiple', - top_k: 4, - }, - }, - ], - }, - } -} - export async function createTestAgent({ description = 'Created by Dify E2E.', name = createE2EResourceName('Agent'), @@ -354,138 +131,6 @@ export async function saveAgentComposerDraft( } } -export async function uploadAgentDriveSkill({ - agentId, - fileName, - filePath, -}: { - agentId: string - fileName: string - filePath: string -}): Promise { - const ctx = await createApiContext() - try { - const upload = await toSkillArchiveUpload({ fileName, filePath }) - const response = await ctx.post(`/console/api/agent/${agentId}/skills/upload`, { - multipart: { - file: { - buffer: upload.buffer, - mimeType: 'application/zip', - name: upload.name, - }, - }, - }) - await expectApiResponseOK(response, `Upload Agent v2 drive skill ${fileName} for ${agentId}`) - return (await response.json()) as AgentSkillUploadResponse - } - finally { - await ctx.dispose() - } -} - -export async function uploadAgentConfigFileToDraft({ - agentId, - fileName, - filePath, -}: { - agentId: string - fileName: string - filePath: string -}): Promise { - const ctx = await createApiContext() - try { - const uploadResponse = await ctx.post('/console/api/files/upload', { - multipart: { - file: { - buffer: await readFile(filePath), - mimeType: 'text/plain', - name: fileName, - }, - }, - }) - await expectApiResponseOK(uploadResponse, `Upload Agent v2 config source file ${fileName}`) - const uploadedFile = (await uploadResponse.json()) as UploadedConsoleFile - - const commitResponse = await ctx.post(`/console/api/agent/${agentId}/config/files`, { - data: { - upload_file_id: uploadedFile.id, - }, - }) - await expectApiResponseOK(commitResponse, `Commit Agent v2 config file ${fileName} for ${agentId}`) - const body = (await commitResponse.json()) as AgentConfigFileUploadResponse - const file = body.file - if (!file.file_id) - throw new Error(`Agent v2 config file ${fileName} did not return a file_id.`) - - return { - file_id: file.file_id, - file_kind: 'upload_file', - hash: file.hash, - mime_type: file.mime_type, - name: file.name, - size: file.size, - } - } - finally { - await ctx.dispose() - } -} - -export async function uploadAgentConfigSkillToDraft({ - agentId, - fileName, - filePath, -}: { - agentId: string - fileName: string - filePath: string -}): Promise { - const ctx = await createApiContext() - try { - const upload = await toSkillArchiveUpload({ fileName, filePath }) - const response = await ctx.post(`/console/api/agent/${agentId}/config/skills/upload`, { - multipart: { - file: { - buffer: upload.buffer, - mimeType: 'application/zip', - name: upload.name, - }, - }, - }) - await expectApiResponseOK(response, `Upload Agent v2 config skill ${fileName} for ${agentId}`) - const body = (await response.json()) as AgentConfigSkillUploadResponse - const skill = body.skill - if (!skill.file_id) - throw new Error(`Agent v2 config skill ${fileName} did not return a file_id.`) - - return { - description: skill.description, - file_id: skill.file_id, - file_kind: 'tool_file', - hash: skill.hash, - mime_type: skill.mime_type, - name: skill.name, - size: skill.size, - } - } - finally { - await ctx.dispose() - } -} - -export async function getAgentDriveSkills(agentId: string): Promise { - const ctx = await createApiContext() - try { - const response = await ctx.get(`/console/api/agent/${agentId}/drive/skills`) - await expectApiResponseOK(response, `Get Agent v2 drive skills for ${agentId}`) - const body = (await response.json()) as AgentDriveSkillListResponse - return body.items ?? [] - } - finally { - await ctx.dispose() - } -} - export async function getAgentReferencingWorkflows(agentId: string): Promise { const ctx = await createApiContext() try { @@ -511,52 +156,6 @@ export async function getAgentComposerDraft(agentId: string): Promise { - const ctx = await createApiContext() - try { - const response = await ctx.post(`/console/api/agent/${agentId}/build-draft/checkout`, { - data: { force: true }, - }) - await expectApiResponseOK(response, `Checkout Agent v2 build draft for ${agentId}`) - return (await response.json()) as AgentBuildDraftResponse - } - finally { - await ctx.dispose() - } -} - -export async function saveAgentBuildDraft( - agentId: string, - agentSoul: AgentSoulConfig, -): Promise { - const ctx = await createApiContext() - try { - const response = await ctx.put(`/console/api/agent/${agentId}/build-draft`, { - data: { - agent_soul: agentSoul, - save_strategy: 'save_to_current_version', - variant: 'agent_app', - }, - }) - await expectApiResponseOK(response, `Save Agent v2 build draft for ${agentId}`) - return (await response.json()) as AgentBuildDraftResponse - } - finally { - await ctx.dispose() - } -} - -export async function discardAgentBuildDraft(agentId: string): Promise { - const ctx = await createApiContext() - try { - const response = await ctx.delete(`/console/api/agent/${agentId}/build-draft`) - await expectApiResponseOK(response, `Discard Agent v2 build draft for ${agentId}`) - } - finally { - await ctx.dispose() - } -} - export async function publishAgent(agentId: string, versionNote = 'E2E publish'): Promise { const ctx = await createApiContext() try { @@ -569,37 +168,3 @@ export async function publishAgent(agentId: string, versionNote = 'E2E publish') await ctx.dispose() } } - -export async function deleteAgentConfigFile(agentId: string, name: string): Promise { - const ctx = await createApiContext() - try { - const response = await ctx.delete(`/console/api/agent/${agentId}/config/files/${encodeURIComponent(name)}`) - await expectApiResponseOK(response, `Delete Agent v2 config file ${name} for ${agentId}`) - } - finally { - await ctx.dispose() - } -} - -export async function deleteAgentConfigSkill(agentId: string, name: string): Promise { - const ctx = await createApiContext() - try { - const response = await ctx.delete(`/console/api/agent/${agentId}/config/skills/${encodeURIComponent(name)}`) - await expectApiResponseOK(response, `Delete Agent v2 config skill ${name} for ${agentId}`) - } - finally { - await ctx.dispose() - } -} - -export async function deleteAgentDriveFile(agentId: string, key: string): Promise { - const ctx = await createApiContext() - try { - const searchParams = new URLSearchParams({ key }) - const response = await ctx.delete(`/console/api/agent/${agentId}/files?${searchParams}`) - await expectApiResponseOK(response, `Delete Agent v2 drive file ${key} for ${agentId}`) - } - finally { - await ctx.dispose() - } -} diff --git a/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts b/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts index 8ba3cab332f1e2..e3b7d3bba077d7 100644 --- a/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts +++ b/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts @@ -6,9 +6,9 @@ import { createE2EResourceName } from '../../../support/naming' import { getAgentComposerDraft, getTestAgent, - normalAgentPrompt, } from '../../agent-v2/support/agent' import { agentBuilderExpectedTokens, agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../agent-v2/support/agent-builder-resources' +import { normalAgentPrompt } from '../../agent-v2/support/agent-soul' import { asArray, asRecord, diff --git a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts index a646a92f33f105..7db97ee0767bb5 100644 --- a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts +++ b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts @@ -3,17 +3,19 @@ import { readFile } from 'node:fs/promises' import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { - createAgentSoulConfigWithModel, getAgentComposerDraft, + saveAgentComposerDraft, +} from '../../agent-v2/support/agent' +import { saveAgentBuildDraft } from '../../agent-v2/support/agent-build-draft' +import { agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../agent-v2/support/agent-builder-resources' +import { uploadAgentConfigFileToDraft } from '../../agent-v2/support/agent-drive' +import { + createAgentSoulConfigWithModel, normalAgentPrompt, normalAgentSoulConfig, - saveAgentBuildDraft, - saveAgentComposerDraft, updatedAgentPrompt, updatedAgentSoulConfig, - uploadAgentConfigFileToDraft, -} from '../../agent-v2/support/agent' -import { agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../agent-v2/support/agent-builder-resources' +} from '../../agent-v2/support/agent-soul' import { skipBlockedPrecondition } from '../../agent-v2/support/preflight/common' import { agentBuilderTestMaterials, getAgentBuilderTestMaterialPath } from '../../agent-v2/support/test-materials' import { diff --git a/e2e/features/step-definitions/agent-v2/configure-helpers.ts b/e2e/features/step-definitions/agent-v2/configure-helpers.ts index d6a77db353796c..e51c89b1da020b 100644 --- a/e2e/features/step-definitions/agent-v2/configure-helpers.ts +++ b/e2e/features/step-definitions/agent-v2/configure-helpers.ts @@ -1,12 +1,10 @@ import type { Locator } from '@playwright/test' -import type { AgentComposerEnvVariable } from '../../agent-v2/support/agent' +import type { AgentComposerEnvVariable } from '../../agent-v2/support/agent-soul' import type { DifyWorld } from '../../support/world' import { expect } from '@playwright/test' -import { - getAgentComposerDraft, - normalAgentPrompt, - uploadAgentConfigSkillToDraft, -} from '../../agent-v2/support/agent' +import { getAgentComposerDraft } from '../../agent-v2/support/agent' +import { uploadAgentConfigSkillToDraft } from '../../agent-v2/support/agent-drive' +import { normalAgentPrompt } from '../../agent-v2/support/agent-soul' import { agentBuilderTestMaterials, getAgentBuilderTestMaterialPath, diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 4e48fc46a4881d..63016ed5f73967 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -4,20 +4,21 @@ import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { waitForAgentConfigureAutosaved } from '../../../support/agent-configure' import { - concurrentFirstAgentPrompt, - concurrentSecondAgentPrompt, - createAgentSoulConfigWithModel, createConfiguredTestAgent, createTestAgent, getAgentComposerDraft, getAgentConfigurePath, - getAgentDriveSkills, + saveAgentComposerDraft, +} from '../../agent-v2/support/agent' +import { getAgentDriveSkills, uploadAgentDriveSkill } from '../../agent-v2/support/agent-drive' +import { + concurrentFirstAgentPrompt, + concurrentSecondAgentPrompt, + createAgentSoulConfigWithModel, normalAgentPrompt, normalAgentSoulConfig, - saveAgentComposerDraft, updatedAgentPrompt, - uploadAgentDriveSkill, -} from '../../agent-v2/support/agent' +} from '../../agent-v2/support/agent-soul' import { agentBuilderTestMaterials, getAgentBuilderTestMaterialPath } from '../../agent-v2/support/test-materials' import { expectNormalAgentPromptDraft, diff --git a/e2e/features/step-definitions/agent-v2/knowledge.steps.ts b/e2e/features/step-definitions/agent-v2/knowledge.steps.ts index 7682ce6956c82d..fad39fac655975 100644 --- a/e2e/features/step-definitions/agent-v2/knowledge.steps.ts +++ b/e2e/features/step-definitions/agent-v2/knowledge.steps.ts @@ -3,12 +3,14 @@ import type { DifyWorld } from '../../support/world' import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { - createAgentSoulConfigWithKnowledgeDataset, createConfiguredTestAgent, getAgentComposerDraft, - normalAgentSoulConfig, } from '../../agent-v2/support/agent' import { agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../agent-v2/support/agent-builder-resources' +import { + createAgentSoulConfigWithKnowledgeDataset, + normalAgentSoulConfig, +} from '../../agent-v2/support/agent-soul' import { asArray, asRecord } from '../../agent-v2/support/preflight/common' import { getCurrentAgentId } from './configure-helpers' diff --git a/e2e/features/step-definitions/agent-v2/publish.steps.ts b/e2e/features/step-definitions/agent-v2/publish.steps.ts index e6f6cb2ee855a5..8bc8b94c751377 100644 --- a/e2e/features/step-definitions/agent-v2/publish.steps.ts +++ b/e2e/features/step-definitions/agent-v2/publish.steps.ts @@ -2,7 +2,8 @@ import type { DifyWorld } from '../../support/world' import { Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { waitForAgentConfigureAutosaved } from '../../../support/agent-configure' -import { getAgentVersionDetail, getTestAgent, normalAgentPrompt } from '../../agent-v2/support/agent' +import { getAgentVersionDetail, getTestAgent } from '../../agent-v2/support/agent' +import { normalAgentPrompt } from '../../agent-v2/support/agent-soul' import { getCurrentAgentId } from './configure-helpers' When('I publish the Agent v2 draft', async function (this: DifyWorld) { diff --git a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts index 5e73387d63851c..80817026a9d99a 100644 --- a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts +++ b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts @@ -9,12 +9,12 @@ import { syncAgentV2WorkflowDraft, } from '../../../support/api' import { createE2EResourceName } from '../../../support/naming' +import { createConfiguredTestAgent } from '../../agent-v2/support/agent' import { createAgentSoulConfigWithModel, - createConfiguredTestAgent, normalAgentPrompt, normalAgentSoulConfig, -} from '../../agent-v2/support/agent' +} from '../../agent-v2/support/agent-soul' const agentV2WorkflowNodeId = 'agent-v2' const taskFileOutputName = 'e2e_report.pdf' diff --git a/e2e/features/support/hooks.ts b/e2e/features/support/hooks.ts index 87be4ac4460465..b9cd6eaabb3659 100644 --- a/e2e/features/support/hooks.ts +++ b/e2e/features/support/hooks.ts @@ -11,7 +11,8 @@ import { deleteTestApp } from '../../support/api' import { deleteTestDataset } from '../../support/datasets' import { deleteBuiltinToolCredential } from '../../support/tools' import { baseURL, cucumberHeadless, cucumberSlowMo } from '../../test-env' -import { deleteAgentConfigFile, deleteAgentConfigSkill, deleteAgentDriveFile, deleteTestAgent } from '../agent-v2/support/agent' +import { deleteTestAgent } from '../agent-v2/support/agent' +import { deleteAgentConfigFile, deleteAgentConfigSkill, deleteAgentDriveFile } from '../agent-v2/support/agent-drive' const e2eRoot = fileURLToPath(new URL('../..', import.meta.url)) const artifactsDir = path.join(e2eRoot, 'cucumber-report', 'artifacts') From 237fc8af5dde39312f9543159caf5bfac33a30bd Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 02:25:33 +0800 Subject: [PATCH 141/185] test(e2e): cover advanced settings collapse state --- e2e/features/agent-v2/advanced-settings.feature | 2 ++ .../agent-v2/advanced-settings.steps.ts | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/e2e/features/agent-v2/advanced-settings.feature b/e2e/features/agent-v2/advanced-settings.feature index 7381bc1bdc4391..a5aaf9e57522cf 100644 --- a/e2e/features/agent-v2/advanced-settings.feature +++ b/e2e/features/agent-v2/advanced-settings.feature @@ -8,6 +8,8 @@ Feature: Agent v2 advanced settings Then Agent v2 Advanced Settings should describe supported entries while collapsed When I expand Agent v2 Advanced Settings Then I should see the supported Agent v2 Advanced Settings entries + When I collapse Agent v2 Advanced Settings + Then Agent v2 Advanced Settings should describe supported entries while collapsed @core Scenario: Plain environment variables are saved and restored diff --git a/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts b/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts index 0ac160dddf28e0..8199b61b8e99b0 100644 --- a/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts +++ b/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts @@ -11,6 +11,16 @@ When('I expand Agent v2 Advanced Settings', async function (this: DifyWorld) { await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() }) +When('I collapse Agent v2 Advanced Settings', async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })) + .not + .toBeVisible() +}) + Then( 'Agent v2 Advanced Settings should describe supported entries while collapsed', async function (this: DifyWorld) { From 945d524d38596c0d30990044de79b698b230bd85 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 02:34:46 +0800 Subject: [PATCH 142/185] test(e2e): move output variable steps to owner file --- .../agent-v2/output-variables.steps.ts | 332 +++++++++++++++++- .../agent-v2/workflow-node.steps.ts | 329 +---------------- 2 files changed, 331 insertions(+), 330 deletions(-) diff --git a/e2e/features/step-definitions/agent-v2/output-variables.steps.ts b/e2e/features/step-definitions/agent-v2/output-variables.steps.ts index 8161fdb714e216..f21f19442177b1 100644 --- a/e2e/features/step-definitions/agent-v2/output-variables.steps.ts +++ b/e2e/features/step-definitions/agent-v2/output-variables.steps.ts @@ -1,7 +1,335 @@ -import type { DifyWorld } from '../../support/world' -import { Given, Then } from '@cucumber/cucumber' +import type { DataTable } from '@cucumber/cucumber' +import type { DeclaredOutputConfig } from '@dify/contracts/api/console/apps/types.gen' +import type { AgentV2WorkflowOutputVariable, DifyWorld } from '../../support/world' +import { Given, Then, When } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { getWorkflowDraft } from '../../../support/api' import { skipBlockedPrecondition } from '../../agent-v2/support/preflight/common' +const agentV2WorkflowNodeId = 'agent-v2' +const taskFileOutputName = 'e2e_report.pdf' +const renamedTaskFileOutputName = 'e2e_final_report.pdf' + +const getAgentOutputToken = (name: string) => `[§output:${name}:${name}§]` + +const getCurrentAppId = (world: DifyWorld) => { + const appId = world.createdAppIds.at(-1) + if (!appId) + throw new Error('No app ID found. Create a workflow app first.') + + return appId +} + +const getAgentV2WorkflowNodeData = async (appId: string) => { + const draft = await getWorkflowDraft(appId) + const agentNode = draft.graph.nodes.find(node => node.id === agentV2WorkflowNodeId) + if (!agentNode) + throw new Error(`Workflow draft ${appId} does not include Agent v2 node ${agentV2WorkflowNodeId}.`) + + return agentNode.data ?? {} +} + +const getDeclaredOutputsFromDraft = async (appId: string): Promise => { + const data = await getAgentV2WorkflowNodeData(appId) + const outputs = data.agent_declared_outputs + if (!Array.isArray(outputs)) + return [] + + return outputs as DeclaredOutputConfig[] +} + +const getOutputVariablesFromDraft = async (appId: string) => getDeclaredOutputsFromDraft(appId) + +const waitForWorkflowDraftSave = (world: DifyWorld, appId: string) => + world.getPage().waitForResponse(response => ( + response.request().method() === 'POST' + && new URL(response.url()).pathname.endsWith(`/console/api/apps/${appId}/workflows/draft`) + )) + +const openWorkflowOutputVariablesPanel = async (world: DifyWorld) => { + const page = world.getPage() + const newOutputButton = page.getByRole('button', { name: 'New output' }) + + if (!await newOutputButton.isVisible().catch(() => false)) + await page.getByRole('button', { name: 'Output Variables' }).click() + + await expect(newOutputButton).toBeVisible() +} + +const fillOutputVariableEditor = async ( + world: DifyWorld, + { + name, + required = false, + type = 'string', + }: { + name: string + required?: boolean + type?: string + }, +) => { + const page = world.getPage() + const editor = page.getByRole('form', { name: 'Output variable editor' }) + + await expect(editor).toBeVisible() + await editor.getByRole('textbox', { name: 'Field name' }).fill(name) + if (type !== 'string') { + await editor.getByRole('button', { name: 'Output type' }).click() + await page.getByRole('option', { name: type, exact: true }).click() + } + if (required) + await editor.getByRole('switch', { name: 'Required' }).click() +} + +When( + 'I insert a file output reference from the Agent v2 workflow node task editor', + async function (this: DifyWorld) { + const page = this.getPage() + const appId = getCurrentAppId(this) + const taskEditor = page.getByRole('textbox', { name: 'Agent task' }) + + await expect(taskEditor).toBeVisible() + await taskEditor.click() + await page.getByRole('button', { name: 'Insert' }).click() + await page.getByRole('button', { name: 'New output' }).click() + + const nameInput = page.getByRole('textbox', { name: 'Field name' }) + await expect(nameInput).toBeVisible() + await nameInput.fill(taskFileOutputName) + + const saveResponse = waitForWorkflowDraftSave(this, appId) + await nameInput.press('Enter') + expect((await saveResponse).ok()).toBe(true) + }, +) + +When( + 'I rename the Agent v2 workflow node task output reference', + async function (this: DifyWorld) { + const page = this.getPage() + const appId = getCurrentAppId(this) + + await page.getByText(taskFileOutputName, { exact: true }).hover() + const editor = page.getByRole('form', { name: 'Output variable editor' }) + await expect(editor).toBeVisible() + await editor.getByRole('textbox', { name: 'Field name' }).fill(renamedTaskFileOutputName) + + const saveResponse = waitForWorkflowDraftSave(this, appId) + await editor.getByRole('button', { name: 'Confirm' }).click() + expect((await saveResponse).ok()).toBe(true) + await expect(editor).not.toBeVisible() + }, +) + +When( + 'I add these Agent v2 workflow node output variables', + async function (this: DifyWorld, table: DataTable) { + const page = this.getPage() + const appId = getCurrentAppId(this) + const rows = table.hashes() as AgentV2WorkflowOutputVariable[] + this.agentBuilder.workflow.outputVariables = rows + + await openWorkflowOutputVariablesPanel(this) + + for (const row of rows) { + await page.getByRole('button', { name: 'New output' }).click() + await fillOutputVariableEditor(this, row) + + const editor = page.getByRole('form', { name: 'Output variable editor' }) + const saveResponse = waitForWorkflowDraftSave(this, appId) + await editor.getByRole('button', { name: 'Confirm' }).click() + expect((await saveResponse).ok()).toBe(true) + await expect(editor).not.toBeVisible() + } + }, +) + +When( + 'I add a required Agent v2 workflow node object output variable with text and analysis fields', + async function (this: DifyWorld) { + const page = this.getPage() + const appId = getCurrentAppId(this) + + await openWorkflowOutputVariablesPanel(this) + await page.getByRole('button', { name: 'New output' }).click() + await fillOutputVariableEditor(this, { + name: 'response', + required: true, + type: 'object', + }) + + let saveResponse = waitForWorkflowDraftSave(this, appId) + await page.getByRole('form', { name: 'Output variable editor' }).getByRole('button', { name: 'Confirm' }).click() + expect((await saveResponse).ok()).toBe(true) + + for (const fieldName of ['text', 'analysis']) { + await page.getByText('response', { exact: true }).hover() + await page.getByRole('button', { name: 'Add response' }).click() + await fillOutputVariableEditor(this, { name: fieldName }) + + saveResponse = waitForWorkflowDraftSave(this, appId) + await page.getByRole('form', { name: 'Output variable editor' }).getByRole('button', { name: 'Confirm' }).click() + expect((await saveResponse).ok()).toBe(true) + } + }, +) + +Then( + 'the Agent v2 workflow node output variables should be saved in the workflow draft', + async function (this: DifyWorld) { + const appId = getCurrentAppId(this) + const expectedOutputVariables = this.agentBuilder.workflow.outputVariables + if (expectedOutputVariables.length === 0) + throw new Error('No Agent v2 workflow output variables were recorded for this scenario.') + + await expect + .poll(async () => { + const outputs = await getOutputVariablesFromDraft(appId) + + return expectedOutputVariables.map((expected) => { + const output = outputs.find(item => item.name === expected.name) + return { + name: output?.name, + type: output?.type === 'array' + ? `array[${output.array_item?.type ?? 'object'}]` + : output?.type, + } + }) + }, { + timeout: 30_000, + }) + .toEqual(expectedOutputVariables) + }, +) + +Then('I should see the Agent v2 workflow node output variables', async function (this: DifyWorld) { + const page = this.getPage() + const expectedOutputVariables = this.agentBuilder.workflow.outputVariables + if (expectedOutputVariables.length === 0) + throw new Error('No Agent v2 workflow output variables were recorded for this scenario.') + + await openWorkflowOutputVariablesPanel(this) + + for (const output of expectedOutputVariables) { + await expect(page.getByText(output.name, { exact: true })).toBeVisible() + await expect(page.getByText(output.type, { exact: true })).toBeVisible() + } +}) + +Then( + 'the Agent v2 workflow node nested object output variable should be saved in the workflow draft', + async function (this: DifyWorld) { + const appId = getCurrentAppId(this) + + await expect + .poll(async () => { + const outputs = await getDeclaredOutputsFromDraft(appId) + const response = outputs.find(output => output.name === 'response') + + return { + children: response?.children?.map(child => ({ + name: child.name, + required: child.required, + type: child.type, + })), + name: response?.name, + required: response?.required, + type: response?.type, + } + }, { + timeout: 30_000, + }) + .toEqual({ + children: [ + { + name: 'text', + required: false, + type: 'string', + }, + { + name: 'analysis', + required: false, + type: 'string', + }, + ], + name: 'response', + required: true, + type: 'object', + }) + }, +) + +Then( + 'the Agent v2 workflow node task should reference the file output', + async function (this: DifyWorld) { + await expectAgentTaskOutputReference(this, taskFileOutputName) + }, +) + +Then( + 'the Agent v2 workflow node task should reference the renamed file output', + async function (this: DifyWorld) { + await expectAgentTaskOutputReference(this, renamedTaskFileOutputName, taskFileOutputName) + }, +) + +Then('I should see the Agent v2 workflow node nested object output variable', async function (this: DifyWorld) { + const page = this.getPage() + + await openWorkflowOutputVariablesPanel(this) + await expect(page.getByText('response', { exact: true })).toBeVisible() + await expect(page.getByText('object', { exact: true })).toBeVisible() + await expect(page.getByText('Required', { exact: true })).toBeVisible() + await expect(page.getByText('text', { exact: true })).toBeVisible() + await expect(page.getByText('analysis', { exact: true })).toBeVisible() + await expect(page.getByText('string', { exact: true })).toBeVisible() +}) + +async function expectAgentTaskOutputReference( + world: DifyWorld, + expectedName: string, + unexpectedName?: string, +) { + const page = world.getPage() + const appId = getCurrentAppId(world) + + await expect.poll( + async () => { + const data = await getAgentV2WorkflowNodeData(appId) + const outputs = Array.isArray(data.agent_declared_outputs) + ? data.agent_declared_outputs as DeclaredOutputConfig[] + : [] + const expectedOutput = outputs.find(output => output.name === expectedName) + + return { + agentTask: data.agent_task, + expectedOutput: expectedOutput + ? { + name: expectedOutput.name, + type: expectedOutput.type, + } + : undefined, + unexpectedOutput: unexpectedName + ? outputs.some(output => output.name === unexpectedName) + : false, + } + }, + { timeout: 30_000 }, + ).toEqual({ + agentTask: expect.stringContaining(getAgentOutputToken(expectedName)), + expectedOutput: { + name: expectedName, + type: 'file', + }, + unexpectedOutput: false, + }) + + await expect(page.getByText(expectedName, { exact: true })).toBeVisible() + await expect(page.getByText('file', { exact: true })).toBeVisible() + if (unexpectedName) + await expect(page.getByText(unexpectedName, { exact: true })).toHaveCount(0) +} + async function skipStandaloneOutputVariables(world: DifyWorld) { return skipBlockedPrecondition( world, diff --git a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts index 80817026a9d99a..8e385485aaa004 100644 --- a/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts +++ b/e2e/features/step-definitions/agent-v2/workflow-node.steps.ts @@ -1,11 +1,8 @@ -import type { DataTable } from '@cucumber/cucumber' -import type { DeclaredOutputConfig } from '@dify/contracts/api/console/apps/types.gen' -import type { AgentV2WorkflowOutputVariable, DifyWorld } from '../../support/world' +import type { DifyWorld } from '../../support/world' import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { createTestApp, - getWorkflowDraft, syncAgentV2WorkflowDraft, } from '../../../support/api' import { createE2EResourceName } from '../../../support/naming' @@ -16,81 +13,6 @@ import { normalAgentSoulConfig, } from '../../agent-v2/support/agent-soul' -const agentV2WorkflowNodeId = 'agent-v2' -const taskFileOutputName = 'e2e_report.pdf' -const renamedTaskFileOutputName = 'e2e_final_report.pdf' - -const getAgentOutputToken = (name: string) => `[§output:${name}:${name}§]` - -const getCurrentAppId = (world: DifyWorld) => { - const appId = world.createdAppIds.at(-1) - if (!appId) - throw new Error('No app ID found. Create a workflow app first.') - - return appId -} - -const getAgentV2WorkflowNodeData = async (appId: string) => { - const draft = await getWorkflowDraft(appId) - const agentNode = draft.graph.nodes.find(node => node.id === agentV2WorkflowNodeId) - if (!agentNode) - throw new Error(`Workflow draft ${appId} does not include Agent v2 node ${agentV2WorkflowNodeId}.`) - - return agentNode.data ?? {} -} - -const getDeclaredOutputsFromDraft = async (appId: string): Promise => { - const data = await getAgentV2WorkflowNodeData(appId) - const outputs = data.agent_declared_outputs - if (!Array.isArray(outputs)) - return [] - - return outputs as DeclaredOutputConfig[] -} - -const getOutputVariablesFromDraft = async (appId: string) => getDeclaredOutputsFromDraft(appId) - -const waitForWorkflowDraftSave = (world: DifyWorld, appId: string) => - world.getPage().waitForResponse(response => ( - response.request().method() === 'POST' - && new URL(response.url()).pathname.endsWith(`/console/api/apps/${appId}/workflows/draft`) - )) - -const openWorkflowOutputVariablesPanel = async (world: DifyWorld) => { - const page = world.getPage() - const newOutputButton = page.getByRole('button', { name: 'New output' }) - - if (!await newOutputButton.isVisible().catch(() => false)) - await page.getByRole('button', { name: 'Output Variables' }).click() - - await expect(newOutputButton).toBeVisible() -} - -const fillOutputVariableEditor = async ( - world: DifyWorld, - { - name, - required = false, - type = 'string', - }: { - name: string - required?: boolean - type?: string - }, -) => { - const page = world.getPage() - const editor = page.getByRole('form', { name: 'Output variable editor' }) - - await expect(editor).toBeVisible() - await editor.getByRole('textbox', { name: 'Field name' }).fill(name) - if (type !== 'string') { - await editor.getByRole('button', { name: 'Output type' }).click() - await page.getByRole('option', { name: type, exact: true }).click() - } - if (required) - await editor.getByRole('switch', { name: 'Required' }).click() -} - Given( 'a workflow app with an Agent v2 node has been created via API', async function (this: DifyWorld) { @@ -150,99 +72,6 @@ When('I open the Agent v2 workflow Agent in Agent Console', async function (this this.agentBuilder.workflow.agentConsolePage = agentConsolePage }) -When( - 'I insert a file output reference from the Agent v2 workflow node task editor', - async function (this: DifyWorld) { - const page = this.getPage() - const appId = getCurrentAppId(this) - const taskEditor = page.getByRole('textbox', { name: 'Agent task' }) - - await expect(taskEditor).toBeVisible() - await taskEditor.click() - await page.getByRole('button', { name: 'Insert' }).click() - await page.getByRole('button', { name: 'New output' }).click() - - const nameInput = page.getByRole('textbox', { name: 'Field name' }) - await expect(nameInput).toBeVisible() - await nameInput.fill(taskFileOutputName) - - const saveResponse = waitForWorkflowDraftSave(this, appId) - await nameInput.press('Enter') - expect((await saveResponse).ok()).toBe(true) - }, -) - -When( - 'I rename the Agent v2 workflow node task output reference', - async function (this: DifyWorld) { - const page = this.getPage() - const appId = getCurrentAppId(this) - - await page.getByText(taskFileOutputName, { exact: true }).hover() - const editor = page.getByRole('form', { name: 'Output variable editor' }) - await expect(editor).toBeVisible() - await editor.getByRole('textbox', { name: 'Field name' }).fill(renamedTaskFileOutputName) - - const saveResponse = waitForWorkflowDraftSave(this, appId) - await editor.getByRole('button', { name: 'Confirm' }).click() - expect((await saveResponse).ok()).toBe(true) - await expect(editor).not.toBeVisible() - }, -) - -When( - 'I add these Agent v2 workflow node output variables', - async function (this: DifyWorld, table: DataTable) { - const page = this.getPage() - const appId = getCurrentAppId(this) - const rows = table.hashes() as AgentV2WorkflowOutputVariable[] - this.agentBuilder.workflow.outputVariables = rows - - await openWorkflowOutputVariablesPanel(this) - - for (const row of rows) { - await page.getByRole('button', { name: 'New output' }).click() - await fillOutputVariableEditor(this, row) - - const editor = page.getByRole('form', { name: 'Output variable editor' }) - const saveResponse = waitForWorkflowDraftSave(this, appId) - await editor.getByRole('button', { name: 'Confirm' }).click() - expect((await saveResponse).ok()).toBe(true) - await expect(editor).not.toBeVisible() - } - }, -) - -When( - 'I add a required Agent v2 workflow node object output variable with text and analysis fields', - async function (this: DifyWorld) { - const page = this.getPage() - const appId = getCurrentAppId(this) - - await openWorkflowOutputVariablesPanel(this) - await page.getByRole('button', { name: 'New output' }).click() - await fillOutputVariableEditor(this, { - name: 'response', - required: true, - type: 'object', - }) - - let saveResponse = waitForWorkflowDraftSave(this, appId) - await page.getByRole('form', { name: 'Output variable editor' }).getByRole('button', { name: 'Confirm' }).click() - expect((await saveResponse).ok()).toBe(true) - - for (const fieldName of ['text', 'analysis']) { - await page.getByText('response', { exact: true }).hover() - await page.getByRole('button', { name: 'Add response' }).click() - await fillOutputVariableEditor(this, { name: fieldName }) - - saveResponse = waitForWorkflowDraftSave(this, appId) - await page.getByRole('form', { name: 'Output variable editor' }).getByRole('button', { name: 'Confirm' }).click() - expect((await saveResponse).ok()).toBe(true) - } - }, -) - Then( 'I should see the Agent v2 workflow Agent details for the created Agent', async function (this: DifyWorld) { @@ -298,159 +127,3 @@ Then( this.agentBuilder.workflow.agentConsolePage = undefined }, ) - -Then( - 'the Agent v2 workflow node output variables should be saved in the workflow draft', - async function (this: DifyWorld) { - const appId = getCurrentAppId(this) - const expectedOutputVariables = this.agentBuilder.workflow.outputVariables - if (expectedOutputVariables.length === 0) - throw new Error('No Agent v2 workflow output variables were recorded for this scenario.') - - await expect - .poll(async () => { - const outputs = await getOutputVariablesFromDraft(appId) - - return expectedOutputVariables.map((expected) => { - const output = outputs.find(item => item.name === expected.name) - return { - name: output?.name, - type: output?.type === 'array' - ? `array[${output.array_item?.type ?? 'object'}]` - : output?.type, - } - }) - }, { - timeout: 30_000, - }) - .toEqual(expectedOutputVariables) - }, -) - -Then('I should see the Agent v2 workflow node output variables', async function (this: DifyWorld) { - const page = this.getPage() - const expectedOutputVariables = this.agentBuilder.workflow.outputVariables - if (expectedOutputVariables.length === 0) - throw new Error('No Agent v2 workflow output variables were recorded for this scenario.') - - await openWorkflowOutputVariablesPanel(this) - - for (const output of expectedOutputVariables) { - await expect(page.getByText(output.name, { exact: true })).toBeVisible() - await expect(page.getByText(output.type, { exact: true })).toBeVisible() - } -}) - -Then( - 'the Agent v2 workflow node nested object output variable should be saved in the workflow draft', - async function (this: DifyWorld) { - const appId = getCurrentAppId(this) - - await expect - .poll(async () => { - const outputs = await getDeclaredOutputsFromDraft(appId) - const response = outputs.find(output => output.name === 'response') - - return { - children: response?.children?.map(child => ({ - name: child.name, - required: child.required, - type: child.type, - })), - name: response?.name, - required: response?.required, - type: response?.type, - } - }, { - timeout: 30_000, - }) - .toEqual({ - children: [ - { - name: 'text', - required: false, - type: 'string', - }, - { - name: 'analysis', - required: false, - type: 'string', - }, - ], - name: 'response', - required: true, - type: 'object', - }) - }, -) - -Then( - 'the Agent v2 workflow node task should reference the file output', - async function (this: DifyWorld) { - await expectAgentTaskOutputReference(this, taskFileOutputName) - }, -) - -Then( - 'the Agent v2 workflow node task should reference the renamed file output', - async function (this: DifyWorld) { - await expectAgentTaskOutputReference(this, renamedTaskFileOutputName, taskFileOutputName) - }, -) - -Then('I should see the Agent v2 workflow node nested object output variable', async function (this: DifyWorld) { - const page = this.getPage() - - await openWorkflowOutputVariablesPanel(this) - await expect(page.getByText('response', { exact: true })).toBeVisible() - await expect(page.getByText('object', { exact: true })).toBeVisible() - await expect(page.getByText('Required', { exact: true })).toBeVisible() - await expect(page.getByText('text', { exact: true })).toBeVisible() - await expect(page.getByText('analysis', { exact: true })).toBeVisible() - await expect(page.getByText('string', { exact: true })).toBeVisible() -}) - -async function expectAgentTaskOutputReference( - world: DifyWorld, - expectedName: string, - unexpectedName?: string, -) { - const page = world.getPage() - const appId = getCurrentAppId(world) - - await expect.poll( - async () => { - const data = await getAgentV2WorkflowNodeData(appId) - const outputs = Array.isArray(data.agent_declared_outputs) - ? data.agent_declared_outputs as DeclaredOutputConfig[] - : [] - const expectedOutput = outputs.find(output => output.name === expectedName) - - return { - agentTask: data.agent_task, - expectedOutput: expectedOutput - ? { - name: expectedOutput.name, - type: expectedOutput.type, - } - : undefined, - unexpectedOutput: unexpectedName - ? outputs.some(output => output.name === unexpectedName) - : false, - } - }, - { timeout: 30_000 }, - ).toEqual({ - agentTask: expect.stringContaining(getAgentOutputToken(expectedName)), - expectedOutput: { - name: expectedName, - type: 'file', - }, - unexpectedOutput: false, - }) - - await expect(page.getByText(expectedName, { exact: true })).toBeVisible() - await expect(page.getByText('file', { exact: true })).toBeVisible() - if (unexpectedName) - await expect(page.getByText(unexpectedName, { exact: true })).toHaveCount(0) -} From e525176d68be806b70461d9dd42f9a4013c4eaeb Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 02:39:53 +0800 Subject: [PATCH 143/185] test(e2e): avoid failure artifacts for skipped scenarios --- e2e/features/support/hooks.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/e2e/features/support/hooks.ts b/e2e/features/support/hooks.ts index b9cd6eaabb3659..2f770e411779d1 100644 --- a/e2e/features/support/hooks.ts +++ b/e2e/features/support/hooks.ts @@ -21,6 +21,13 @@ let browser: Browser | undefined setDefaultTimeout(60_000) +const diagnosticArtifactStatuses = new Set([ + Status.FAILED, + Status.AMBIGUOUS, + Status.PENDING, + Status.UNDEFINED, +]) + const sanitizeForPath = (value: string) => value.replaceAll(/[^\w-]+/g, '-').replaceAll(/^-+|-+$/g, '') @@ -68,8 +75,9 @@ Before(async function (this: DifyWorld, { pickle }) { After(async function (this: DifyWorld, { pickle, result }) { const elapsedMs = this.scenarioStartedAt ? Date.now() - this.scenarioStartedAt : undefined + const status = result?.status || Status.UNKNOWN - if (result?.status !== Status.PASSED && this.page) { + if (diagnosticArtifactStatuses.has(status) && this.page) { const screenshot = await this.page.screenshot({ fullPage: true, }) @@ -89,7 +97,6 @@ After(async function (this: DifyWorld, { pickle, result }) { this.attach(`Artifacts:\n${[screenshotPath, htmlPath].join('\n')}`, 'text/plain') } - const status = result?.status || 'UNKNOWN' console.warn( `[e2e] end ${pickle.name} status=${status}${elapsedMs ? ` durationMs=${elapsedMs}` : ''}`, ) From 43d2cfa8020074f8be5f6d47db26653d4939f233 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 02:42:54 +0800 Subject: [PATCH 144/185] test(e2e): document skipped preflight reporting --- e2e/AGENTS.md | 4 +++- e2e/features/support/hooks.ts | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 69d3f259e06aa6..e7a3ca5cf2ecae 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -284,7 +284,7 @@ When('I fill in the app name in the dialog', async function (this: DifyWorld) { ### Failure diagnostics -The `After` hook automatically captures on failure: +The `After` hook automatically captures diagnostics for failed, ambiguous, pending, undefined, or unknown scenarios: - Full-page screenshot (PNG) - Page HTML dump @@ -292,6 +292,8 @@ The `After` hook automatically captures on failure: Artifacts are saved to `cucumber-report/artifacts/` and attached to the HTML report. No extra code needed in step definitions. +Skipped preflight scenarios should attach the blocked-precondition reason to the skipped step and should not create screenshot or HTML artifacts. + ### Seed resources and preflight checks Use `support/naming.ts` for generated test resource names. New app, Agent, dataset, file, or credential seeds should start with `E2E` so local and shared environments can identify disposable resources. diff --git a/e2e/features/support/hooks.ts b/e2e/features/support/hooks.ts index 2f770e411779d1..ac8b57766abf46 100644 --- a/e2e/features/support/hooks.ts +++ b/e2e/features/support/hooks.ts @@ -26,6 +26,7 @@ const diagnosticArtifactStatuses = new Set([ Status.AMBIGUOUS, Status.PENDING, Status.UNDEFINED, + Status.UNKNOWN, ]) const sanitizeForPath = (value: string) => From ad6dbc339d6a164b641124fb80547dc52298dc97 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 02:46:41 +0800 Subject: [PATCH 145/185] test(e2e): verify published web app launch --- e2e/features/agent-v2/access-point.feature | 15 ++++++++++++++- .../agent-v2/access-point-web-app.steps.ts | 1 + 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/e2e/features/agent-v2/access-point.feature b/e2e/features/agent-v2/access-point.feature index 926342f35eadea..c5528c73c7791a 100644 --- a/e2e/features/agent-v2/access-point.feature +++ b/e2e/features/agent-v2/access-point.feature @@ -9,7 +9,7 @@ Feature: Agent v2 Access Point Then I should see the Agent v2 Access Point overview @core @web-app-access - Scenario: Web app access URL can be copied and launched + Scenario: Web app access URL can be copied without changing orchestration Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API And Agent v2 Web app access has been enabled via API @@ -19,6 +19,19 @@ Feature: Agent v2 Access Point And I record the current Agent v2 orchestration draft When I copy the Agent v2 Web app access URL Then the Agent v2 Web app access URL should show it was copied + And the current Agent v2 orchestration draft should be unchanged + + @core @web-app-access + Scenario: Published Web app can be launched from Access Point + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API + And the Agent v2 draft has been published via API + And Agent v2 Web app access has been enabled via API + When I open the Agent v2 configure page from the Agent Roster + And I switch to the Agent v2 Access Point section + Then I should see the Agent v2 Web app access URL + And I record the current Agent v2 orchestration draft When I launch the Agent v2 Web app Then the Agent v2 Web app should open in a new tab And the current Agent v2 orchestration draft should be unchanged diff --git a/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts b/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts index 85dc25a9cf7a01..9a8bf8c3d0dadb 100644 --- a/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts @@ -92,6 +92,7 @@ Then('the Agent v2 Web app should open in a new tab', async function (this: Dify throw new Error('No Agent v2 Web app page was opened.') await expect(webAppPage).toHaveURL(webAppURL) + await expect(webAppPage.getByRole('textbox').last()).toBeEditable({ timeout: 30_000 }) await webAppPage.close() this.agentBuilder.accessPoint.webAppPage = undefined this.agentBuilder.accessPoint.webAppURL = undefined From eb1d3636294e28974070fae9eaefbb4d910dc85d Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 02:49:14 +0800 Subject: [PATCH 146/185] test(e2e): assert web app entrypoints keep orchestration --- e2e/features/agent-v2/access-point.feature | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/e2e/features/agent-v2/access-point.feature b/e2e/features/agent-v2/access-point.feature index c5528c73c7791a..83ba923612e08e 100644 --- a/e2e/features/agent-v2/access-point.feature +++ b/e2e/features/agent-v2/access-point.feature @@ -43,8 +43,10 @@ Feature: Agent v2 Access Point And Agent v2 Web app access has been enabled via API When I open the Agent v2 configure page from the Agent Roster And I switch to the Agent v2 Access Point section + And I record the current Agent v2 orchestration draft And I open Agent v2 Embedded configuration Then I should see the Agent v2 Embedded configuration dialog + And the current Agent v2 orchestration draft should be unchanged @core @web-app-access Scenario: Web app customization opens from Access Point @@ -53,8 +55,10 @@ Feature: Agent v2 Access Point And Agent v2 Web app access has been enabled via API When I open the Agent v2 configure page from the Agent Roster And I switch to the Agent v2 Access Point section + And I record the current Agent v2 orchestration draft And I open Agent v2 Web app customization Then I should see the Agent v2 Web app customization dialog + And the current Agent v2 orchestration draft should be unchanged @core @web-app-access Scenario: Web app settings open from Access Point without changing orchestration From c78e1c71e39b4ae4366d4da255ebd6261fd648d3 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 02:52:13 +0800 Subject: [PATCH 147/185] test(e2e): verify generated api key copy feedback --- e2e/features/agent-v2/access-point.feature | 2 ++ .../access-point-service-api.steps.ts | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/e2e/features/agent-v2/access-point.feature b/e2e/features/agent-v2/access-point.feature index 83ba923612e08e..1744322a74dfd4 100644 --- a/e2e/features/agent-v2/access-point.feature +++ b/e2e/features/agent-v2/access-point.feature @@ -125,6 +125,8 @@ Feature: Agent v2 Access Point Then Agent v2 API keys should not expose a secret by default When I create a new Agent v2 API key Then I should see the newly generated Agent v2 API key once + When I copy the newly generated Agent v2 API key + Then the newly generated Agent v2 API key should show it was copied When I close the newly generated Agent v2 API key Then the Agent v2 API key list should not expose the full generated secret diff --git a/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts b/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts index ef0f2fe858f3e8..ef8c99bf1c173d 100644 --- a/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts @@ -104,6 +104,25 @@ Then('I should see the newly generated Agent v2 API key once', async function (t throw new Error('Generated Agent v2 API key was empty.') }) +When('I copy the newly generated Agent v2 API key', async function (this: DifyWorld) { + const generatedKeyDialog = this.getPage() + .getByRole('dialog', { name: /API Secret key/i }) + .last() + + await generatedKeyDialog.getByLabel('Copy').first().click() +}) + +Then( + 'the newly generated Agent v2 API key should show it was copied', + async function (this: DifyWorld) { + const generatedKeyDialog = this.getPage() + .getByRole('dialog', { name: /API Secret key/i }) + .last() + + await expect(generatedKeyDialog.getByLabel('Copied')).toBeVisible() + }, +) + When('I close the newly generated Agent v2 API key', async function (this: DifyWorld) { const page = this.getPage() const generatedKeyDialog = page.getByRole('dialog', { name: /API Secret key/i }).last() From f8fad1dfe62682b32c13bbce59611f201a716188 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 02:58:02 +0800 Subject: [PATCH 148/185] test(e2e): verify api reference link target --- .../step-definitions/agent-v2/access-point-service-api.steps.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts b/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts index ef8c99bf1c173d..8482c6dda7a477 100644 --- a/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts @@ -159,6 +159,8 @@ When('I open the Agent v2 API Reference', async function (this: DifyWorld) { const apiReferenceLink = page.getByRole('link', { name: 'API Reference' }) await expect(apiReferenceLink).toBeVisible() + await expect(apiReferenceLink).toHaveAttribute('href', /\/use-dify\/publish\/developing-with-apis/) + await expect(apiReferenceLink).toHaveAttribute('target', '_blank') const [apiReferencePage] = await Promise.all([ page.waitForEvent('popup'), From 12270f23ed1b5ee81418919da6e97f08390a69c6 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 03:08:01 +0800 Subject: [PATCH 149/185] test(e2e): assert build draft keeps tools isolated --- e2e/features/agent-v2/build-draft.feature | 1 + e2e/features/agent-v2/support/tools.ts | 25 ++++++++++++++++ .../agent-v2/build-draft.steps.ts | 22 +++++++++++++- .../step-definitions/agent-v2/tools.steps.ts | 30 ++----------------- 4 files changed, 50 insertions(+), 28 deletions(-) create mode 100644 e2e/features/agent-v2/support/tools.ts diff --git a/e2e/features/agent-v2/build-draft.feature b/e2e/features/agent-v2/build-draft.feature index dc4b8353c6ff26..4d1e2570430638 100644 --- a/e2e/features/agent-v2/build-draft.feature +++ b/e2e/features/agent-v2/build-draft.feature @@ -12,6 +12,7 @@ Feature: Agent v2 build draft Then I should see the Agent v2 Build draft pending changes And I should see the Agent v2 Build mode confirmation state And the normal Agent v2 draft should still use the normal E2E prompt + And the normal Agent v2 draft should not include the Agent Builder JSON Replace tool @core Scenario: Discarding a Build draft keeps the original Agent configuration diff --git a/e2e/features/agent-v2/support/tools.ts b/e2e/features/agent-v2/support/tools.ts new file mode 100644 index 00000000000000..d7f6eb60fe5db8 --- /dev/null +++ b/e2e/features/agent-v2/support/tools.ts @@ -0,0 +1,25 @@ +import type { DifyWorld } from '../../support/world' +import { splitToolDisplayName } from './preflight/tools' + +export const getPreseededToolContract = (world: DifyWorld, resourceName: string) => { + const resource = world.agentBuilder.preflight.preseededResources[resourceName] + if (!resource || resource.kind !== 'tool') { + throw new Error( + `Preseeded tool "${resourceName}" is not available. Run the matching preflight step first.`, + ) + } + + const parsedDisplayName = splitToolDisplayName(resource.name) + const parsedToolId = splitToolDisplayName(resource.id) + if (!parsedDisplayName.ok) + throw new Error(parsedDisplayName.reason) + if (!parsedToolId.ok) + throw new Error(parsedToolId.reason) + + return { + providerDisplayName: parsedDisplayName.providerName, + providerName: parsedToolId.providerName, + toolDisplayName: parsedDisplayName.toolName, + toolName: parsedToolId.toolName, + } +} diff --git a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts index 7db97ee0767bb5..3bb4a215276840 100644 --- a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts +++ b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts @@ -16,8 +16,10 @@ import { updatedAgentPrompt, updatedAgentSoulConfig, } from '../../agent-v2/support/agent-soul' -import { skipBlockedPrecondition } from '../../agent-v2/support/preflight/common' +import { asArray, asRecord, skipBlockedPrecondition } from '../../agent-v2/support/preflight/common' +import { hasToolEntry } from '../../agent-v2/support/preflight/tools' import { agentBuilderTestMaterials, getAgentBuilderTestMaterialPath } from '../../agent-v2/support/test-materials' +import { getPreseededToolContract } from '../../agent-v2/support/tools' import { getAgentEnvVariableValue, getCurrentAgentId, @@ -240,6 +242,24 @@ Then('I should see one e2e-summary-skill Skill in the Skills section', async fun })).toHaveCount(1) }) +Then( + 'the normal Agent v2 draft should not include the Agent Builder JSON Replace tool', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + const tool = getPreseededToolContract(this, agentBuilderPreseededResources.jsonReplaceTool) + + await expect.poll( + async () => { + const draft = await getAgentComposerDraft(agentId) + const tools = asArray(asRecord(draft.agent_soul?.tools).dify_tools) + + return hasToolEntry(tools, tool) + }, + { timeout: 30_000 }, + ).toBe(false) + }, +) + Then( 'the Agent v2 draft should include the supported Build draft config', async function (this: DifyWorld) { diff --git a/e2e/features/step-definitions/agent-v2/tools.steps.ts b/e2e/features/step-definitions/agent-v2/tools.steps.ts index 2189bc823b9fbc..4bbc0d876e44c8 100644 --- a/e2e/features/step-definitions/agent-v2/tools.steps.ts +++ b/e2e/features/step-definitions/agent-v2/tools.steps.ts @@ -4,7 +4,8 @@ import { expect } from '@playwright/test' import { getAgentComposerDraft } from '../../agent-v2/support/agent' import { agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../agent-v2/support/agent-builder-resources' import { asArray, asRecord } from '../../agent-v2/support/preflight/common' -import { hasToolEntry, splitToolDisplayName } from '../../agent-v2/support/preflight/tools' +import { hasToolEntry } from '../../agent-v2/support/preflight/tools' +import { getPreseededToolContract } from '../../agent-v2/support/tools' import { expectProviderToolActionVisible, getCurrentAgentId } from './configure-helpers' const getToolsSection = (world: DifyWorld) => @@ -13,34 +14,9 @@ const getToolsSection = (world: DifyWorld) => const getToolSelectorSearch = (world: DifyWorld) => world.getPage().getByRole('textbox', { name: 'Search integrations...' }) -const getPreseededJsonReplaceTool = (world: DifyWorld) => { - const resource = world.agentBuilder.preflight.preseededResources[ - agentBuilderPreseededResources.jsonReplaceTool - ] - if (!resource || resource.kind !== 'tool') { - throw new Error( - `Preseeded tool "${agentBuilderPreseededResources.jsonReplaceTool}" is not available. Run the matching preflight step first.`, - ) - } - - const parsedDisplayName = splitToolDisplayName(resource.name) - const parsedToolId = splitToolDisplayName(resource.id) - if (!parsedDisplayName.ok) - throw new Error(parsedDisplayName.reason) - if (!parsedToolId.ok) - throw new Error(parsedToolId.reason) - - return { - providerDisplayName: parsedDisplayName.providerName, - providerName: parsedToolId.providerName, - toolDisplayName: parsedDisplayName.toolName, - toolName: parsedToolId.toolName, - } -} - const expectJsonReplaceToolDraft = async (world: DifyWorld) => { const agentId = getCurrentAgentId(world) - const tool = getPreseededJsonReplaceTool(world) + const tool = getPreseededToolContract(world, agentBuilderPreseededResources.jsonReplaceTool) await expect.poll( async () => { From 54740ff2747278785b88fcffa46eb67311c23808 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 03:16:43 +0800 Subject: [PATCH 150/185] test(e2e): tag stable model dependent scenarios --- e2e/features/agent-v2/AGENTS.md | 1 + e2e/features/agent-v2/access-point.feature | 6 +++--- e2e/features/agent-v2/agent-edit.feature | 6 +++--- e2e/features/agent-v2/build-draft.feature | 10 +++++----- e2e/features/agent-v2/configure-persistence.feature | 4 ++-- e2e/features/agent-v2/output-variables.feature | 12 ++++++------ e2e/features/agent-v2/publish.feature | 12 ++++++------ 7 files changed, 26 insertions(+), 25 deletions(-) diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 94b92fb944fc43..2d5479ea151882 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -35,6 +35,7 @@ Use tags in three layers: - `@agent-edit` — saved Agent detail/configuration display surfaces. - `@publish` — publish and publish-bar state. - `@access-point` — Web app, Backend service API, and Workflow access surfaces. +- `@stable-model` — active model fixture dependency. Apply this to every scenario that includes `the Agent Builder stable chat model is available` or otherwise requires an active model configured in the workspace. - `@web-app-runtime` — published public Web app runtime behavior. Use it for scenarios that open the public Web app and assert real chat responses. Access Point URL, launch, customization, and settings surfaces remain `@access-point` behavior unless they send messages through the public Web app. - `@service-api-runtime` — Backend service API runtime behavior. Use it for scenarios that call the published service API and assert real chat responses. Endpoint display, copy, API key, and API reference surfaces remain `@access-point` behavior. - `@feature-gated` — product capability is optional. This tag alone does not skip execution; the scenario must include an explicit step that returns `skipped` with a blocked-precondition reason when the feature is unavailable. diff --git a/e2e/features/agent-v2/access-point.feature b/e2e/features/agent-v2/access-point.feature index 1744322a74dfd4..1e83c04a662f0d 100644 --- a/e2e/features/agent-v2/access-point.feature +++ b/e2e/features/agent-v2/access-point.feature @@ -21,7 +21,7 @@ Feature: Agent v2 Access Point Then the Agent v2 Web app access URL should show it was copied And the current Agent v2 orchestration draft should be unchanged - @core @web-app-access + @core @web-app-access @stable-model Scenario: Published Web app can be launched from Access Point Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -72,7 +72,7 @@ Feature: Agent v2 Access Point Then I should see the Agent v2 Web app settings dialog And the current Agent v2 orchestration draft should be unchanged - @core @web-app-access + @core @web-app-access @stable-model Scenario: Web app access can be disabled and restored Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -140,7 +140,7 @@ Feature: Agent v2 Access Point And I open the Agent v2 API Reference Then the Agent v2 API Reference should open in a new tab - @service-api-runtime + @service-api-runtime @stable-model Scenario: Backend service API can be disabled and restored Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available diff --git a/e2e/features/agent-v2/agent-edit.feature b/e2e/features/agent-v2/agent-edit.feature index 2eb6d00f100760..e48afa9903f659 100644 --- a/e2e/features/agent-v2/agent-edit.feature +++ b/e2e/features/agent-v2/agent-edit.feature @@ -1,6 +1,6 @@ @agent-v2 @authenticated @agent-edit Feature: Agent v2 Agent Edit page - @core + @core @stable-model Scenario: Saved orchestration sections are visible on the Agent Edit page Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -9,7 +9,7 @@ Feature: Agent v2 Agent Edit page When I open the preseeded Agent v2 configure page for "E2E New Agent Builder Full Config" from the Agent Roster Then I should see the Agent v2 full-config fixture sections - @core + @core @stable-model Scenario: Duplicated Agent inherits configuration without changing the original Agent Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -57,7 +57,7 @@ Feature: Agent v2 Agent Edit page When I open the preseeded Agent v2 configure page for "E2E Agent With Dual Retrieval" from the Agent Roster Then I should see the Agent v2 dual retrieval fixture settings - @core + @core @stable-model Scenario: Agent Edit opens the same Agent in Agent Console Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available diff --git a/e2e/features/agent-v2/build-draft.feature b/e2e/features/agent-v2/build-draft.feature index 4d1e2570430638..283ea92368a26c 100644 --- a/e2e/features/agent-v2/build-draft.feature +++ b/e2e/features/agent-v2/build-draft.feature @@ -1,6 +1,6 @@ @agent-v2 @authenticated @build Feature: Agent v2 build draft - @core + @core @stable-model Scenario: Generating a Build draft leaves the normal Agent configuration unchanged Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -57,7 +57,7 @@ Feature: Agent v2 build draft And the Agent v2 draft should not include the supported Build draft config And the Agent v2 Build draft should no longer be active - @core + @core @stable-model Scenario: Applying a pending Build draft updates the normal Agent configuration Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -75,7 +75,7 @@ Feature: Agent v2 build draft Then I should see the updated E2E prompt in the Agent v2 prompt editor And the Agent v2 Build draft should no longer be active - @core + @core @stable-model Scenario: Applying a Build draft updates supported configuration sections Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -102,7 +102,7 @@ Feature: Agent v2 build draft And I should see the supported E2E environment variable in Advanced Settings And the Agent v2 Build draft should no longer be active - @core + @core @stable-model Scenario: Applying a Build draft with an existing Skill keeps a single Skill entry Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -143,7 +143,7 @@ Feature: Agent v2 build draft When I open the Agent v2 configure page Then Agent v2 Build chat Dify Tool writeback should be available - @build-unavailable-resources @feature-gated + @build-unavailable-resources @feature-gated @stable-model Scenario: Build chat reports unavailable Skill or Tool requests clearly Given I am signed in as the default E2E admin And Agent v2 Build chat unavailable Skill and Tool recovery is available diff --git a/e2e/features/agent-v2/configure-persistence.feature b/e2e/features/agent-v2/configure-persistence.feature index 7a177dba8e0e8d..0c6abede24779d 100644 --- a/e2e/features/agent-v2/configure-persistence.feature +++ b/e2e/features/agent-v2/configure-persistence.feature @@ -1,6 +1,6 @@ @agent-v2 @authenticated @core Feature: Agent v2 configure persistence - @configure-persistence + @configure-persistence @stable-model Scenario: Selecting a stable model in Configure persists after refresh Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -16,7 +16,7 @@ Feature: Agent v2 configure persistence And I should see the normal E2E prompt in the Agent v2 prompt editor And the Agent v2 draft should use the stable E2E model - @configure-persistence + @configure-persistence @stable-model Scenario: Persisted Agent v2 instructions remain visible after refresh Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available diff --git a/e2e/features/agent-v2/output-variables.feature b/e2e/features/agent-v2/output-variables.feature index aa351531ccb434..0877d1ea0ae81c 100644 --- a/e2e/features/agent-v2/output-variables.feature +++ b/e2e/features/agent-v2/output-variables.feature @@ -8,7 +8,7 @@ Feature: Agent v2 output variables When I open the Agent v2 configure page Then Agent v2 standalone Output Variables should be available - @core + @core @stable-model Scenario: Workflow Agent v2 output variables persist after refresh Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -25,7 +25,7 @@ Feature: Agent v2 output variables And I open the Agent v2 workflow node panel Then I should see the Agent v2 workflow node output variables - @core + @core @stable-model Scenario: Workflow Agent v2 nested object output variables persist after refresh Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -38,7 +38,7 @@ Feature: Agent v2 output variables And I open the Agent v2 workflow node panel Then I should see the Agent v2 workflow node nested object output variable - @core + @core @stable-model Scenario: Workflow Agent v2 prompt output reference stays synced when renamed Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -53,7 +53,7 @@ Feature: Agent v2 output variables And I open the Agent v2 workflow node panel Then the Agent v2 workflow node task should reference the renamed file output - @output-reference-delete @feature-gated + @output-reference-delete @feature-gated @stable-model Scenario: Workflow Agent v2 prompt output reference deletion remains explicit Given I am signed in as the default E2E admin And Agent v2 workflow task output reference deletion consistency is available @@ -64,7 +64,7 @@ Feature: Agent v2 output variables And I insert a file output reference from the Agent v2 workflow node task editor Then Agent v2 workflow task output reference deletion consistency should be available - @output-retry-strategy @feature-gated + @output-retry-strategy @feature-gated @stable-model Scenario: Workflow Agent v2 output retry strategy can be saved after refresh Given I am signed in as the default E2E admin And Agent v2 workflow output retry strategy is available @@ -74,7 +74,7 @@ Feature: Agent v2 output variables And I open the Agent v2 workflow node panel Then Agent v2 workflow output retry strategy should be available - @output-retry-validation @feature-gated + @output-retry-validation @feature-gated @stable-model Scenario: Workflow Agent v2 output retry count validation is enforced Given I am signed in as the default E2E admin And Agent v2 workflow output retry count validation is available diff --git a/e2e/features/agent-v2/publish.feature b/e2e/features/agent-v2/publish.feature index 4f04d2771be062..6b92f52a916055 100644 --- a/e2e/features/agent-v2/publish.feature +++ b/e2e/features/agent-v2/publish.feature @@ -1,6 +1,6 @@ @agent-v2 @authenticated @publish Feature: Agent v2 publish - @core + @core @stable-model Scenario: Publish a configured Agent v2 draft Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -9,7 +9,7 @@ Feature: Agent v2 publish And I publish the Agent v2 draft Then the Agent v2 draft should be published and up to date - @core + @core @stable-model Scenario: Publish action follows unpublished changes Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -23,7 +23,7 @@ Feature: Agent v2 publish Then the Agent v2 configuration should be saved automatically And the Agent v2 publish action should be available for unpublished changes - @core + @core @stable-model Scenario: Published Agent v2 version remains isolated from draft edits Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -36,7 +36,7 @@ Feature: Agent v2 publish And the normal Agent v2 draft should use the updated E2E prompt And the active published Agent v2 version should still use the normal E2E prompt - @core + @core @stable-model Scenario: Restoring a published Agent v2 version shows the restored configuration in Builder Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -58,7 +58,7 @@ Feature: Agent v2 publish And the normal Agent v2 draft should use the normal E2E prompt And the Agent v2 publish action should be available for unpublished changes - @web-app-runtime + @web-app-runtime @stable-model Scenario: Published Web app remains isolated from unpublished Agent v2 draft edits Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -75,7 +75,7 @@ Feature: Agent v2 publish Then the Agent v2 Web app response should include the normal E2E marker And the Agent v2 Web app response should not include the updated E2E marker - @web-app-runtime + @web-app-runtime @stable-model Scenario: Published Web app uses the latest Agent v2 published configuration Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available From 038ed9a3506b7efec19da9843c6013c61d66c4d1 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 03:29:05 +0800 Subject: [PATCH 151/185] test(e2e): tag agent builder fixture dependencies --- e2e/features/agent-v2/AGENTS.md | 9 +++++++++ e2e/features/agent-v2/access-point.feature | 20 ++++++++++---------- e2e/features/agent-v2/agent-edit.feature | 12 ++++++------ e2e/features/agent-v2/build-draft.feature | 8 ++++---- e2e/features/agent-v2/knowledge.feature | 2 +- e2e/features/agent-v2/preflight.feature | 18 ++++++++++++++++++ e2e/features/agent-v2/publish.feature | 4 ++-- e2e/features/agent-v2/tools.feature | 2 +- 8 files changed, 51 insertions(+), 24 deletions(-) diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 2d5479ea151882..284885b7bae56a 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -36,6 +36,15 @@ Use tags in three layers: - `@publish` — publish and publish-bar state. - `@access-point` — Web app, Backend service API, and Workflow access surfaces. - `@stable-model` — active model fixture dependency. Apply this to every scenario that includes `the Agent Builder stable chat model is available` or otherwise requires an active model configured in the workspace. +- `@tool-fixture` — preseeded Tool dependency such as `JSON Process / JSON Replace` or `Tavily / Tavily Search`. +- `@skill-fixture` — checked-in or preseeded Skill dependency such as `e2e-summary-skill`. +- `@knowledge-fixture` — preseeded dataset dependency such as `E2E Agent Knowledge Base`. +- `@full-config-agent` — fixed `E2E New Agent Builder Full Config` Agent dependency. +- `@tool-states-agent` — fixed `E2E New Agent Builder Tool States` Agent dependency. +- `@file-tree-fixture` — fixed file-tree Agent drive/config-files dependency. +- `@dual-retrieval-fixture` — fixed dual Knowledge Retrieval Agent dependency. +- `@backend-api-access` — fixed or scenario-owned Backend service API access dependency. +- `@published-web-app` — fixed or scenario-owned published Web app access dependency. - `@web-app-runtime` — published public Web app runtime behavior. Use it for scenarios that open the public Web app and assert real chat responses. Access Point URL, launch, customization, and settings surfaces remain `@access-point` behavior unless they send messages through the public Web app. - `@service-api-runtime` — Backend service API runtime behavior. Use it for scenarios that call the published service API and assert real chat responses. Endpoint display, copy, API key, and API reference surfaces remain `@access-point` behavior. - `@feature-gated` — product capability is optional. This tag alone does not skip execution; the scenario must include an explicit step that returns `skipped` with a blocked-precondition reason when the feature is unavailable. diff --git a/e2e/features/agent-v2/access-point.feature b/e2e/features/agent-v2/access-point.feature index 1e83c04a662f0d..0d6349759b879f 100644 --- a/e2e/features/agent-v2/access-point.feature +++ b/e2e/features/agent-v2/access-point.feature @@ -8,7 +8,7 @@ Feature: Agent v2 Access Point And I switch to the Agent v2 Access Point section Then I should see the Agent v2 Access Point overview - @core @web-app-access + @core @web-app-access @published-web-app Scenario: Web app access URL can be copied without changing orchestration Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API @@ -21,7 +21,7 @@ Feature: Agent v2 Access Point Then the Agent v2 Web app access URL should show it was copied And the current Agent v2 orchestration draft should be unchanged - @core @web-app-access @stable-model + @core @web-app-access @published-web-app @stable-model Scenario: Published Web app can be launched from Access Point Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -36,7 +36,7 @@ Feature: Agent v2 Access Point Then the Agent v2 Web app should open in a new tab And the current Agent v2 orchestration draft should be unchanged - @core @web-app-access + @core @web-app-access @published-web-app Scenario: Web app Embedded configuration opens from Access Point Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API @@ -48,7 +48,7 @@ Feature: Agent v2 Access Point Then I should see the Agent v2 Embedded configuration dialog And the current Agent v2 orchestration draft should be unchanged - @core @web-app-access + @core @web-app-access @published-web-app Scenario: Web app customization opens from Access Point Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API @@ -60,7 +60,7 @@ Feature: Agent v2 Access Point Then I should see the Agent v2 Web app customization dialog And the current Agent v2 orchestration draft should be unchanged - @core @web-app-access + @core @web-app-access @published-web-app Scenario: Web app settings open from Access Point without changing orchestration Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API @@ -72,7 +72,7 @@ Feature: Agent v2 Access Point Then I should see the Agent v2 Web app settings dialog And the current Agent v2 orchestration draft should be unchanged - @core @web-app-access @stable-model + @core @web-app-access @published-web-app @stable-model Scenario: Web app access can be disabled and restored Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -103,7 +103,7 @@ Feature: Agent v2 Access Point When I open the Agent v2 Workflow access reference for "E2E Agent Reference Workflow" Then the Agent v2 Workflow access reference for "E2E Agent Reference Workflow" should open in Studio - @core + @core @backend-api-access Scenario: Backend service API endpoint can be copied Given I am signed in as the default E2E admin And an Agent v2 test agent has been created via API @@ -114,7 +114,7 @@ Feature: Agent v2 Access Point When I copy the Agent v2 Backend service API endpoint Then the Agent v2 Backend service API endpoint should show it was copied - @core + @core @backend-api-access Scenario: Backend service API keys are managed without exposing existing secrets Given I am signed in as the default E2E admin And an Agent v2 test agent has been created via API @@ -130,7 +130,7 @@ Feature: Agent v2 Access Point When I close the newly generated Agent v2 API key Then the Agent v2 API key list should not expose the full generated secret - @core + @core @backend-api-access Scenario: Backend service API Reference opens from Access Point Given I am signed in as the default E2E admin And an Agent v2 test agent has been created via API @@ -140,7 +140,7 @@ Feature: Agent v2 Access Point And I open the Agent v2 API Reference Then the Agent v2 API Reference should open in a new tab - @service-api-runtime @stable-model + @service-api-runtime @backend-api-access @stable-model Scenario: Backend service API can be disabled and restored Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available diff --git a/e2e/features/agent-v2/agent-edit.feature b/e2e/features/agent-v2/agent-edit.feature index e48afa9903f659..823a3a91015068 100644 --- a/e2e/features/agent-v2/agent-edit.feature +++ b/e2e/features/agent-v2/agent-edit.feature @@ -1,6 +1,6 @@ @agent-v2 @authenticated @agent-edit Feature: Agent v2 Agent Edit page - @core @stable-model + @core @stable-model @full-config-agent Scenario: Saved orchestration sections are visible on the Agent Edit page Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -9,7 +9,7 @@ Feature: Agent v2 Agent Edit page When I open the preseeded Agent v2 configure page for "E2E New Agent Builder Full Config" from the Agent Roster Then I should see the Agent v2 full-config fixture sections - @core @stable-model + @core @stable-model @full-config-agent Scenario: Duplicated Agent inherits configuration without changing the original Agent Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -23,7 +23,7 @@ Feature: Agent v2 Agent Edit page And the normal Agent v2 draft should use the updated E2E prompt And the preseeded Agent v2 "E2E New Agent Builder Full Config" should still use the normal E2E prompt - @core + @core @tool-states-agent Scenario: Tool states are visible on the Agent Edit page Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" is available @@ -31,7 +31,7 @@ Feature: Agent v2 Agent Edit page When I open the preseeded Agent v2 configure page for "E2E New Agent Builder Tool States" from the Agent Roster Then I should see the Agent v2 tool state fixture tools - @tool-error-state @feature-gated + @tool-error-state @tool-states-agent @feature-gated Scenario: Tool credential error states are visible on the Agent Edit page Given I am signed in as the default E2E admin And Agent v2 Tool credential error state is available @@ -40,7 +40,7 @@ Feature: Agent v2 Agent Edit page When I open the preseeded Agent v2 configure page for "E2E New Agent Builder Tool States" from the Agent Roster Then Agent v2 Tool credential error state should be available - @core + @core @file-tree-fixture Scenario: File fixture entries are visible in the current flat Files list Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E Agent With File Tree" is available @@ -49,7 +49,7 @@ Feature: Agent v2 Agent Edit page When I open the preseeded Agent v2 configure page for "E2E Agent With File Tree" from the Agent Roster Then I should see the Agent v2 file fixture entries in the current flat Files list - @core + @core @dual-retrieval-fixture Scenario: Dual Knowledge Retrieval settings are visible on the Agent Edit page Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E Agent With Dual Retrieval" is available diff --git a/e2e/features/agent-v2/build-draft.feature b/e2e/features/agent-v2/build-draft.feature index 283ea92368a26c..313cff30a8f0da 100644 --- a/e2e/features/agent-v2/build-draft.feature +++ b/e2e/features/agent-v2/build-draft.feature @@ -1,6 +1,6 @@ @agent-v2 @authenticated @build Feature: Agent v2 build draft - @core @stable-model + @core @stable-model @tool-fixture @skill-fixture Scenario: Generating a Build draft leaves the normal Agent configuration unchanged Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -31,7 +31,7 @@ Feature: Agent v2 build draft Then I should see the normal E2E prompt in the Agent v2 prompt editor And the Agent v2 Build draft should no longer be active - @core + @core @skill-fixture Scenario: Discarding a Build draft does not apply supported configuration changes Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API @@ -75,7 +75,7 @@ Feature: Agent v2 build draft Then I should see the updated E2E prompt in the Agent v2 prompt editor And the Agent v2 Build draft should no longer be active - @core @stable-model + @core @stable-model @skill-fixture Scenario: Applying a Build draft updates supported configuration sections Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -102,7 +102,7 @@ Feature: Agent v2 build draft And I should see the supported E2E environment variable in Advanced Settings And the Agent v2 Build draft should no longer be active - @core @stable-model + @core @stable-model @skill-fixture Scenario: Applying a Build draft with an existing Skill keeps a single Skill entry Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available diff --git a/e2e/features/agent-v2/knowledge.feature b/e2e/features/agent-v2/knowledge.feature index 0d60cf6e7f406c..16c7c8daeb8968 100644 --- a/e2e/features/agent-v2/knowledge.feature +++ b/e2e/features/agent-v2/knowledge.feature @@ -1,4 +1,4 @@ -@agent-v2 @authenticated @knowledge @core +@agent-v2 @authenticated @knowledge @knowledge-fixture @core Feature: Agent v2 Knowledge Retrieval Scenario: Agent decide Knowledge Retrieval settings are saved and restored Given I am signed in as the default E2E admin diff --git a/e2e/features/agent-v2/preflight.feature b/e2e/features/agent-v2/preflight.feature index 0928eb9be85b8f..54b19d813dc180 100644 --- a/e2e/features/agent-v2/preflight.feature +++ b/e2e/features/agent-v2/preflight.feature @@ -19,10 +19,12 @@ Feature: Agent Builder preseeded environment Given I am signed in as the default E2E admin And the Agent Builder broken chat model is available + @tool-fixture Scenario: JSON Replace tool is available Given I am signed in as the default E2E admin And the Agent Builder preseeded tool "JSON Process / JSON Replace" is available + @tool-fixture Scenario: Tavily Search tool is available Given I am signed in as the default E2E admin And the Agent Builder preseeded tool "Tavily / Tavily Search" is available @@ -34,22 +36,27 @@ Feature: Agent Builder preseeded environment And the e2e-summary-skill Skill is available to the Agent v2 test agent Then the Agent v2 test agent should include drive skill "e2e-summary-skill" + @knowledge-fixture Scenario: Agent knowledge base is available Given I am signed in as the default E2E admin And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready + @knowledge-fixture Scenario: Indexing knowledge base is available Given I am signed in as the default E2E admin And the Agent Builder preseeded dataset "E2E Agent Knowledge Base Indexing" is indexing + @full-config-agent Scenario: Full config Agent is available Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" is available + @full-config-agent @skill-fixture Scenario: Full config Agent includes the summary Skill Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" includes drive skill "e2e-summary-skill" + @full-config-agent @stable-model @tool-fixture @skill-fixture @knowledge-fixture Scenario: Full config Agent includes core fixture configuration Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" includes the core fixture configuration @@ -62,46 +69,57 @@ Feature: Agent Builder preseeded environment And I expand Agent v2 Advanced Settings Then Agent v2 Content Moderation Settings should be available + @tool-states-agent Scenario: Tool states Agent is available Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" is available + @tool-states-agent @tool-fixture @skill-fixture Scenario: Tool states Agent includes tool state fixture configuration Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" includes the tool state fixture configuration + @file-tree-fixture Scenario: File tree Agent includes fixture files Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E Agent With File Tree" includes the file tree fixture files + @dual-retrieval-fixture Scenario: Dual retrieval Agent is available Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E Agent With Dual Retrieval" is available + @dual-retrieval-fixture @knowledge-fixture Scenario: Dual retrieval Agent includes dual retrieval fixture configuration Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E Agent With Dual Retrieval" includes the dual retrieval fixture configuration + @published-web-app Scenario: Published Web app Agent exposes Web app access Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E Agent Published Web App" has published Web app access + @backend-api-access Scenario: Backend API-enabled Agent is available Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E Agent Backend API Enabled" is available + @backend-api-access Scenario: Backend API-enabled Agent exposes API access with a key Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E Agent Backend API Enabled" has Backend service API access with an API key + @workflow-reference Scenario: Workflow reference Agent is available Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E Agent With Workflow Reference" is available + @workflow-reference Scenario: Reference workflow is available Given I am signed in as the default E2E admin And the Agent Builder preseeded workflow "E2E Agent Reference Workflow" is available + @workflow-reference Scenario: Workflow reference Agent is used by the reference workflow Given I am signed in as the default E2E admin And the Agent Builder preseeded Agent "E2E Agent With Workflow Reference" is referenced by workflow "E2E Agent Reference Workflow" diff --git a/e2e/features/agent-v2/publish.feature b/e2e/features/agent-v2/publish.feature index 6b92f52a916055..71f050c9d46a77 100644 --- a/e2e/features/agent-v2/publish.feature +++ b/e2e/features/agent-v2/publish.feature @@ -58,7 +58,7 @@ Feature: Agent v2 publish And the normal Agent v2 draft should use the normal E2E prompt And the Agent v2 publish action should be available for unpublished changes - @web-app-runtime @stable-model + @web-app-runtime @published-web-app @stable-model Scenario: Published Web app remains isolated from unpublished Agent v2 draft edits Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available @@ -75,7 +75,7 @@ Feature: Agent v2 publish Then the Agent v2 Web app response should include the normal E2E marker And the Agent v2 Web app response should not include the updated E2E marker - @web-app-runtime @stable-model + @web-app-runtime @published-web-app @stable-model Scenario: Published Web app uses the latest Agent v2 published configuration Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available diff --git a/e2e/features/agent-v2/tools.feature b/e2e/features/agent-v2/tools.feature index 4039ef90c76911..83006b09af1133 100644 --- a/e2e/features/agent-v2/tools.feature +++ b/e2e/features/agent-v2/tools.feature @@ -1,4 +1,4 @@ -@agent-v2 @authenticated @tools @core +@agent-v2 @authenticated @tools @core @tool-fixture Feature: Agent v2 tools Scenario: JSON Replace tool is saved after adding it from the Tools selector Given I am signed in as the default E2E admin From 51ea38d0420930a84d6299488aad3ea0738ecff8 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 03:32:12 +0800 Subject: [PATCH 152/185] test(e2e): narrow agent builder fixture tags --- e2e/features/agent-v2/access-point.feature | 8 ++++---- e2e/features/agent-v2/tools.feature | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/e2e/features/agent-v2/access-point.feature b/e2e/features/agent-v2/access-point.feature index 0d6349759b879f..b24da11cbc1972 100644 --- a/e2e/features/agent-v2/access-point.feature +++ b/e2e/features/agent-v2/access-point.feature @@ -8,7 +8,7 @@ Feature: Agent v2 Access Point And I switch to the Agent v2 Access Point section Then I should see the Agent v2 Access Point overview - @core @web-app-access @published-web-app + @core @web-app-access Scenario: Web app access URL can be copied without changing orchestration Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API @@ -36,7 +36,7 @@ Feature: Agent v2 Access Point Then the Agent v2 Web app should open in a new tab And the current Agent v2 orchestration draft should be unchanged - @core @web-app-access @published-web-app + @core @web-app-access Scenario: Web app Embedded configuration opens from Access Point Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API @@ -48,7 +48,7 @@ Feature: Agent v2 Access Point Then I should see the Agent v2 Embedded configuration dialog And the current Agent v2 orchestration draft should be unchanged - @core @web-app-access @published-web-app + @core @web-app-access Scenario: Web app customization opens from Access Point Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API @@ -60,7 +60,7 @@ Feature: Agent v2 Access Point Then I should see the Agent v2 Web app customization dialog And the current Agent v2 orchestration draft should be unchanged - @core @web-app-access @published-web-app + @core @web-app-access Scenario: Web app settings open from Access Point without changing orchestration Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API diff --git a/e2e/features/agent-v2/tools.feature b/e2e/features/agent-v2/tools.feature index 83006b09af1133..4eb67bf230c426 100644 --- a/e2e/features/agent-v2/tools.feature +++ b/e2e/features/agent-v2/tools.feature @@ -1,5 +1,6 @@ -@agent-v2 @authenticated @tools @core @tool-fixture +@agent-v2 @authenticated @tools @core Feature: Agent v2 tools + @tool-fixture Scenario: JSON Replace tool is saved after adding it from the Tools selector Given I am signed in as the default E2E admin And the Agent Builder preseeded tool "JSON Process / JSON Replace" is available From 2eec6e327b9b57227a54428206f079f03e389057 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 03:36:56 +0800 Subject: [PATCH 153/185] test(e2e): read agent config files from current contract --- e2e/features/agent-v2/support/preflight/agents.ts | 2 +- e2e/features/step-definitions/agent-v2/agent-edit.steps.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/features/agent-v2/support/preflight/agents.ts b/e2e/features/agent-v2/support/preflight/agents.ts index bc9a5cdf960f88..c4223d86e56cb8 100644 --- a/e2e/features/agent-v2/support/preflight/agents.ts +++ b/e2e/features/agent-v2/support/preflight/agents.ts @@ -210,7 +210,7 @@ export async function skipMissingPreseededFullConfigAgentCoreConfiguration( if (!prompt.includes(agentBuilderExpectedTokens.agentReply)) missing.push(`Prompt token ${agentBuilderExpectedTokens.agentReply}`) - const files = asArray(asRecord(soul.files).files) + const files = asArray(soul.config_files) for (const fileName of [ agentBuilderTestMaterials.smallFile, agentBuilderTestMaterials.specialFilename, diff --git a/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts b/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts index e3b7d3bba077d7..e9ccd1d6e2f114 100644 --- a/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts +++ b/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts @@ -28,7 +28,7 @@ const getComposerInheritanceSnapshot = async (agentId: string) => { const soul = draft.agent_soul ?? {} const model = asRecord(soul.model) const prompt = asRecord(soul.prompt) - const files = asArray(asRecord(soul.files).files) + const files = asArray(soul.config_files) const tools = asArray(asRecord(soul.tools).dify_tools) const knowledgeSets = asArray(asRecord(soul.knowledge).sets) From 6b1a638808ce5ef7eb9622d3f981c24bcaa5fac0 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 03:39:01 +0800 Subject: [PATCH 154/185] test(e2e): verify agent builder skill seed contracts --- e2e/features/agent-v2/support/preflight/agents.ts | 8 ++++++++ .../step-definitions/agent-v2/agent-edit.steps.ts | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/e2e/features/agent-v2/support/preflight/agents.ts b/e2e/features/agent-v2/support/preflight/agents.ts index c4223d86e56cb8..8fcb9519aa03c8 100644 --- a/e2e/features/agent-v2/support/preflight/agents.ts +++ b/e2e/features/agent-v2/support/preflight/agents.ts @@ -219,6 +219,10 @@ export async function skipMissingPreseededFullConfigAgentCoreConfiguration( missing.push(`file ${fileName}`) } + const skills = asArray(soul.config_skills) + if (!hasNamedOrKeyedEntry(skills, agentBuilderPreseededResources.summarySkill)) + missing.push(agentBuilderPreseededResources.summarySkill) + const [providerName = '', toolName = ''] = jsonTool.id.split('/') const parsedTool = splitToolDisplayName(agentBuilderPreseededResources.jsonReplaceTool) if ( @@ -289,6 +293,10 @@ export async function skipMissingPreseededToolStatesAgentConfiguration( const toolItems = asArray(asRecord(soul.tools).dify_tools) const missing: string[] = [] + const skills = asArray(soul.config_skills) + if (!hasNamedOrKeyedEntry(skills, agentBuilderPreseededResources.summarySkill)) + missing.push(agentBuilderPreseededResources.summarySkill) + const [jsonProviderName = '', jsonToolName = ''] = jsonTool.id.split('/') const parsedJsonTool = splitToolDisplayName(agentBuilderPreseededResources.jsonReplaceTool) if ( diff --git a/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts b/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts index e9ccd1d6e2f114..b2da0b61cb09fb 100644 --- a/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts +++ b/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts @@ -29,6 +29,7 @@ const getComposerInheritanceSnapshot = async (agentId: string) => { const model = asRecord(soul.model) const prompt = asRecord(soul.prompt) const files = asArray(soul.config_files) + const skills = asArray(soul.config_skills) const tools = asArray(asRecord(soul.tools).dify_tools) const knowledgeSets = asArray(asRecord(soul.knowledge).sets) @@ -44,6 +45,7 @@ const getComposerInheritanceSnapshot = async (agentId: string) => { provider: asString(model.model_provider), }, prompt: asString(prompt.system_prompt), + skillNames: skills.map(skill => asString(asRecord(skill).name)).filter(Boolean).sort(), toolSignatures: tools .map((tool) => { const record = asRecord(tool) @@ -181,6 +183,10 @@ Then( agentBuilderTestMaterials.smallFile, agentBuilderTestMaterials.specialFilename, ])) + expect(duplicatedSnapshot.skillNames).toEqual(expect.arrayContaining([ + agentBuilderPreseededResources.summarySkill, + ])) + expect(duplicatedSnapshot.skillNames).toEqual(sourceSnapshot.skillNames) expect(duplicatedSnapshot.toolSignatures).toEqual(sourceSnapshot.toolSignatures) expect(duplicatedSnapshot.knowledgeDatasetNames).toEqual(expect.arrayContaining([ agentBuilderPreseededResources.agentKnowledgeBase, From b1702c6f47cf282386d34ac80c4e493f86e1a0dd Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 03:43:57 +0800 Subject: [PATCH 155/185] test(e2e): remove unused preview prompt fixture --- e2e/features/agent-v2/support/agent-builder-resources.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/e2e/features/agent-v2/support/agent-builder-resources.ts b/e2e/features/agent-v2/support/agent-builder-resources.ts index 34097059d22a70..0b6a70853a3f13 100644 --- a/e2e/features/agent-v2/support/agent-builder-resources.ts +++ b/e2e/features/agent-v2/support/agent-builder-resources.ts @@ -33,7 +33,6 @@ export const agentBuilderFixedInputs = { moderationKeyword: 'E2E_BLOCKED_KEYWORD', inputModerationReply: 'E2E_INPUT_BLOCKED_REPLY', outputModerationReply: 'E2E_OUTPUT_BLOCKED_REPLY', - previewSuccessQuery: '请回复测试成功', backendApiUser: 'e2e-agent-access-point', } as const From 10da91735cd5a9338558c0ae806ad0af805c32d3 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 03:48:53 +0800 Subject: [PATCH 156/185] test(e2e): verify knowledge seed content token --- e2e/features/agent-v2/AGENTS.md | 2 +- .../agent-v2/support/preflight/datasets.ts | 92 +++++++++++++++++++ 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 284885b7bae56a..d7b2df3163bce8 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -169,7 +169,7 @@ Use `the Agent Builder broken chat model is available` before model-recovery sce Use `the Agent Builder preseeded Agent "{name}" is available`, `the Agent Builder preseeded workflow "{name}" is available`, `the Agent Builder preseeded dataset "{name}" is available`, and `the Agent Builder preseeded tool "{provider} / {tool}" is available` when a scenario depends on a fixed environment resource. These steps verify the resource through Console APIs, store the result in `DifyWorld.agentBuilder.preflight.preseededResources`, and return `skipped` when the resource is missing. -Use `the Agent Builder preseeded dataset "{name}" is indexed and ready` for knowledge retrieval scenarios that require a completed knowledge base. It verifies that the dataset exists, has documents, all listed documents are available, and every document indexing status is `completed`. +Use `the Agent Builder preseeded dataset "{name}" is indexed and ready` for knowledge retrieval scenarios that require a completed knowledge base. It verifies that the dataset exists, has documents, all listed documents are available, and every document indexing status is `completed`. For `E2E Agent Knowledge Base`, it also verifies through the Console segment list API that at least one enabled segment contains `AGENT_KNOWLEDGE_PASS`. Use `the Agent Builder preseeded dataset "{name}" is indexing` for failure-recovery scenarios that require an indexing or queued knowledge base. It verifies at least one document is in `waiting`, `parsing`, `cleaning`, `splitting`, or `indexing`. diff --git a/e2e/features/agent-v2/support/preflight/datasets.ts b/e2e/features/agent-v2/support/preflight/datasets.ts index a2677f21683ef0..b344ded7940878 100644 --- a/e2e/features/agent-v2/support/preflight/datasets.ts +++ b/e2e/features/agent-v2/support/preflight/datasets.ts @@ -1,10 +1,16 @@ import type { + ConsoleSegmentListResponse, DatasetListItemResponse, DocumentStatusListResponse, + DocumentWithSegmentsListResponse, } from '@dify/contracts/api/console/datasets/types.gen' import type { DifyWorld } from '../../../support/world' import type { PreseededResource } from './common' import { createApiContext, expectApiResponseOK } from '../../../../support/api' +import { + agentBuilderExpectedTokens, + agentBuilderPreseededResources, +} from '../agent-builder-resources' import { buildQuery, findConsoleResourceByName, @@ -53,6 +59,74 @@ const getDatasetIndexingStatuses = async (datasetId: string, resourceName: strin } } +const getDatasetDocuments = async (datasetId: string, resourceName: string) => { + const documents: DocumentWithSegmentsListResponse['data'] = [] + const ctx = await createApiContext() + try { + let page = 1 + let hasMore = true + + while (hasMore) { + const query = buildQuery({ limit: '100', page: String(page) }) + const response = await ctx.get(`/console/api/datasets/${datasetId}/documents?${query}`) + await expectApiResponseOK(response, `List preseeded dataset documents ${resourceName}`) + const body = (await response.json()) as DocumentWithSegmentsListResponse + + documents.push(...body.data) + hasMore = body.has_more + page += 1 + } + + return documents + } + finally { + await ctx.dispose() + } +} + +const datasetHasEnabledSegmentContainingToken = async ( + datasetId: string, + resourceName: string, + expectedToken: string, +) => { + const documents = await getDatasetDocuments(datasetId, resourceName) + const ctx = await createApiContext() + try { + for (const document of documents) { + const query = buildQuery({ + enabled: 'true', + keyword: expectedToken, + limit: '20', + page: '1', + }) + const response = await ctx.get( + `/console/api/datasets/${datasetId}/documents/${document.id}/segments?${query}`, + ) + await expectApiResponseOK( + response, + `Check preseeded dataset segment content ${resourceName}`, + ) + const body = (await response.json()) as ConsoleSegmentListResponse + const matchingSegment = body.data.find( + segment => + segment.enabled + && ( + segment.content.includes(expectedToken) + || segment.keywords?.some(keyword => keyword.includes(expectedToken)) + ), + ) + + if (matchingSegment) + return true + } + + return false + } + finally { + await ctx.dispose() + } +} + export const toDatasetResource = (resource: DatasetListItemResponse): PreseededResource => ({ id: resource.id, kind: 'dataset', @@ -109,6 +183,24 @@ export async function skipMissingReadyPreseededDataset( ) } + if (resourceName === agentBuilderPreseededResources.agentKnowledgeBase) { + const hasExpectedToken = await datasetHasEnabledSegmentContainingToken( + resource.id, + resourceName, + agentBuilderExpectedTokens.knowledgeReply, + ) + + if (!hasExpectedToken) { + return skipBlockedPrecondition( + world, + `Preseeded dataset "${resourceName}" has no enabled segment containing "${agentBuilderExpectedTokens.knowledgeReply}".`, + { + remediation: `Seed the dataset from the Agent Builder knowledge fixture and wait until an enabled segment contains "${agentBuilderExpectedTokens.knowledgeReply}".`, + }, + ) + } + } + return toDatasetResource(resource) } From bbedadc7d3f48c0e6ed73e4cefa5fee3b4700dc1 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 03:52:13 +0800 Subject: [PATCH 157/185] test(e2e): align stable model seed selector --- e2e/features/agent-v2/AGENTS.md | 4 ++-- e2e/features/agent-v2/support/preflight/models.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index d7b2df3163bce8..6556732acf914b 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -151,7 +151,7 @@ Agent Builder resource checks live under `features/agent-v2/support/preflight/`. Agent Builder preflight is read-only. It checks long-lived seed resources and records their IDs or normalized metadata for later steps, but it must not create missing resources, toggle fixed access settings, upload missing Skills/files, publish fixed Agents, or patch model/provider credentials. Seed creation and repair belong to the environment setup process, not to Cucumber scenarios. -Use `the Agent Builder stable chat model is available` before scenarios that need a real Agent Soul model configuration. This includes true runtime scenarios and build-mode workflows whose backend API requires model config, such as Agent workflow nodes. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` are selectors for a model already configured in the workspace; they are not provider credentials. The step defaults to `openai` / `gpt-5` / `llm`, verifies the selected model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilder.preflight.stableModel`. +Use `the Agent Builder stable chat model is available` before scenarios that need a real Agent Soul model configuration. This includes true runtime scenarios and build-mode workflows whose backend API requires model config, such as Agent workflow nodes. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` are selectors for a model already configured in the workspace; they are not provider credentials. The step defaults to `openai` / `gpt-5.4-mini` / `llm`, verifies the selected model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilder.preflight.stableModel`. Do not pass model provider API keys through Cucumber or Playwright env vars. Provider credentials belong to the Dify environment seed/admin setup. If the selected provider/model is missing or inactive, the scenario must be blocked by preflight instead of trying to create or patch provider credentials during the test. @@ -159,7 +159,7 @@ Override the default selector only when a scenario or environment explicitly nee ```bash E2E_STABLE_MODEL_PROVIDER=openai -E2E_STABLE_MODEL_NAME=gpt-5 +E2E_STABLE_MODEL_NAME=gpt-5.4-mini E2E_STABLE_MODEL_TYPE=llm ``` diff --git a/e2e/features/agent-v2/support/preflight/models.ts b/e2e/features/agent-v2/support/preflight/models.ts index 2171ffdf9d4c8e..330c61c9d482f6 100644 --- a/e2e/features/agent-v2/support/preflight/models.ts +++ b/e2e/features/agent-v2/support/preflight/models.ts @@ -12,7 +12,7 @@ const brokenChatModelNameEnv = 'E2E_BROKEN_MODEL_NAME' const brokenChatModelTypeEnv = 'E2E_BROKEN_MODEL_TYPE' const activeModelStatus = 'active' const defaultStableChatModelProvider = 'openai' -const defaultStableChatModelName = 'gpt-5' +const defaultStableChatModelName = 'gpt-5.4-mini' const defaultStableChatModelType = 'llm' const defaultBrokenChatModelName = agentBuilderPreseededResources.brokenModel From 1f339d61ea72c0a5314bb3f7c0523bd76b52458e Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 03:58:12 +0800 Subject: [PATCH 158/185] docs(e2e): clarify workflow node model preflight --- e2e/features/agent-v2/AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 6556732acf914b..7da079ac084fe9 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -151,7 +151,7 @@ Agent Builder resource checks live under `features/agent-v2/support/preflight/`. Agent Builder preflight is read-only. It checks long-lived seed resources and records their IDs or normalized metadata for later steps, but it must not create missing resources, toggle fixed access settings, upload missing Skills/files, publish fixed Agents, or patch model/provider credentials. Seed creation and repair belong to the environment setup process, not to Cucumber scenarios. -Use `the Agent Builder stable chat model is available` before scenarios that need a real Agent Soul model configuration. This includes true runtime scenarios and build-mode workflows whose backend API requires model config, such as Agent workflow nodes. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` are selectors for a model already configured in the workspace; they are not provider credentials. The step defaults to `openai` / `gpt-5.4-mini` / `llm`, verifies the selected model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilder.preflight.stableModel`. +Use `the Agent Builder stable chat model is available` before scenarios that need a real Agent Soul model configuration. This includes true runtime scenarios, model-backed build-mode assertions, and Workflow Agent v2 node setup because the backend rejects Agent nodes without model config. Do not add the model preflight to pure navigation or identity checks unless the setup API itself requires model config. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` are selectors for a model already configured in the workspace; they are not provider credentials. The step defaults to `openai` / `gpt-5.4-mini` / `llm`, verifies the selected model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilder.preflight.stableModel`. Do not pass model provider API keys through Cucumber or Playwright env vars. Provider credentials belong to the Dify environment seed/admin setup. If the selected provider/model is missing or inactive, the scenario must be blocked by preflight instead of trying to create or patch provider credentials during the test. From 662d71a622ee5753d692b3c5745a1fe71aedbe50 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 04:08:30 +0800 Subject: [PATCH 159/185] test(e2e): clean workflow apps before agents --- e2e/AGENTS.md | 2 +- e2e/features/support/hooks.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index e7a3ca5cf2ecae..b6b25e9262c26f 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -308,7 +308,7 @@ Keep package-level support limited to broadly reusable primitives such as API cl Use generated API contracts for Console/Web/Service API request, response, and payload shapes. Import the concrete type directly from `@dify/contracts/.../types.gen` when it exists, and do not hand-write duplicate response shapes or wrap generated types in local aliases just to preserve an older helper name. Keep local E2E types only for scenario state, fixture registries, helper input options, preflight resource state, and intentionally narrowed test view models that are not complete API responses. -Use typed cleanup fields on `DifyWorld` for resource types created by scenarios, and use `DifyWorld.registerCleanup(...)` when a scenario creates any resource type that is not covered by typed cleanup fields. Cleanup callbacks run after typed cleanup queues, even when the scenario fails. +Use typed cleanup fields on `DifyWorld` for resource types created by scenarios, and use `DifyWorld.registerCleanup(...)` when a scenario creates any resource type that is not covered by typed cleanup fields. Typed cleanup should remove child or referencing resources before their owners, such as Agent files before Agents and workflow apps before Agents they reference. Cleanup callbacks run after typed cleanup queues, even when the scenario fails. Scenario-owned setup may create disposable apps, Agents, files, credentials, drafts, or access toggles when the scenario owns their lifecycle and cleanup. Do not use scenario setup to silently fix or complete a shared preseeded resource; if a fixed resource is missing or drifted, report it as blocked and route it to the seed owner. diff --git a/e2e/features/support/hooks.ts b/e2e/features/support/hooks.ts index ac8b57766abf46..2103626e0f881f 100644 --- a/e2e/features/support/hooks.ts +++ b/e2e/features/support/hooks.ts @@ -108,8 +108,8 @@ After(async function (this: DifyWorld, { pickle, result }) { await deleteAgentConfigFile(file.agentId, file.name).catch(() => {}) for (const file of this.createdAgentDriveFiles.toReversed()) await deleteAgentDriveFile(file.agentId, file.key).catch(() => {}) - for (const id of this.createdAgentIds) await deleteTestAgent(id).catch(() => {}) for (const id of this.createdAppIds) await deleteTestApp(id).catch(() => {}) + for (const id of this.createdAgentIds) await deleteTestAgent(id).catch(() => {}) for (const id of this.createdDatasetIds) await deleteTestDataset(id).catch(() => {}) for (const credential of this.createdBuiltinToolCredentials.toReversed()) await deleteBuiltinToolCredential(credential.provider, credential.credentialId).catch(() => {}) From 892fb222d6c99a50b4271a014923499b8897b56b Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 04:12:02 +0800 Subject: [PATCH 160/185] test(e2e): report typed cleanup failures --- e2e/AGENTS.md | 2 +- e2e/features/support/hooks.ts | 53 +++++++++++++++++++++++++++-------- 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index b6b25e9262c26f..3eede7f263e50c 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -308,7 +308,7 @@ Keep package-level support limited to broadly reusable primitives such as API cl Use generated API contracts for Console/Web/Service API request, response, and payload shapes. Import the concrete type directly from `@dify/contracts/.../types.gen` when it exists, and do not hand-write duplicate response shapes or wrap generated types in local aliases just to preserve an older helper name. Keep local E2E types only for scenario state, fixture registries, helper input options, preflight resource state, and intentionally narrowed test view models that are not complete API responses. -Use typed cleanup fields on `DifyWorld` for resource types created by scenarios, and use `DifyWorld.registerCleanup(...)` when a scenario creates any resource type that is not covered by typed cleanup fields. Typed cleanup should remove child or referencing resources before their owners, such as Agent files before Agents and workflow apps before Agents they reference. Cleanup callbacks run after typed cleanup queues, even when the scenario fails. +Use typed cleanup fields on `DifyWorld` for resource types created by scenarios, and use `DifyWorld.registerCleanup(...)` when a scenario creates any resource type that is not covered by typed cleanup fields. Typed cleanup should remove child or referencing resources before their owners, such as Agent files before Agents and workflow apps before Agents they reference. Cleanup failures should be attached to the report instead of being swallowed silently. Cleanup callbacks run after typed cleanup queues, even when the scenario fails. Scenario-owned setup may create disposable apps, Agents, files, credentials, drafts, or access toggles when the scenario owns their lifecycle and cleanup. Do not use scenario setup to silently fix or complete a shared preseeded resource; if a fixed resource is missing or drifted, report it as blocked and route it to the seed owner. diff --git a/e2e/features/support/hooks.ts b/e2e/features/support/hooks.ts index 2103626e0f881f..8fd59934dae32e 100644 --- a/e2e/features/support/hooks.ts +++ b/e2e/features/support/hooks.ts @@ -46,6 +46,19 @@ const writeArtifact = async ( return artifactPath } +const recordCleanup = async ( + errors: string[], + label: string, + cleanup: () => Promise, +) => { + try { + await cleanup() + } + catch (error) { + errors.push(`${label}: ${error instanceof Error ? error.message : String(error)}`) + } +} + BeforeAll({ timeout: AUTH_BOOTSTRAP_TIMEOUT_MS }, async () => { await mkdir(artifactsDir, { recursive: true }) @@ -102,17 +115,35 @@ After(async function (this: DifyWorld, { pickle, result }) { `[e2e] end ${pickle.name} status=${status}${elapsedMs ? ` durationMs=${elapsedMs}` : ''}`, ) - for (const skill of this.createdAgentConfigSkills.toReversed()) - await deleteAgentConfigSkill(skill.agentId, skill.name).catch(() => {}) - for (const file of this.createdAgentConfigFiles.toReversed()) - await deleteAgentConfigFile(file.agentId, file.name).catch(() => {}) - for (const file of this.createdAgentDriveFiles.toReversed()) - await deleteAgentDriveFile(file.agentId, file.key).catch(() => {}) - for (const id of this.createdAppIds) await deleteTestApp(id).catch(() => {}) - for (const id of this.createdAgentIds) await deleteTestAgent(id).catch(() => {}) - for (const id of this.createdDatasetIds) await deleteTestDataset(id).catch(() => {}) - for (const credential of this.createdBuiltinToolCredentials.toReversed()) - await deleteBuiltinToolCredential(credential.provider, credential.credentialId).catch(() => {}) + const cleanupErrors: string[] = [] + + for (const skill of this.createdAgentConfigSkills.toReversed()) { + await recordCleanup(cleanupErrors, `Delete Agent config skill ${skill.name}`, () => + deleteAgentConfigSkill(skill.agentId, skill.name)) + } + for (const file of this.createdAgentConfigFiles.toReversed()) { + await recordCleanup(cleanupErrors, `Delete Agent config file ${file.name}`, () => + deleteAgentConfigFile(file.agentId, file.name)) + } + for (const file of this.createdAgentDriveFiles.toReversed()) { + await recordCleanup(cleanupErrors, `Delete Agent drive file ${file.key}`, () => + deleteAgentDriveFile(file.agentId, file.key)) + } + for (const id of this.createdAppIds) + await recordCleanup(cleanupErrors, `Delete app ${id}`, () => deleteTestApp(id)) + for (const id of this.createdAgentIds) + await recordCleanup(cleanupErrors, `Delete Agent ${id}`, () => deleteTestAgent(id)) + for (const id of this.createdDatasetIds) + await recordCleanup(cleanupErrors, `Delete dataset ${id}`, () => deleteTestDataset(id)) + for (const credential of this.createdBuiltinToolCredentials.toReversed()) { + await recordCleanup( + cleanupErrors, + `Delete builtin tool credential ${credential.provider}/${credential.credentialId}`, + () => deleteBuiltinToolCredential(credential.provider, credential.credentialId), + ) + } + if (cleanupErrors.length > 0) + this.attach(`Typed cleanup errors:\n${cleanupErrors.join('\n')}`, 'text/plain') await this.runRegisteredCleanups() await this.closeSession() From 9c0a89157aa19a45c3b428827345c93fd337ab64 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 04:19:55 +0800 Subject: [PATCH 161/185] test(e2e): clarify content moderation blocker --- .../step-definitions/agent-v2/content-moderation.steps.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/features/step-definitions/agent-v2/content-moderation.steps.ts b/e2e/features/step-definitions/agent-v2/content-moderation.steps.ts index 2987e030d85c0f..1a9604a04915bf 100644 --- a/e2e/features/step-definitions/agent-v2/content-moderation.steps.ts +++ b/e2e/features/step-definitions/agent-v2/content-moderation.steps.ts @@ -70,7 +70,7 @@ Then('Agent v2 Content Moderation Settings should be available', async function 'Agent v2 Content Moderation Settings is not available in this build.', { owner: 'product', - remediation: 'Enable ENABLE_AGENT_CONTENT_MODERATION or keep this scenario feature-gated.', + remediation: 'Enable the Agent v2 Content Moderation feature flag in the product or keep this scenario feature-gated.', }, ) } From 22dcd48c2e8f7a19def3ee86432978089f4b1ea1 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 04:22:50 +0800 Subject: [PATCH 162/185] docs(e2e): clarify preflight seed boundary --- e2e/AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 3eede7f263e50c..fb3cac6b36a2a9 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -302,7 +302,7 @@ Use `fixtures/test-materials/` for checked-in files that scenarios upload, previ Use scoped feature support for scenarios that require optional external resources such as a model provider, plugin/tool credential, knowledge base seed, or fixed app. Prefer an explicit `Given` step that returns a skipped result with a clear blocked-precondition reason over hidden setup in hooks. -Treat preflight checks as read-only readiness checks. A preflight step may query the environment, record typed state on `DifyWorld`, attach a blocked-precondition reason, or return `skipped`; it must not create, repair, publish, reconfigure, or mutate shared seed resources. Long-lived resources belong to the environment seed/setup process and need an explicit owner outside individual scenarios. +Treat preflight checks as read-only readiness checks. A preflight step may query the environment, record typed state on `DifyWorld`, attach a blocked-precondition reason, or return `skipped`; it must not create, repair, publish, reconfigure, or mutate shared seed resources. Treat the preflight suite as a readiness and drift report, not as a seed manager. Long-lived resources belong to the environment seed/setup process and need an explicit owner outside individual scenarios. Keep package-level support limited to broadly reusable primitives such as API clients, naming, fixture path resolution, and cleanup helpers. Feature-specific seed contracts and preflight checks belong under the owning feature's support folder. From ec68f272968a8cd36f7f7fc96e567fb79db1e6b2 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 04:26:55 +0800 Subject: [PATCH 163/185] test(e2e): decouple web app launch from model seed --- e2e/features/agent-v2/access-point.feature | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/e2e/features/agent-v2/access-point.feature b/e2e/features/agent-v2/access-point.feature index b24da11cbc1972..1c9d497cc12acc 100644 --- a/e2e/features/agent-v2/access-point.feature +++ b/e2e/features/agent-v2/access-point.feature @@ -21,11 +21,10 @@ Feature: Agent v2 Access Point Then the Agent v2 Web app access URL should show it was copied And the current Agent v2 orchestration draft should be unchanged - @core @web-app-access @published-web-app @stable-model + @core @web-app-access @published-web-app Scenario: Published Web app can be launched from Access Point Given I am signed in as the default E2E admin - And the Agent Builder stable chat model is available - And a runnable Agent v2 test agent has been created via API + And a basic configured Agent v2 test agent has been created via API And the Agent v2 draft has been published via API And Agent v2 Web app access has been enabled via API When I open the Agent v2 configure page from the Agent Roster From 5c1847ab494680c64d2bac1fdf7a8de9c143ae86 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 04:33:50 +0800 Subject: [PATCH 164/185] test(e2e): split web app disabled access coverage --- e2e/features/agent-v2/access-point.feature | 25 +++++++++++++------ .../agent-v2/access-point-web-app.steps.ts | 17 ++++++++++++- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/e2e/features/agent-v2/access-point.feature b/e2e/features/agent-v2/access-point.feature index 1c9d497cc12acc..e73f4c8ad07f2f 100644 --- a/e2e/features/agent-v2/access-point.feature +++ b/e2e/features/agent-v2/access-point.feature @@ -71,26 +71,35 @@ Feature: Agent v2 Access Point Then I should see the Agent v2 Web app settings dialog And the current Agent v2 orchestration draft should be unchanged - @core @web-app-access @published-web-app @stable-model - Scenario: Web app access can be disabled and restored + @core @web-app-access @published-web-app + Scenario: Web app access can be disabled and restored from Access Point Given I am signed in as the default E2E admin - And the Agent Builder stable chat model is available - And a runnable Agent v2 test agent has been created via API + And a basic configured Agent v2 test agent has been created via API And the Agent v2 draft has been published via API And Agent v2 Web app access has been enabled via API When I open the Agent v2 configure page from the Agent Roster And I switch to the Agent v2 Access Point section And I disable Agent v2 Web app access Then Agent v2 Web app access should be out of service - When I open the disabled Agent v2 Web app URL - Then the disabled Agent v2 Web app should show an unavailable state When I enable Agent v2 Web app access Then Agent v2 Web app access should be in service - When I open the restored Agent v2 Web app URL - Then the restored Agent v2 Web app should not show an unavailable state When I refresh the current page Then Agent v2 Web app access should be in service + @web-app-access @published-web-app @feature-gated + Scenario: Disabled Web app public URL shows an unavailable state + Given I am signed in as the default E2E admin + And Agent v2 disabled Web app public unavailable state is available + And a basic configured Agent v2 test agent has been created via API + And the Agent v2 draft has been published via API + And Agent v2 Web app access has been enabled via API + When I open the Agent v2 configure page from the Agent Roster + And I switch to the Agent v2 Access Point section + And I disable Agent v2 Web app access + Then Agent v2 Web app access should be out of service + When I open the disabled Agent v2 Web app URL + Then the disabled Agent v2 Web app should show an unavailable state + @core @workflow-reference Scenario: Workflow access shows the referencing workflow Given I am signed in as the default E2E admin diff --git a/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts b/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts index 9a8bf8c3d0dadb..a109a5fcec80d5 100644 --- a/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts @@ -1,9 +1,10 @@ import type { DifyWorld } from '../../support/world' -import { Then, When } from '@cucumber/cucumber' +import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { setAgentSiteAccessAndGetURL } from '../../agent-v2/support/access-point' import { getAgentComposerDraft } from '../../agent-v2/support/agent' import { agentBuilderExpectedTokens } from '../../agent-v2/support/agent-builder-resources' +import { skipBlockedPrecondition } from '../../agent-v2/support/preflight/common' import { getCurrentAgentId, getDialog, @@ -208,6 +209,20 @@ Then('Agent v2 Web app access should be out of service', async function (this: D await expect(webAppCard.getByRole('button', { name: 'Launch' })).toBeDisabled() }) +Given( + 'Agent v2 disabled Web app public unavailable state is available', + async function (this: DifyWorld) { + return skipBlockedPrecondition( + this, + 'Disabled Agent v2 Web app public URL does not expose a stable user-visible unavailable state; the current route redirects to Web app sign-in.', + { + owner: 'product', + remediation: 'Define and implement the disabled public Web app UX before enabling this scenario.', + }, + ) + }, +) + When('I open the disabled Agent v2 Web app URL', async function (this: DifyWorld) { const webAppURL = this.agentBuilder.accessPoint.webAppURL if (!webAppURL) From 94a408ddadb760340de6fd36a1a5488378030e51 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 04:39:23 +0800 Subject: [PATCH 165/185] docs(e2e): document build draft model boundary --- e2e/features/agent-v2/AGENTS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 7da079ac084fe9..a26c2ff48bc753 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -153,6 +153,8 @@ Agent Builder preflight is read-only. It checks long-lived seed resources and re Use `the Agent Builder stable chat model is available` before scenarios that need a real Agent Soul model configuration. This includes true runtime scenarios, model-backed build-mode assertions, and Workflow Agent v2 node setup because the backend rejects Agent nodes without model config. Do not add the model preflight to pure navigation or identity checks unless the setup API itself requires model config. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` are selectors for a model already configured in the workspace; they are not provider credentials. The step defaults to `openai` / `gpt-5.4-mini` / `llm`, verifies the selected model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilder.preflight.stableModel`. +Keep `@stable-model` on Build draft apply scenarios that click `Apply`. The current product path calls `/build-chat/finalize` before applying the draft, and the backend returns `model is required` when the Agent Soul has no model config. Discard-only and pending-draft isolation scenarios can stay model-free when they do not finalize the Build draft. + Do not pass model provider API keys through Cucumber or Playwright env vars. Provider credentials belong to the Dify environment seed/admin setup. If the selected provider/model is missing or inactive, the scenario must be blocked by preflight instead of trying to create or patch provider credentials during the test. Override the default selector only when a scenario or environment explicitly needs a different stable model: From 242449f4f14243f8a7f771388a3e81fcb0f40de0 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 04:45:17 +0800 Subject: [PATCH 166/185] test(e2e): improve build draft apply diagnostics --- .../agent-v2/build-draft.steps.ts | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts index 3bb4a215276840..3e8291df056d06 100644 --- a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts +++ b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts @@ -1,3 +1,4 @@ +import type { Response } from '@playwright/test' import type { DifyWorld } from '../../support/world' import { readFile } from 'node:fs/promises' import { Given, Then, When } from '@cucumber/cucumber' @@ -132,6 +133,22 @@ When('I generate an Agent v2 Build draft from the fixed instruction', async func await expect(page.getByRole('button', { name: 'Discard' })).toBeEnabled() }) +const expectPageResponseOK = async (response: Response, action: string) => { + if (response.ok()) + return + + let body = '' + try { + body = await response.text() + } + catch { + body = '' + } + + const trimmedBody = body.length > 1000 ? `${body.slice(0, 1000)}...` : body + throw new Error(`${action} failed with ${response.status()} ${response.statusText()} at ${response.url()}: ${trimmedBody}`) +} + When('I discard the Agent v2 Build draft', async function (this: DifyWorld) { await this.getPage().getByRole('button', { name: 'Discard' }).click() }) @@ -152,8 +169,8 @@ When('I apply the Agent v2 Build draft', async function (this: DifyWorld) { ), { timeout: 120_000 }) await applyButton.click() - expect((await finalizeResponsePromise).ok()).toBe(true) - expect((await applyResponsePromise).ok()).toBe(true) + await expectPageResponseOK(await finalizeResponsePromise, 'Finalize Agent v2 Build draft') + await expectPageResponseOK(await applyResponsePromise, 'Apply Agent v2 Build draft') await expect(page.getByText('Action succeeded')).toBeVisible() }) From 19a845fa517de924168302e46facb6fe471d7899 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 04:52:21 +0800 Subject: [PATCH 167/185] test(e2e): expose agent tool runtime gap --- e2e/features/agent-v2/tools.feature | 15 ++++++++++-- .../step-definitions/agent-v2/tools.steps.ts | 23 +++++++++++++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/e2e/features/agent-v2/tools.feature b/e2e/features/agent-v2/tools.feature index 4eb67bf230c426..6ab45dfd2430a3 100644 --- a/e2e/features/agent-v2/tools.feature +++ b/e2e/features/agent-v2/tools.feature @@ -1,6 +1,6 @@ -@agent-v2 @authenticated @tools @core +@agent-v2 @authenticated @tools Feature: Agent v2 tools - @tool-fixture + @core @tool-fixture Scenario: JSON Replace tool is saved after adding it from the Tools selector Given I am signed in as the default E2E admin And the Agent Builder preseeded tool "JSON Process / JSON Replace" is available @@ -12,6 +12,17 @@ Feature: Agent v2 tools When I refresh the current page Then I should see the Agent v2 JSON Replace tool in the Tools section + @service-api-runtime @stable-model @tool-fixture + Scenario: JSON Replace tool runtime returns the replacement marker + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And the Agent Builder preseeded tool "JSON Process / JSON Replace" is available + And Agent v2 JSON Replace runtime verification is available + And a runnable Agent v2 test agent has been created via API + When I open the Agent v2 configure page + Then Agent v2 JSON Replace runtime verification should be available + + @core Scenario: Tool selector shows an empty state for a missing tool search Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API diff --git a/e2e/features/step-definitions/agent-v2/tools.steps.ts b/e2e/features/step-definitions/agent-v2/tools.steps.ts index 4bbc0d876e44c8..889d4b2a3a1779 100644 --- a/e2e/features/step-definitions/agent-v2/tools.steps.ts +++ b/e2e/features/step-definitions/agent-v2/tools.steps.ts @@ -1,9 +1,9 @@ import type { DifyWorld } from '../../support/world' -import { Then, When } from '@cucumber/cucumber' +import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { getAgentComposerDraft } from '../../agent-v2/support/agent' import { agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../agent-v2/support/agent-builder-resources' -import { asArray, asRecord } from '../../agent-v2/support/preflight/common' +import { asArray, asRecord, skipBlockedPrecondition } from '../../agent-v2/support/preflight/common' import { hasToolEntry } from '../../agent-v2/support/preflight/tools' import { getPreseededToolContract } from '../../agent-v2/support/tools' import { expectProviderToolActionVisible, getCurrentAgentId } from './configure-helpers' @@ -29,6 +29,17 @@ const expectJsonReplaceToolDraft = async (world: DifyWorld) => { ).toBe(true) } +async function skipJsonReplaceRuntimeVerification(world: DifyWorld) { + return skipBlockedPrecondition( + world, + 'Agent v2 JSON Replace runtime verification is blocked: the suite needs the JSON Process / JSON Replace runtime parameter contract and a deterministic published-runtime prompt before asserting tool execution.', + { + owner: 'test/seed', + remediation: 'Seed the JSON Replace tool runtime contract, then verify execution through published Web app or Backend service API instead of Builder Preview.', + }, + ) +} + When( 'I add the Agent Builder JSON Replace tool from the Tools selector', async function (this: DifyWorld) { @@ -72,6 +83,14 @@ When('I clear the Agent v2 tool selector search', async function (this: DifyWorl await search.fill('') }) +Given('Agent v2 JSON Replace runtime verification is available', async function (this: DifyWorld) { + return skipJsonReplaceRuntimeVerification(this) +}) + +Then('Agent v2 JSON Replace runtime verification should be available', async function (this: DifyWorld) { + return skipJsonReplaceRuntimeVerification(this) +}) + Then( 'the Agent v2 JSON Replace tool should be saved in the Agent v2 draft', async function (this: DifyWorld) { From e19382190906f77ce9d9314ec2f52dcd3f6ee845 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 04:55:57 +0800 Subject: [PATCH 168/185] test(e2e): prioritize tool runtime blocker --- e2e/features/agent-v2/tools.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/features/agent-v2/tools.feature b/e2e/features/agent-v2/tools.feature index 6ab45dfd2430a3..305bf64fbc0781 100644 --- a/e2e/features/agent-v2/tools.feature +++ b/e2e/features/agent-v2/tools.feature @@ -15,9 +15,9 @@ Feature: Agent v2 tools @service-api-runtime @stable-model @tool-fixture Scenario: JSON Replace tool runtime returns the replacement marker Given I am signed in as the default E2E admin + And Agent v2 JSON Replace runtime verification is available And the Agent Builder stable chat model is available And the Agent Builder preseeded tool "JSON Process / JSON Replace" is available - And Agent v2 JSON Replace runtime verification is available And a runnable Agent v2 test agent has been created via API When I open the Agent v2 configure page Then Agent v2 JSON Replace runtime verification should be available From e00b7b3ba098f0c538e1f695ff35f12d57cf8705 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 04:57:33 +0800 Subject: [PATCH 169/185] docs(e2e): clarify blocked preflight ordering --- e2e/features/agent-v2/AGENTS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index a26c2ff48bc753..abcfafa60348a8 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -207,6 +207,8 @@ Blocked messages should be specific enough to route ownership: Blocked precondition: missing . Owner: seed/product. Remediation: . ``` +Order blocked steps by the real owner of the first unresolved condition. If a scenario is not automatable yet because the product behavior or test fixture contract is undefined, put that explicit availability step before model/tool/dataset preflights so the report is not masked by an unrelated missing seed. If the scenario is otherwise automatable and only depends on an external seed resource, run the matching resource preflight before creating scenario-owned state. + Use partial coverage only when current product behavior is intentionally narrower than the written requirement and the test still asserts a real user-visible behavior. Example: Files are currently flat in Agent config files, so the flat Files list can be asserted while tree display remains blocked until product support exists. File format, size, count, and in-progress upload limit cases are feature-gated until the product exposes stable Agent config file restrictions and user-visible recovery/error states. Do not convert `@files-limits` scenarios to passing tests by relying on default environment behavior; first align the product contract or seed configuration. From 3c58625575fca0ee061a6af59991e7a29b6b6f86 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 05:04:11 +0800 Subject: [PATCH 170/185] test(e2e): split backend api access toggle coverage --- e2e/features/agent-v2/access-point.feature | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/e2e/features/agent-v2/access-point.feature b/e2e/features/agent-v2/access-point.feature index e73f4c8ad07f2f..4714542d617b16 100644 --- a/e2e/features/agent-v2/access-point.feature +++ b/e2e/features/agent-v2/access-point.feature @@ -148,8 +148,22 @@ Feature: Agent v2 Access Point And I open the Agent v2 API Reference Then the Agent v2 API Reference should open in a new tab + @core @backend-api-access + Scenario: Backend service API access can be disabled and restored from Access Point + Given I am signed in as the default E2E admin + And an Agent v2 test agent has been created via API + And Agent v2 Backend service API access has been enabled via API + When I open the Agent v2 configure page from the Agent Roster + And I switch to the Agent v2 Access Point section + And I disable Agent v2 Backend service API access + Then Agent v2 Backend service API access should be out of service + When I enable Agent v2 Backend service API access + Then Agent v2 Backend service API access should be in service + When I refresh the current page + Then Agent v2 Backend service API access should be in service + @service-api-runtime @backend-api-access @stable-model - Scenario: Backend service API can be disabled and restored + Scenario: Disabled Backend service API rejects requests and restored access succeeds Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available And a runnable Agent v2 test agent has been created via API From 3630c42cb0d3cc84e88cb8622b914590d0c72b75 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 05:07:46 +0800 Subject: [PATCH 171/185] test(e2e): verify existing backend api keys stay masked --- e2e/features/agent-v2/access-point.feature | 2 +- .../agent-v2/access-point-service-api.steps.ts | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/e2e/features/agent-v2/access-point.feature b/e2e/features/agent-v2/access-point.feature index 4714542d617b16..89aad7ab128411 100644 --- a/e2e/features/agent-v2/access-point.feature +++ b/e2e/features/agent-v2/access-point.feature @@ -126,7 +126,7 @@ Feature: Agent v2 Access Point Scenario: Backend service API keys are managed without exposing existing secrets Given I am signed in as the default E2E admin And an Agent v2 test agent has been created via API - And Agent v2 Backend service API access has been enabled via API + And Agent v2 Backend service API access has been enabled with a key via API When I open the Agent v2 configure page from the Agent Roster And I switch to the Agent v2 Access Point section And I open Agent v2 API key management diff --git a/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts b/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts index 8482c6dda7a477..797dba9653c2a9 100644 --- a/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts @@ -69,13 +69,15 @@ When('I open Agent v2 API key management', async function (this: DifyWorld) { Then('Agent v2 API keys should not expose a secret by default', async function (this: DifyWorld) { const page = this.getPage() const dialog = page.getByRole('dialog', { name: /API Secret key/i }) + const existingSecret = this.agentBuilder.accessPoint.generatedApiKey await expect(dialog).toBeVisible() await expect(dialog.getByText('Secret Key', { exact: true })).toBeVisible() await expect(dialog.getByText('CREATED', { exact: true })).toBeVisible() await expect(dialog.getByText('LAST USED', { exact: true })).toBeVisible() - await expect(dialog.getByText('No data', { exact: true })).toBeVisible() await expect(dialog.getByRole('button', { name: 'Create new Secret key' })).toBeVisible() + if (existingSecret) + await expect(dialog.getByText(existingSecret, { exact: true })).not.toBeVisible() await expect(dialog.getByText(/^app-/)).not.toBeVisible() await expect(page.getByRole('dialog', { name: 'Internal Server Error' })).not.toBeVisible() }) @@ -143,7 +145,7 @@ Then( await expect(apiKeyDialog).toBeVisible() await expect(apiKeyDialog.getByText(fullSecret, { exact: true })).not.toBeVisible() await expect(apiKeyDialog.getByText(/^app-/)).not.toBeVisible() - await expect(apiKeyDialog.getByLabel('Copy')).toBeVisible() + await expect(apiKeyDialog.getByLabel('Copy').first()).toBeVisible() }, ) From ed7f857b4e6b07b450ed0ed16909e06c1d4374fc Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 05:10:45 +0800 Subject: [PATCH 172/185] docs(e2e): clarify ui availability blockers --- e2e/features/agent-v2/AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index abcfafa60348a8..2fae704d9d4f7f 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -207,7 +207,7 @@ Blocked messages should be specific enough to route ownership: Blocked precondition: missing . Owner: seed/product. Remediation: . ``` -Order blocked steps by the real owner of the first unresolved condition. If a scenario is not automatable yet because the product behavior or test fixture contract is undefined, put that explicit availability step before model/tool/dataset preflights so the report is not masked by an unrelated missing seed. If the scenario is otherwise automatable and only depends on an external seed resource, run the matching resource preflight before creating scenario-owned state. +Order blocked steps by the real owner of the first unresolved condition. If a scenario is not automatable yet because the product behavior or test fixture contract is undefined, put that explicit availability step before model/tool/dataset preflights so the report is not masked by an unrelated missing seed. If the scenario is otherwise automatable and only depends on an external seed resource, run the matching resource preflight before creating scenario-owned state. When an availability check must inspect a real Configure UI surface, create only scenario-owned disposable state first and rely on the shared cleanup path after the skipped result. Use partial coverage only when current product behavior is intentionally narrower than the written requirement and the test still asserts a real user-visible behavior. Example: Files are currently flat in Agent config files, so the flat Files list can be asserted while tree display remains blocked until product support exists. From a43da8e43e9d907d81ad6f512d4f5b15d9ad6e73 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 05:13:56 +0800 Subject: [PATCH 173/185] test(e2e): move access point publish setup --- .../agent-v2/access-point-service-api.steps.ts | 5 ----- .../step-definitions/agent-v2/access-point.steps.ts | 8 ++++++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts b/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts index 797dba9653c2a9..84c11776bd16d6 100644 --- a/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts @@ -6,7 +6,6 @@ import { sendAgentServiceApiChatMessage, setAgentApiAccess, } from '../../agent-v2/support/access-point' -import { publishAgent } from '../../agent-v2/support/agent' import { agentBuilderExpectedTokens } from '../../agent-v2/support/agent-builder-resources' import { getCurrentAgentId, getServiceApiCard } from './access-point-helpers' @@ -19,10 +18,6 @@ Given( }, ) -Given('the Agent v2 draft has been published via API', async function (this: DifyWorld) { - await publishAgent(getCurrentAgentId(this)) -}) - Given( 'Agent v2 Backend service API access has been enabled with a key via API', async function (this: DifyWorld) { diff --git a/e2e/features/step-definitions/agent-v2/access-point.steps.ts b/e2e/features/step-definitions/agent-v2/access-point.steps.ts index c563379933f60c..00f99593719f7b 100644 --- a/e2e/features/step-definitions/agent-v2/access-point.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point.steps.ts @@ -1,13 +1,17 @@ import type { DifyWorld } from '../../support/world' -import { Then, When } from '@cucumber/cucumber' +import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' -import { getAgentAccessPath } from '../../agent-v2/support/agent' +import { getAgentAccessPath, publishAgent } from '../../agent-v2/support/agent' import { getAccessRegion, getCurrentAgentId, getPreseededResource, } from './access-point-helpers' +Given('the Agent v2 draft has been published via API', async function (this: DifyWorld) { + await publishAgent(getCurrentAgentId(this)) +}) + When('I open the Agent v2 Access Point page', async function (this: DifyWorld) { await this.getPage().goto(getAgentAccessPath(getCurrentAgentId(this))) }) From f9e24d0308e737948f1b68fbabdb6798c8d11dee Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 05:16:30 +0800 Subject: [PATCH 174/185] test(e2e): mark web app api setup as given --- .../step-definitions/agent-v2/access-point-web-app.steps.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts b/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts index a109a5fcec80d5..f06ef7216d4c47 100644 --- a/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts @@ -11,7 +11,7 @@ import { getWebAppCard, } from './access-point-helpers' -When( +Given( 'Agent v2 Web app access has been enabled via API', async function (this: DifyWorld) { this.agentBuilder.accessPoint.webAppURL = await setAgentSiteAccessAndGetURL( From cbddfac4614c2bf0b1948b534b23884477c07e1c Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 05:18:25 +0800 Subject: [PATCH 175/185] test(e2e): mark publish web app setup as given --- e2e/features/agent-v2/publish.feature | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e/features/agent-v2/publish.feature b/e2e/features/agent-v2/publish.feature index 71f050c9d46a77..f6b030fcf27bb6 100644 --- a/e2e/features/agent-v2/publish.feature +++ b/e2e/features/agent-v2/publish.feature @@ -66,8 +66,8 @@ Feature: Agent v2 publish When I open the Agent v2 configure page And I publish the Agent v2 draft Then the Agent v2 draft should be published and up to date - When Agent v2 Web app access has been enabled via API - And I fill the Agent v2 prompt editor with the updated E2E prompt + Given Agent v2 Web app access has been enabled via API + When I fill the Agent v2 prompt editor with the updated E2E prompt Then the Agent v2 configuration should be saved automatically And the normal Agent v2 draft should use the updated E2E prompt When I open the Agent v2 Web app URL @@ -88,7 +88,7 @@ Feature: Agent v2 publish And the normal Agent v2 draft should use the updated E2E prompt When I publish the Agent v2 draft Then the Agent v2 draft should be published and up to date - When Agent v2 Web app access has been enabled via API - And I open the Agent v2 Web app URL + Given Agent v2 Web app access has been enabled via API + When I open the Agent v2 Web app URL And I send an E2E message in the Agent v2 Web app Then the Agent v2 Web app response should include the updated E2E marker From 73f9268cc69ca0c76b093d70e0f8944c0dae3e71 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 05:23:10 +0800 Subject: [PATCH 176/185] test(e2e): group publish web app setup --- e2e/features/agent-v2/publish.feature | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/features/agent-v2/publish.feature b/e2e/features/agent-v2/publish.feature index f6b030fcf27bb6..0882a9359182c0 100644 --- a/e2e/features/agent-v2/publish.feature +++ b/e2e/features/agent-v2/publish.feature @@ -63,10 +63,10 @@ Feature: Agent v2 publish Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available And a runnable Agent v2 test agent has been created via API + And Agent v2 Web app access has been enabled via API When I open the Agent v2 configure page And I publish the Agent v2 draft Then the Agent v2 draft should be published and up to date - Given Agent v2 Web app access has been enabled via API When I fill the Agent v2 prompt editor with the updated E2E prompt Then the Agent v2 configuration should be saved automatically And the normal Agent v2 draft should use the updated E2E prompt @@ -80,6 +80,7 @@ Feature: Agent v2 publish Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available And a runnable Agent v2 test agent has been created via API + And Agent v2 Web app access has been enabled via API When I open the Agent v2 configure page And I publish the Agent v2 draft Then the Agent v2 draft should be published and up to date @@ -88,7 +89,6 @@ Feature: Agent v2 publish And the normal Agent v2 draft should use the updated E2E prompt When I publish the Agent v2 draft Then the Agent v2 draft should be published and up to date - Given Agent v2 Web app access has been enabled via API When I open the Agent v2 Web app URL And I send an E2E message in the Agent v2 Web app Then the Agent v2 Web app response should include the updated E2E marker From bb322e13e6a718895b71046eedbd5b1b4905effa Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 05:29:03 +0800 Subject: [PATCH 177/185] test(e2e): assert build draft skill isolation --- e2e/features/agent-v2/build-draft.feature | 1 + .../agent-v2/build-draft.steps.ts | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/e2e/features/agent-v2/build-draft.feature b/e2e/features/agent-v2/build-draft.feature index 313cff30a8f0da..c36f9245280d03 100644 --- a/e2e/features/agent-v2/build-draft.feature +++ b/e2e/features/agent-v2/build-draft.feature @@ -12,6 +12,7 @@ Feature: Agent v2 build draft Then I should see the Agent v2 Build draft pending changes And I should see the Agent v2 Build mode confirmation state And the normal Agent v2 draft should still use the normal E2E prompt + And the normal Agent v2 draft should not include the e2e-summary-skill Skill And the normal Agent v2 draft should not include the Agent Builder JSON Replace tool @core diff --git a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts index 3e8291df056d06..163932deb10743 100644 --- a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts +++ b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts @@ -259,6 +259,22 @@ Then('I should see one e2e-summary-skill Skill in the Skills section', async fun })).toHaveCount(1) }) +Then( + 'the normal Agent v2 draft should not include the e2e-summary-skill Skill', + async function (this: DifyWorld) { + await expect.poll( + async () => { + const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul + + return agentSoul?.config_skills?.some( + skill => skill.name === agentBuilderPreseededResources.summarySkill, + ) ?? false + }, + { timeout: 30_000 }, + ).toBe(false) + }, +) + Then( 'the normal Agent v2 draft should not include the Agent Builder JSON Replace tool', async function (this: DifyWorld) { From 571d9e3a13e090582a376fcff398303cca9ab918 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 05:35:33 +0800 Subject: [PATCH 178/185] test(e2e): cover knowledge retrieval runtime --- e2e/features/agent-v2/knowledge.feature | 37 +++++++++++++- .../access-point-service-api.steps.ts | 50 ++++++++++++++++--- 2 files changed, 80 insertions(+), 7 deletions(-) diff --git a/e2e/features/agent-v2/knowledge.feature b/e2e/features/agent-v2/knowledge.feature index 16c7c8daeb8968..37f787e31f85ea 100644 --- a/e2e/features/agent-v2/knowledge.feature +++ b/e2e/features/agent-v2/knowledge.feature @@ -1,5 +1,6 @@ -@agent-v2 @authenticated @knowledge @knowledge-fixture @core +@agent-v2 @authenticated @knowledge @knowledge-fixture Feature: Agent v2 Knowledge Retrieval + @core Scenario: Agent decide Knowledge Retrieval settings are saved and restored Given I am signed in as the default E2E admin And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready @@ -11,6 +12,7 @@ Feature: Agent v2 Knowledge Retrieval When I refresh the current page Then I should see the Agent v2 Agent decide Knowledge Retrieval settings + @core Scenario: Custom query Knowledge Retrieval settings are saved and restored Given I am signed in as the default E2E admin And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready @@ -22,6 +24,39 @@ Feature: Agent v2 Knowledge Retrieval When I refresh the current page Then I should see the Agent v2 Custom query Knowledge Retrieval settings + @service-api-runtime @stable-model @backend-api-access + Scenario: Agent decide Knowledge Retrieval answers through Backend service API + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready + And a runnable Agent v2 test agent has been created via API + When I open the Agent v2 configure page + And I add the Agent Builder knowledge base as an Agent decide Knowledge Retrieval + Then the Agent v2 Agent decide Knowledge Retrieval should be saved in the Agent v2 draft + And the Agent v2 configuration should be saved automatically + When I publish the Agent v2 draft + Then the Agent v2 draft should be published and up to date + When I enable Agent v2 Backend service API access with a key via API + And I send the Agent v2 Backend service API knowledge request + Then the Agent v2 Backend service API response should include the knowledge E2E marker + + @service-api-runtime @stable-model @backend-api-access + Scenario: Custom query Knowledge Retrieval answers through Backend service API + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready + And a runnable Agent v2 test agent has been created via API + When I open the Agent v2 configure page + And I add the Agent Builder knowledge base as a Custom query Knowledge Retrieval + Then the Agent v2 Custom query Knowledge Retrieval should be saved in the Agent v2 draft + And the Agent v2 configuration should be saved automatically + When I publish the Agent v2 draft + Then the Agent v2 draft should be published and up to date + When I enable Agent v2 Backend service API access with a key via API + And I send the Agent v2 Backend service API knowledge request + Then the Agent v2 Backend service API response should include the knowledge E2E marker + + @core Scenario: Removing Knowledge Retrieval clears the saved dataset reference Given I am signed in as the default E2E admin And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready diff --git a/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts b/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts index 84c11776bd16d6..e632f6ab74001c 100644 --- a/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts @@ -6,9 +6,18 @@ import { sendAgentServiceApiChatMessage, setAgentApiAccess, } from '../../agent-v2/support/access-point' -import { agentBuilderExpectedTokens } from '../../agent-v2/support/agent-builder-resources' +import { agentBuilderExpectedTokens, agentBuilderFixedInputs } from '../../agent-v2/support/agent-builder-resources' import { getCurrentAgentId, getServiceApiCard } from './access-point-helpers' +async function enableAgentApiAccessWithKey(world: DifyWorld) { + const agentId = getCurrentAgentId(world) + const apiAccess = await setAgentApiAccess(agentId, true) + const apiKey = await createAgentApiKey(agentId) + + world.agentBuilder.accessPoint.serviceApiBaseURL = apiAccess.service_api_base_url + world.agentBuilder.accessPoint.generatedApiKey = apiKey.token +} + Given( 'Agent v2 Backend service API access has been enabled via API', async function (this: DifyWorld) { @@ -21,12 +30,14 @@ Given( Given( 'Agent v2 Backend service API access has been enabled with a key via API', async function (this: DifyWorld) { - const agentId = getCurrentAgentId(this) - const apiAccess = await setAgentApiAccess(agentId, true) - const apiKey = await createAgentApiKey(agentId) + await enableAgentApiAccessWithKey(this) + }, +) - this.agentBuilder.accessPoint.serviceApiBaseURL = apiAccess.service_api_base_url - this.agentBuilder.accessPoint.generatedApiKey = apiKey.token +When( + 'I enable Agent v2 Backend service API access with a key via API', + async function (this: DifyWorld) { + await enableAgentApiAccessWithKey(this) }, ) @@ -211,6 +222,21 @@ When('I send the Agent v2 Backend service API minimal request', async function ( }) }) +When('I send the Agent v2 Backend service API knowledge request', async function (this: DifyWorld) { + const serviceApiBaseURL = this.agentBuilder.accessPoint.serviceApiBaseURL + const apiKey = this.agentBuilder.accessPoint.generatedApiKey + if (!serviceApiBaseURL) + throw new Error('No Agent v2 service API endpoint found. Enable Backend service API first.') + if (!apiKey) + throw new Error('No Agent v2 API key found. Create a Backend service API key first.') + + this.agentBuilder.accessPoint.serviceApiResponse = await sendAgentServiceApiChatMessage({ + apiKey, + query: agentBuilderFixedInputs.customKnowledgeQuery, + serviceApiBaseURL, + }) +}) + Then( 'the Agent v2 Backend service API request should be rejected while disabled', async function (this: DifyWorld) { @@ -224,6 +250,18 @@ Then( }, ) +Then( + 'the Agent v2 Backend service API response should include the knowledge E2E marker', + async function (this: DifyWorld) { + const response = this.agentBuilder.accessPoint.serviceApiResponse + if (!response) + throw new Error('No Agent v2 Backend service API response was recorded.') + + expect(response.ok).toBe(true) + expect(JSON.stringify(response.body)).toContain(agentBuilderExpectedTokens.knowledgeReply) + }, +) + Then( 'the Agent v2 Backend service API request should succeed with the normal E2E marker', async function (this: DifyWorld) { From 4ad62556ba142a58756409bbf8f807f2378992ac Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 05:37:45 +0800 Subject: [PATCH 179/185] test(e2e): keep knowledge api setup as precondition --- e2e/features/agent-v2/knowledge.feature | 8 ++++---- .../agent-v2/access-point-service-api.steps.ts | 7 ------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/e2e/features/agent-v2/knowledge.feature b/e2e/features/agent-v2/knowledge.feature index 37f787e31f85ea..8e111418b9cb30 100644 --- a/e2e/features/agent-v2/knowledge.feature +++ b/e2e/features/agent-v2/knowledge.feature @@ -30,14 +30,14 @@ Feature: Agent v2 Knowledge Retrieval And the Agent Builder stable chat model is available And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready And a runnable Agent v2 test agent has been created via API + And Agent v2 Backend service API access has been enabled with a key via API When I open the Agent v2 configure page And I add the Agent Builder knowledge base as an Agent decide Knowledge Retrieval Then the Agent v2 Agent decide Knowledge Retrieval should be saved in the Agent v2 draft And the Agent v2 configuration should be saved automatically When I publish the Agent v2 draft Then the Agent v2 draft should be published and up to date - When I enable Agent v2 Backend service API access with a key via API - And I send the Agent v2 Backend service API knowledge request + When I send the Agent v2 Backend service API knowledge request Then the Agent v2 Backend service API response should include the knowledge E2E marker @service-api-runtime @stable-model @backend-api-access @@ -46,14 +46,14 @@ Feature: Agent v2 Knowledge Retrieval And the Agent Builder stable chat model is available And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready And a runnable Agent v2 test agent has been created via API + And Agent v2 Backend service API access has been enabled with a key via API When I open the Agent v2 configure page And I add the Agent Builder knowledge base as a Custom query Knowledge Retrieval Then the Agent v2 Custom query Knowledge Retrieval should be saved in the Agent v2 draft And the Agent v2 configuration should be saved automatically When I publish the Agent v2 draft Then the Agent v2 draft should be published and up to date - When I enable Agent v2 Backend service API access with a key via API - And I send the Agent v2 Backend service API knowledge request + When I send the Agent v2 Backend service API knowledge request Then the Agent v2 Backend service API response should include the knowledge E2E marker @core diff --git a/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts b/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts index e632f6ab74001c..0ea9e23a25122b 100644 --- a/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point-service-api.steps.ts @@ -34,13 +34,6 @@ Given( }, ) -When( - 'I enable Agent v2 Backend service API access with a key via API', - async function (this: DifyWorld) { - await enableAgentApiAccessWithKey(this) - }, -) - Then('I should see the Agent v2 Backend service API endpoint', async function (this: DifyWorld) { const serviceApiCard = getServiceApiCard(this) From 0a40834374686a76336d7307aaeee0ef57061143 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 05:41:39 +0800 Subject: [PATCH 180/185] test(e2e): clarify configure leave scenario --- e2e/features/agent-v2/configure-persistence.feature | 4 ++-- e2e/features/step-definitions/agent-v2/configure.steps.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e/features/agent-v2/configure-persistence.feature b/e2e/features/agent-v2/configure-persistence.feature index 0c6abede24779d..6b2f97ea5e1b3f 100644 --- a/e2e/features/agent-v2/configure-persistence.feature +++ b/e2e/features/agent-v2/configure-persistence.feature @@ -32,12 +32,12 @@ Feature: Agent v2 configure persistence And the Agent v2 draft should use the stable E2E model @configure-persistence - Scenario: Leaving Configure before autosave completes preserves prompt changes + Scenario: Leaving Configure immediately after editing preserves prompt changes Given I am signed in as the default E2E admin And a basic configured Agent v2 test agent has been created via API When I open the Agent v2 configure page And I fill the Agent v2 prompt editor with the updated E2E prompt - And I leave the Agent v2 configure page before autosave completes + And I leave the Agent v2 configure page immediately after editing When I open the Agent v2 configure page from the Agent Roster Then I should see the updated E2E prompt in the Agent v2 prompt editor And the normal Agent v2 draft should use the updated E2E prompt diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index 63016ed5f73967..7d2af762d5d494 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -152,7 +152,7 @@ When('I switch to the Agent v2 Configure section', async function (this: DifyWor await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) }) -When('I leave the Agent v2 configure page before autosave completes', async function (this: DifyWorld) { +When('I leave the Agent v2 configure page immediately after editing', async function (this: DifyWorld) { const page = this.getPage() await page.goto('/roster') From 2a873c7ac1a750a8d9c0b8c0e5aa2c36c4e4e9f6 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 05:44:50 +0800 Subject: [PATCH 181/185] test(e2e): split backend api runtime success --- e2e/features/agent-v2/access-point.feature | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/e2e/features/agent-v2/access-point.feature b/e2e/features/agent-v2/access-point.feature index 89aad7ab128411..877776dc0cf439 100644 --- a/e2e/features/agent-v2/access-point.feature +++ b/e2e/features/agent-v2/access-point.feature @@ -162,6 +162,18 @@ Feature: Agent v2 Access Point When I refresh the current page Then Agent v2 Backend service API access should be in service + @service-api-runtime @backend-api-access @stable-model + Scenario: Published Agent v2 answers through Backend service API + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API + And Agent v2 Backend service API access has been enabled with a key via API + When I open the Agent v2 configure page + And I publish the Agent v2 draft + Then the Agent v2 draft should be published and up to date + When I send the Agent v2 Backend service API minimal request + Then the Agent v2 Backend service API request should succeed with the normal E2E marker + @service-api-runtime @backend-api-access @stable-model Scenario: Disabled Backend service API rejects requests and restored access succeeds Given I am signed in as the default E2E admin From fa8c70b2c7b995ce4afc71094485e135ad25b714 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 05:47:19 +0800 Subject: [PATCH 182/185] docs(e2e): clarify agent builder seed boundary --- e2e/features/agent-v2/AGENTS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 2fae704d9d4f7f..13d32647c11c3d 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -151,6 +151,8 @@ Agent Builder resource checks live under `features/agent-v2/support/preflight/`. Agent Builder preflight is read-only. It checks long-lived seed resources and records their IDs or normalized metadata for later steps, but it must not create missing resources, toggle fixed access settings, upload missing Skills/files, publish fixed Agents, or patch model/provider credentials. Seed creation and repair belong to the environment setup process, not to Cucumber scenarios. +Treat preseeded Agent Builder resources as environment contracts. Preflight can report that a stable model, dataset, Skill, Tool credential, fixed Agent, published Web app, or workflow reference is missing, inactive, not indexed, or drifted, but it must not repair that drift during a scenario. Seed scripts, CI bootstrap, or the documented environment maintenance flow own creating and keeping those resources valid. + Use `the Agent Builder stable chat model is available` before scenarios that need a real Agent Soul model configuration. This includes true runtime scenarios, model-backed build-mode assertions, and Workflow Agent v2 node setup because the backend rejects Agent nodes without model config. Do not add the model preflight to pure navigation or identity checks unless the setup API itself requires model config. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` are selectors for a model already configured in the workspace; they are not provider credentials. The step defaults to `openai` / `gpt-5.4-mini` / `llm`, verifies the selected model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilder.preflight.stableModel`. Keep `@stable-model` on Build draft apply scenarios that click `Apply`. The current product path calls `/build-chat/finalize` before applying the draft, and the backend returns `model is required` when the Agent Soul has no model config. Discard-only and pending-draft isolation scenarios can stay model-free when they do not finalize the Build draft. From 5a832c5cc5ac9e6bead0f0ec37aa55a9d7e42760 Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 05:52:01 +0800 Subject: [PATCH 183/185] test(e2e): keep preview scenarios opt-in --- e2e/AGENTS.md | 6 +++--- e2e/cucumber.config.ts | 6 +++++- e2e/scripts/run-cucumber.ts | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index fb3cac6b36a2a9..d80f6755bad5fc 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -74,8 +74,8 @@ flowchart TD C --> D["Cucumber loads config, steps, and support modules"] D --> E["BeforeAll bootstraps shared auth state via /install"] E --> F{"Which command is running?"} - F -->|`pnpm -C e2e e2e`| G["Run config default tags: not @fresh and not @skip"] - F -->|`pnpm -C e2e e2e:full*`| H["Override tags to not @skip"] + F -->|`pnpm -C e2e e2e`| G["Run config default tags: not @fresh and not @skip and not @preview"] + F -->|`pnpm -C e2e e2e:full*`| H["Override tags to not @skip and not @preview"] G --> I["Per-scenario BrowserContext from shared browser"] H --> I I --> J["Failure artifacts written to cucumber-report/artifacts"] @@ -105,7 +105,7 @@ Behavior depends on instance state: - uninitialized instance: completes install and stores authenticated state - initialized instance: signs in and reuses authenticated state -Because of that, the `@fresh` install scenario only runs in the `pnpm -C e2e e2e:full*` flows. The default `pnpm -C e2e e2e*` flows exclude `@fresh` via Cucumber config tags so they can be re-run against an already initialized instance. +Because of that, the `@fresh` install scenario only runs in the `pnpm -C e2e e2e:full*` flows. The default `pnpm -C e2e e2e*` flows exclude `@fresh` and `@preview` via Cucumber config tags so they can be re-run against an already initialized instance while keeping Builder Preview scenarios opt-in. Reset all persisted E2E state: diff --git a/e2e/cucumber.config.ts b/e2e/cucumber.config.ts index 4f768cc9d26f3e..3e6bd722556ea4 100644 --- a/e2e/cucumber.config.ts +++ b/e2e/cucumber.config.ts @@ -1,5 +1,9 @@ import type { IConfiguration } from '@cucumber/cucumber' +const hasCliTags = process.argv.some(arg => arg === '--tags' || arg.startsWith('--tags=')) +const defaultTags = process.env.E2E_CUCUMBER_TAGS + || (hasCliTags ? undefined : 'not @fresh and not @skip and not @preview') + const config = { format: [ 'progress-bar', @@ -10,7 +14,7 @@ const config = { import: ['./tsx-register.js', 'features/**/*.ts'], parallel: 1, paths: ['features/**/*.feature'], - tags: process.env.E2E_CUCUMBER_TAGS || 'not @fresh and not @skip', + ...(defaultTags ? { tags: defaultTags } : {}), timeout: 60_000, } satisfies Partial & { timeout: number diff --git a/e2e/scripts/run-cucumber.ts b/e2e/scripts/run-cucumber.ts index 887e01580903ba..5913450f14b3ab 100644 --- a/e2e/scripts/run-cucumber.ts +++ b/e2e/scripts/run-cucumber.ts @@ -179,7 +179,7 @@ const main = async () => { } if (startMiddlewareForRun && !hasCustomTags(forwardArgs)) - cucumberEnv.E2E_CUCUMBER_TAGS = 'not @skip' + cucumberEnv.E2E_CUCUMBER_TAGS = 'not @skip and not @preview' const result = await runCommand({ command: 'npx', From 8ad902227a85a32697359d56426f3fb3402e182f Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 05:57:56 +0800 Subject: [PATCH 184/185] test(e2e): cover published web app response --- e2e/features/agent-v2/publish.feature | 13 +++++++++++++ .../agent-v2/access-point-web-app.steps.ts | 14 ++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/e2e/features/agent-v2/publish.feature b/e2e/features/agent-v2/publish.feature index 0882a9359182c0..eb393ee96b5589 100644 --- a/e2e/features/agent-v2/publish.feature +++ b/e2e/features/agent-v2/publish.feature @@ -58,6 +58,19 @@ Feature: Agent v2 publish And the normal Agent v2 draft should use the normal E2E prompt And the Agent v2 publish action should be available for unpublished changes + @web-app-runtime @published-web-app @stable-model + Scenario: Published Agent v2 answers through Web app + Given I am signed in as the default E2E admin + And the Agent Builder stable chat model is available + And a runnable Agent v2 test agent has been created via API + And Agent v2 Web app access has been enabled via API + When I open the Agent v2 configure page + And I publish the Agent v2 draft + Then the Agent v2 draft should be published and up to date + When I open the Agent v2 Web app URL + And I send an E2E message in the Agent v2 Web app + Then the Agent v2 Web app should answer with the normal E2E marker + @web-app-runtime @published-web-app @stable-model Scenario: Published Web app remains isolated from unpublished Agent v2 draft edits Given I am signed in as the default E2E admin diff --git a/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts b/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts index f06ef7216d4c47..3674406b8979b1 100644 --- a/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts @@ -125,6 +125,20 @@ Then( }, ) +Then( + 'the Agent v2 Web app should answer with the normal E2E marker', + async function (this: DifyWorld) { + const webAppPage = this.agentBuilder.accessPoint.webAppPage + if (!webAppPage) + throw new Error('No Agent v2 Web app page was opened.') + + await expect(webAppPage.getByText(agentBuilderExpectedTokens.agentReply)) + .toBeVisible({ timeout: 120_000 }) + await webAppPage.close() + this.agentBuilder.accessPoint.webAppPage = undefined + }, +) + Then( 'the Agent v2 Web app response should not include the updated E2E marker', async function (this: DifyWorld) { From ab5e9f4a8d730be78c2de3e8d9b19b40bc55a30d Mon Sep 17 00:00:00 2001 From: yyh Date: Thu, 2 Jul 2026 06:04:31 +0800 Subject: [PATCH 185/185] test(e2e): make web app close explicit --- e2e/features/agent-v2/publish.feature | 5 +++- .../agent-v2/access-point-web-app.steps.ts | 27 +++++++------------ 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/e2e/features/agent-v2/publish.feature b/e2e/features/agent-v2/publish.feature index eb393ee96b5589..746b3c699657da 100644 --- a/e2e/features/agent-v2/publish.feature +++ b/e2e/features/agent-v2/publish.feature @@ -69,7 +69,8 @@ Feature: Agent v2 publish Then the Agent v2 draft should be published and up to date When I open the Agent v2 Web app URL And I send an E2E message in the Agent v2 Web app - Then the Agent v2 Web app should answer with the normal E2E marker + Then the Agent v2 Web app response should include the normal E2E marker + When I close the Agent v2 Web app @web-app-runtime @published-web-app @stable-model Scenario: Published Web app remains isolated from unpublished Agent v2 draft edits @@ -87,6 +88,7 @@ Feature: Agent v2 publish And I send an E2E message in the Agent v2 Web app Then the Agent v2 Web app response should include the normal E2E marker And the Agent v2 Web app response should not include the updated E2E marker + When I close the Agent v2 Web app @web-app-runtime @published-web-app @stable-model Scenario: Published Web app uses the latest Agent v2 published configuration @@ -105,3 +107,4 @@ Feature: Agent v2 publish When I open the Agent v2 Web app URL And I send an E2E message in the Agent v2 Web app Then the Agent v2 Web app response should include the updated E2E marker + When I close the Agent v2 Web app diff --git a/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts b/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts index 3674406b8979b1..34534edcbc8ecf 100644 --- a/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts +++ b/e2e/features/step-definitions/agent-v2/access-point-web-app.steps.ts @@ -108,8 +108,6 @@ Then( await expect(webAppPage.getByText(agentBuilderExpectedTokens.updatedAgentReply)) .toBeVisible({ timeout: 120_000 }) - await webAppPage.close() - this.agentBuilder.accessPoint.webAppPage = undefined }, ) @@ -125,20 +123,6 @@ Then( }, ) -Then( - 'the Agent v2 Web app should answer with the normal E2E marker', - async function (this: DifyWorld) { - const webAppPage = this.agentBuilder.accessPoint.webAppPage - if (!webAppPage) - throw new Error('No Agent v2 Web app page was opened.') - - await expect(webAppPage.getByText(agentBuilderExpectedTokens.agentReply)) - .toBeVisible({ timeout: 120_000 }) - await webAppPage.close() - this.agentBuilder.accessPoint.webAppPage = undefined - }, -) - Then( 'the Agent v2 Web app response should not include the updated E2E marker', async function (this: DifyWorld) { @@ -149,11 +133,18 @@ Then( await expect(webAppPage.getByText(agentBuilderExpectedTokens.updatedAgentReply)) .not .toBeVisible() - await webAppPage.close() - this.agentBuilder.accessPoint.webAppPage = undefined }, ) +When('I close the Agent v2 Web app', async function (this: DifyWorld) { + const webAppPage = this.agentBuilder.accessPoint.webAppPage + if (!webAppPage) + throw new Error('No Agent v2 Web app page was opened.') + + await webAppPage.close() + this.agentBuilder.accessPoint.webAppPage = undefined +}) + When('I open Agent v2 Embedded configuration', async function (this: DifyWorld) { await getWebAppCard(this).getByRole('button', { name: 'Embedded' }).click() })