[AI Evals] Support test setup per suite

src/Tools/AI Test Toolkit/src/AITTestContext.Codeunit.al

@@ -84,6 +84,16 @@ codeunit 149044 "AIT Test Context"
         exit(AITTestContextImpl.GetExpectedData());
     end;
 
+    /// <summary>
+    /// Gets the continue on failure flag for the current turn.
+    /// If the flag is not set in the test input, it defaults to false.
+    /// </summary>
+    /// <returns>True if the eval should continue on failure, false otherwise.</returns>
+    procedure GetCanContinueOnFailure(): Boolean
+    begin
+        exit(AITTestContextImpl.GetCanContinueOnFailure());
+    end;
+
     /// <summary>
     /// Sets the answer for a question and answer evaluation.
     /// This will also copy the context, question and ground truth to the output dataset.
@@ -204,6 +214,34 @@ codeunit 149044 "AIT Test Context"
         AITTestContextImpl.SetTokenConsumption(TokensUsed);
     end;
 
+    /// <summary>
+    /// Gets the per-suite setup data as a Test Input Json.
+    /// The consuming test app must have imported a YAML file with the suite_setup token.
+    /// </summary>
+    /// <returns>Test Input Json containing the suite setup data.</returns>
+    procedure GetEvalSuiteSetupDataInput(): Codeunit "Test Input Json"
+    begin
+        exit(AITTestContextImpl.GetEvalSuiteSetupDataInput());
+    end;
+
+    /// <summary>
+    /// Marks the per-suite setup as completed on the test suite record.
+    /// Call this after your suite setup logic has finished successfully.
+    /// </summary>
+    procedure SetEvalSuiteSetupCompleted()
+    begin
+        AITTestContextImpl.SetEvalSuiteSetupCompleted();
+    end;
+
+    /// <summary>
+    /// Checks if the per-suite setup has been marked as done on the test suite record.
+    /// </summary>
+    /// <returns>True if suite setup has been executed.</returns>
+    procedure IsSuiteSetupDone(): Boolean
+    begin
+        exit(AITTestContextImpl.IsSuiteSetupDone());
+    end;
+
     /// <summary>
     /// Integration event that is raised after an eval run is completed.
     /// </summary>

src/Tools/AI Test Toolkit/src/AITTestContextImpl.Codeunit.al

@@ -18,6 +18,7 @@ codeunit 149043 "AIT Test Context Impl."
     var
         AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt.";
         GlobalTestOutputJson: Codeunit "Test Output Json";
+        GlobalSuiteSetupJson: Codeunit "Test Input Json";
         GlobalAccuracy: Decimal;
         CurrentTurn: Integer;
         NumberOfTurns: Integer;
@@ -29,6 +30,7 @@ codeunit 149043 "AIT Test Context Impl."
         ContextTok: Label 'context', Locked = true;
         GroundTruthTok: Label 'ground_truth', Locked = true;
         ExpectedDataTok: Label 'expected_data', Locked = true;
+        ContinueOnFailureTok: Label 'continue_on_failure', Locked = true;
         TestMetricsTok: Label 'test_metrics', Locked = true;
         TestSetupTok: Label 'test_setup', Locked = true;
         QuestionTok: Label 'question', Locked = true;
@@ -39,6 +41,9 @@ codeunit 149043 "AIT Test Context Impl."
         RoleTok: Label 'role', Locked = true;
         ContentTok: Label 'content', Locked = true;
         ConversationTok: Label 'conversation', Locked = true;
+        HasSuiteSetupData: Boolean;
+        SuiteSetupDataNotLoadedErr: Label 'Per-suite setup data has not been loaded.';
+        SuiteSetupInputCodeTok: Label 'SUITE-SETUP', Locked = true;
 
     /// <summary>
     /// Returns the Test Input value as Test Input Json Codeunit from the input dataset for the current iteration.
@@ -97,6 +102,23 @@ codeunit 149043 "AIT Test Context Impl."
         exit(GetTestInput(ExpectedDataTok));
     end;
 
+    /// <summary>
+    /// Gets the continue on failure flag for the current turn.
+    /// If the flag is not set in the test input, it defaults to false.
+    /// </summary>
+    /// <returns>True if the eval should continue on failure, false otherwise.</returns>
+    procedure GetCanContinueOnFailure(): Boolean
+    var
+        ContinueOnFailureInput: Codeunit "Test Input Json";
+        ElementFound: Boolean;
+    begin
+        ContinueOnFailureInput := GetTestInput(ContinueOnFailureTok, ElementFound);
+        if not ElementFound then
+            exit(false);
+
+        exit(ContinueOnFailureInput.ValueAsBoolean());
+    end;
+
     /// <summary>
     /// Sets the answer for a question and answer evaluation.
     /// This will also copy the context, question and ground truth to the output dataset.
@@ -332,6 +354,58 @@ codeunit 149043 "AIT Test Context Impl."
             NumberOfTurns := TurnsInputJson.GetElementCount()
         else
             NumberOfTurns := 1;
+
+        if not HasSuiteSetupData then
+            LoadSuiteSetupFromDataset();
+    end;
+
+    /// <summary>
+    /// Loads suite setup data from the dataset referenced by the current test's Test Input Group.
+    /// Resolves language variants using the suite's Run Language ID.
+    /// </summary>
+    local procedure LoadSuiteSetupFromDataset()
+    var
+        SuiteSetupGroup: Record "Test Input Group";
+        AITTestSuite: Record "AIT Test Suite";
+        TestInputCU: Codeunit "Test Input";
+        AITTestSuiteLanguage: Codeunit "AIT Test Suite Language";
+        SuiteSetupInputJson: Codeunit "Test Input Json";
+        ResolvedDatasetCode: Code[100];
+    begin
+        if not GetSuiteSetupGroup(SuiteSetupGroup) then
+            exit;
+
+        GetAITTestSuite(AITTestSuite);
+        ResolvedDatasetCode := AITTestSuiteLanguage.GetLanguageDataset(SuiteSetupGroup.Code, AITTestSuite."Run Language ID");
+        SuiteSetupInputJson := TestInputCU.GetTestInputByCode(ResolvedDatasetCode, SuiteSetupInputCodeTok);
+
+        if SuiteSetupInputJson.ToText() = '' then
+            exit;
+
+        ImportSuiteSetupData(SuiteSetupInputJson.AsJsonToken());
+    end;
+
+    /// <summary>
+    /// Finds the suite setup Test Input Group for the current test's dataset.
+    /// Navigates: AITTestMethodLine."Input Dataset" → TestInputGroup."Suite Setup Group Name" → target group.
+    /// </summary>
+    local procedure GetSuiteSetupGroup(var SuiteSetupGroup: Record "Test Input Group"): Boolean
+    var
+        AITTestMethodLine: Record "AIT Test Method Line";
+        DatasetGroup: Record "Test Input Group";
+    begin
+        GetAITTestMethodLine(AITTestMethodLine);
+        if AITTestMethodLine."Input Dataset" = '' then
+            exit(false);
+
+        if not DatasetGroup.Get(AITTestMethodLine."Input Dataset") then
+            exit(false);
+
+        if DatasetGroup."Suite Setup Group Name" = '' then
+            exit(false);
+
+        SuiteSetupGroup.SetRange("Group Name", DatasetGroup."Suite Setup Group Name");
+        exit(SuiteSetupGroup.FindFirst());
     end;
 
     /// <summary>
@@ -349,6 +423,22 @@ codeunit 149043 "AIT Test Context Impl."
             TestInputJson := TestInput.GetTestInput(ElementName);
     end;
 
+    /// <summary>
+    /// Gets the test input for the provided element, returning whether the element was found.
+    /// </summary>
+    /// <param name="ElementName">Element name to get from test input.</param>
+    /// <param name="ElementFound">Set to true if the element exists.</param>
+    /// <returns>Test Input Json for the element</returns>
+    local procedure GetTestInput(ElementName: Text; var ElementFound: Boolean) TestInputJson: Codeunit "Test Input Json"
+    var
+        TestInput: Codeunit "Test Input";
+    begin
+        if IsMultiTurn then
+            TestInputJson := TestInput.GetTestInput(TurnsTok).ElementAt(CurrentTurn - 1).ElementExists(ElementName, ElementFound)
+        else
+            TestInputJson := TestInput.GetTestInput().ElementExists(ElementName, ElementFound);
+    end;
+
     /// <summary>
     /// Adds a message to the test output for the current iteration.
     /// </summary>
@@ -411,6 +501,52 @@ codeunit 149043 "AIT Test Context Impl."
         CurrentTestOutputJson.Add(ElementName, TestInput.GetTestInput(ElementName).ValueAsText());
     end;
 
+    /// <summary>
+    /// Sets the per-suite setup data from a parsed JSON token.
+    /// </summary>
+    /// <param name="SuiteSetupJsonToken">The JSON token containing the parsed suite setup data.</param>
+    internal procedure ImportSuiteSetupData(SuiteSetupJsonToken: JsonToken)
+    begin
+        GlobalSuiteSetupJson.Initialize(SuiteSetupJsonToken);
+        HasSuiteSetupData := true;
+    end;
+
+    /// <summary>
+    /// Gets the per-suite setup data as a Test Input Json.
+    /// </summary>
+    /// <returns>Test Input Json containing the suite setup data.</returns>
+    procedure GetEvalSuiteSetupDataInput(): Codeunit "Test Input Json"
+    begin
+        if not HasSuiteSetupData then
+            Error(SuiteSetupDataNotLoadedErr);
+        exit(GlobalSuiteSetupJson);
+    end;
+
+    /// <summary>
+    /// Marks the per-suite setup as completed on the suite setup test input group.
+    /// </summary>
+    procedure SetEvalSuiteSetupCompleted()
+    var
+        SuiteSetupGroup: Record "Test Input Group";
+    begin
+        if GetSuiteSetupGroup(SuiteSetupGroup) then
+            SuiteSetupGroup.SetSuiteSetupDone();
+    end;
+
+    /// <summary>
+    /// Checks if the per-suite setup has been marked as done on the suite setup test input group.
+    /// </summary>
+    /// <returns>True if suite setup has been executed.</returns>
+    procedure IsSuiteSetupDone(): Boolean
+    var
+        SuiteSetupGroup: Record "Test Input Group";
+    begin
+        if not GetSuiteSetupGroup(SuiteSetupGroup) then
+            exit(false);
+
+        exit(SuiteSetupGroup."Suite Setup Done");
+    end;
+
     /// <summary>
     /// Sets the token consumption for the method line run. Useful if external calls are made outside of AI toolkit.
     /// </summary>

src/Tools/AI Test Toolkit/src/TestSuite/AITALTestSuiteMgt.Codeunit.al

@@ -17,6 +17,7 @@ codeunit 149037 "AIT AL Test Suite Mgt"
     var
         RunProcedureOperationTok: Label 'Run Procedure', Locked = true;
         AITTestSuitePrefixTok: Label 'AIT-', Locked = true;
+        InvalidSuiteSetupYamlErr: Label 'The per-suite setup content is not valid YAML.';
 
     internal procedure GetDefaultRunProcedureOperationLbl(): Text
     begin
@@ -224,6 +225,22 @@ codeunit 149037 "AIT AL Test Suite Mgt"
         exit(OutputJsonArray);
     end;
 
+    /// <summary>
+    /// Import per-suite setup data from YAML content.
+    /// The YAML is parsed and stored for access via AITTestContext.GetSuiteSetupData().
+    /// </summary>
+    /// <param name="SuiteSetupYamlContent">The YAML content as text.</param>
+    procedure ImportSuiteSetupData(SuiteSetupYamlContent: Text)
+    var
+        AITTestContextImpl: Codeunit "AIT Test Context Impl.";
+        SuiteSetupDataJsonObj: JsonObject;
+    begin
+        if not SuiteSetupDataJsonObj.ReadFromYaml(SuiteSetupYamlContent) then
+            Error(InvalidSuiteSetupYamlErr);
+
+        AITTestContextImpl.ImportSuiteSetupData(SuiteSetupDataJsonObj.AsToken());
+    end;
+
     /// <summary>
     /// Import the Test Input Dataset from an InStream of a dataset in a supported format.
     /// Overwrite the dataset if the dataset with same filename is already imported by the same app

src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLines.Page.al

@@ -5,6 +5,8 @@
 
 namespace System.TestTools.AITestToolkit;
 
+using System.TestTools.TestRunner;
+
 page 149034 "AIT Test Method Lines"
 {
     Caption = 'Evals';
@@ -239,6 +241,32 @@ page 149034 "AIT Test Method Lines"
                     CurrPage.Update(false);
                 end;
             }
+            action(ResetSuiteSetup)
+            {
+                Caption = 'Reset Suite Setup';
+                ToolTip = 'Resets the per-suite setup flag so that the setup data can be created again.';
+                Image = ResetStatus;
+
+                trigger OnAction()
+                var
+                    TestInputGroup: Record "Test Input Group";
+                    SuiteSetupGroup: Record "Test Input Group";
+                    InputDatasetCode: Code[100];
+                begin
+                    InputDatasetCode := Rec.GetTestInputCode();
+                    if not TestInputGroup.Get(InputDatasetCode) then
+                        exit;
+
+                    if TestInputGroup."Suite Setup Group Name" = '' then
+                        exit;
+
+                    SuiteSetupGroup.SetRange("Group Name", TestInputGroup."Suite Setup Group Name");
+                    if SuiteSetupGroup.FindFirst() then begin
+                        SuiteSetupGroup.ResetSuiteSetup();
+                        CurrPage.Update(false);
+                    end;
+                end;
+            }
             action(LogEntries)
             {
                 Caption = 'Log Entries';

src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuite.Page.al

@@ -347,6 +347,18 @@ page 149031 "AIT Test Suite"
                 end;
             }
 
+            action(ResetSuiteSetup)
+            {
+                Caption = 'Reset Suite Setup';
+                ToolTip = 'Resets the per-suite setup flag so that the setup can be run again.';
+                Image = ResetStatus;
+
+                trigger OnAction()
+                begin
+                    Rec.ResetSuiteSetup();
+                    CurrPage.Update(false);
+                end;
+            }
             action(Compare)
             {
                 Caption = 'View runs';

src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuite.Table.al

@@ -298,6 +298,22 @@ table 149030 "AIT Test Suite"
         exit(AITTestSuiteLanguage.GetLanguageDataset(Rec."Input Dataset", Rec."Run Language ID"));
     end;
 
+    internal procedure ResetSuiteSetup()
+    var
+        TestInputGroup: Record "Test Input Group";
+        SuiteSetupGroup: Record "Test Input Group";
+    begin
+        if not TestInputGroup.Get(Rec."Input Dataset") then
+            exit;
+
+        if TestInputGroup."Suite Setup Group Name" = '' then
+            exit;
+
+        SuiteSetupGroup.SetRange("Group Name", TestInputGroup."Suite Setup Group Name");
+        if SuiteSetupGroup.FindFirst() then
+            SuiteSetupGroup.ResetSuiteSetup();
+    end;
+
     var
         BaseVersionMustBeLessThanVersionErr: Label 'Base Version must be less than or equal to Version';
 }

src/Tools/Test Framework/Test Runner/src/ALTestSuite.Table.al

@@ -115,5 +115,6 @@ table 130451 "AL Test Suite"
         if "Test Runner Id" = 0 then
             "Test Runner Id" := TestRunnerMgt.GetDefaultTestRunner();
     end;
+
 }
 

src/Tools/Test Framework/Test Runner/src/DataDrivenTest/DataInputs/TestInput.Codeunit.al

@@ -66,6 +66,22 @@ codeunit 130460 "Test Input"
         exit(DataPerTest.GetInput(DataPerTest));
     end;
 
+    procedure GetTestInputByCode(GroupCode: Code[100]; InputCode: Code[100]): Codeunit "Test Input Json"
+    var
+        TestInputRec: Record "Test Input";
+        TestInputJson: Codeunit "Test Input Json";
+        InputContent: Text;
+    begin
+        if not TestInputRec.Get(GroupCode, InputCode) then
+            exit(TestInputJson);
+
+        InputContent := TestInputRec.GetInput(TestInputRec);
+        if InputContent <> '' then
+            TestInputJson.Initialize(InputContent);
+
+        exit(TestInputJson);
+    end;
+
     var
         DataPerTest: Record "Test Input";
         DataPerTestTestInput: Codeunit "Test Input Json";

src/Tools/Test Framework/Test Runner/src/DataDrivenTest/DataInputs/TestInput.Page.al

@@ -50,6 +50,10 @@ page 130457 "Test Input"
                 {
                     Editable = false;
                 }
+                field("Suite Setup Done"; Rec."Suite Setup Done")
+                {
+                    Importance = Additional;
+                }
             }
             part(TestInputPart; "Test Input Part")
             {