dasLLVM: bool and pointer reinterprets, default<T[N]>, try/recover return, inline ptr_diff, speculatable lookups, --jit-remarks

.github/workflows/tree_sitter_daslang.yml

@@ -25,8 +25,10 @@ jobs:
     runs-on: ${{matrix.os}}
     strategy:
       fail-fast: false
+      # The swift toolchain parser-test-action installs is built for ubuntu 24.04 and does not
+      # load on ubuntu-latest (26.04): swift-build fails to find libxml2.so.2.
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-15]
+        os: [ubuntu-24.04, windows-latest, macos-15]
     steps:
       - name: Check out the grammar
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

doc/source/reference/language/statements.rst

@@ -773,6 +773,10 @@ dereference or out-of-bounds array access:
         print("recovered from panic\n")
     }
 
+A ``return`` inside the ``try`` block or the ``recover`` block is a compile error: set a
+variable in the arm and return after the pair. A closure's own ``return`` inside the arm is
+not affected.
+
 .. warning::
     ``try``/``recover`` is **not** a general error-handling mechanism and should not be
     used for control flow. It is designed for catching runtime panics (similar to Go's

include/daScript/ast/REVIEW.md

@@ -3,12 +3,14 @@
 **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc:
 `src/ast/ARCHITECTURE.md` (repo root).
 
-- **A diff that changes the module an existing bind registers into, or the name it registers
-  under (`vectorHomeModule`, `typeFactory<vector<TT>>::make`, `registerVectorFunctions`, the name
-  a `ManagedVectorAnnotation` or `ManagedStructureAnnotation` takes - all in `ast_handle.h`),
-  bumps `LLVM_JIT_CODEGEN_VERSION` in `modules/dasLLVM/daslib/llvm_jit_plan.das` (repo root), in
-  the same change.** The JIT's DLL cache key folds the codegen version and each function's AST
-  hash, never the module an extern lives in, so a cache hit binds the old name and crashes.
+- **A diff that changes the module or the name `ast_handle.h` itself registers an existing C++
+  binding under (`vectorHomeModule`, `typeFactory<vector<TT>>::make`, `registerVectorFunctions`,
+  and any name those templates compute) bumps `LLVM_JIT_CODEGEN_VERSION` in
+  `modules/dasLLVM/daslib/llvm_jit_plan.das` (repo root), in the same change.** The JIT's DLL
+  cache key folds the codegen version and each function's AST hash, never the module an extern
+  lives in, so a cache hit binds the old name and crashes. The name string a
+  `ManagedVectorAnnotation` or `ManagedStructureAnnotation` constructor takes is chosen where the
+  annotation is created, so that rename answers to the creating folder's checklist.
 
 - **A diff that changes the memory layout of a C++ type declared under this folder that some
   `Managed*Annotation` is instantiated over - a data member added, removed, reordered or retyped
@@ -24,17 +26,10 @@
   `skills/internal/abi_break_sweep.md` (repo root) too.** An out-of-tree subclass that omits
   `override` keeps compiling and silently stops being called.
 
-- **A diff that adds a field to `Function` or `Variable` (`ast.h`) holding something one
-  program's compile decides - whether the program uses it, the slot it holds in that program's
-  context, whether this program's JIT selected it - is a defect; the value goes into `Program`'s
-  symbol tables (`usedFunctions`, `functionIndices`, `usedVariables`, `variableIndices`,
-  `jitSelected`) instead.** A shared module's objects are one instance for every program in the
-  process, and a compile that runs inside another would otherwise overwrite the outer program's
-  answer mid-simulate.
-  `src/ast/ARCHITECTURE.md` sec.4 lists the per-compile fields that remain and why each is
-  tolerated.
-
-- **A diff changing `getVersion()` in `include/daScript/ast/ast_serializer.h`
-  replaces its version comment instead of appending to it.** The comment describes the current
-  version rather than earlier versions. A diff that keeps the old note in a parenthesis is a defect; git carries the
-  history.
+- **A diff that adds a field to `Function` or `Variable` (`ast.h`) whose value one compiled
+  program decides, so two programs sharing the module would want different values, is a defect;
+  the value goes into a per-program table on `Program` instead.** A shared module's objects are
+  one instance for every program in the process, and a compile that runs inside another would
+  otherwise overwrite the outer program's answer before the outer program has finished reading
+  it. `src/ast/ARCHITECTURE.md#program-scoped-symbol-state` lists the per-compile fields
+  `Function` and `Variable` carry and why each is safe.

include/daScript/ast/ast_infer_type.h

@@ -50,6 +50,8 @@ namespace das {
         vector<size_t> assumeStack;
         vector<size_t> assumeTypeStack;
         vector<bool> inFinally;
+        int tryDepth = 0;
+        vector<int> tryDepthAtClosure;
         vector<Function *> inInfer;
         ExprReturn * oneReturn;
         int32_t returnCount = 0;
@@ -551,6 +553,7 @@ namespace das {
         virtual ExpressionPtr visit(ExprClone *expr) override;
         // ExprTryCatch
         void preVisit(ExprTryCatch *expr) override;
+        ExpressionPtr visit(ExprTryCatch *expr) override;
         // ExprReturn
         bool inferReturnType(TypeDeclPtr &resType, ExprReturn *expr);
         bool isVoidReturnValueSettled(Expression *subexpr) const;

include/daScript/ast/ast_serializer.h

@@ -362,7 +362,7 @@ namespace das {
         AstSerializer & serializeModule ( Module & module, bool already_exists );
 
         static constexpr uint32_t getVersion () {
-            return 245;   // 245: inline splices are typed and no infer pass follows them
+            return 246;   // 246: a return inside a try or recover arm no longer infers
         }
 
         void serializeProgram ( ProgramPtr program, ModuleGroup & libGroup ) noexcept;

include/daScript/ast/compilation_errors.h

@@ -297,6 +297,7 @@ namespace das
     ,   invalid_empty_name                                          =   30296    // AST node with empty name; usually a macro emission bug
     ,   invalid_distinct_type                                       =   30297    // distinct underlying must be a workhorse type, not another distinct
     ,   invalid_with_module                                         =   30298    // with (module foo) names a module that is not part of the program
+    ,   return_in_try_recover                                       =   30299    // a return inside a try or recover arm
 
 // missing_*
 

modules/dasLLVM/ARCHITECTURE.md

@@ -112,7 +112,7 @@ bitcode and the link runs lld LTO - a dev probe artifact), `DAS_JIT_X64_FORCE_FE
 and `cpu_supports`-based tune eligibility all follow), `DAS_JIT_BASELINE` (build for a CPU class
 instead of the box - the machine, the gates, the tune ladder and the cache keys all follow;
 `ARCHITECTURE_TARGET_FEATURES.md` sec.10), `--jit-debug` / `-g` (emit DWARF or CodeView debug
-info and promote every argument to a stack slot for it; `ARCHITECTURE_DEBUG_INFO.md` sec.12), `--jit-compile-only` / `options jit_compile_only` (build, verify and optimize, then write, load and install nothing - the program runs interpreted; under a `--jit-target` cross triple it also keeps the `[tune]` grid a cross artifact would stamp as reference bodies, which is what the emission dump inspects), `--jit-sanitize` / `options jit_sanitize` (`address` runs LLVM's `asan` pass after the optimizer on functions marked `sanitize_address`; `undefined` runs `bounds-checking`; a sanitizer host defaults to its own, `none` to nothing; an exe or a library links with `-fsanitize=` for the host's sanitizers and the requested ones, through `clang++` unless a linker is named; an ASan host lowers every array push to the runtime call, which keeps the buffer's container annotation - `src/misc/ARCHITECTURE.md` sec.10), and the
+info and promote every argument to a stack slot for it; `ARCHITECTURE_DEBUG_INFO.md` sec.12), `--jit-remarks` / `--jit-remarks-missed` / `--jit-remarks-analysis` (LLVM's `-pass-remarks*` regexes, parsed once per process with the other JIT cl::opts: the matching passes' optimization remarks print to stderr, and codegen runs past the DLL and obj caches so every function's passes run), `--jit-compile-only` / `options jit_compile_only` (build, verify and optimize, then write, load and install nothing - the program runs interpreted; under a `--jit-target` cross triple it also keeps the `[tune]` grid a cross artifact would stamp as reference bodies, which is what the emission dump inspects), `--jit-sanitize` / `options jit_sanitize` (`address` runs LLVM's `asan` pass after the optimizer on functions marked `sanitize_address`; `undefined` runs `bounds-checking`; a sanitizer host defaults to its own, `none` to nothing; an exe or a library links with `-fsanitize=` for the host's sanitizers and the requested ones, through `clang++` unless a linker is named; an ASan host lowers every array push to the runtime call, which keeps the buffer's container annotation - `src/misc/ARCHITECTURE.md` sec.10), and the
 runtime escape API `tune_suppress_mint(knob)` (a library `[init]` suppresses the auto/restart mint; the
 caller passes the knob name it acts for). The announce contract: an override announces at the point it
 CHANGES THE OUTCOME - at least one line naming the knob (its env spelling, or the caller-supplied knob
@@ -195,10 +195,12 @@ shared one): the mangled-name hash is static, and the offset it names is added t
 globals base, which is what a cross-context call needs - the same JIT function runs on any context
 of the program, each with its own base, so the address cannot be baked. The emitter
 (`visitExprVar`, `daslib/llvm_jit.das`) emits the call where the variable is used and declares it
-`memory(none)` (`daslib/llvm_jit_common.das`): the result depends on nothing but its arguments for
-the lifetime of the context, so LLVM CSEs one use against a dominating one, hoists a loop's lookup
-into the preheader, and - the call not being `speculatable` - never moves it ahead of a branch, so
-a global written on one branch is looked up on that branch only. Every access to a global in a
+`memory(none)` and `speculatable` (`daslib/llvm_jit_common.das`): the result depends on nothing but
+its arguments for the lifetime of the context, and the probe neither traps nor writes, so LLVM CSEs
+one use against a dominating one and hoists a loop's lookup into the preheader even when the access
+sits on one arm of a branch inside the loop - one probe per call instead of one per taken iteration.
+Outside a loop the optimizer does not speculate a call, so a global written on one branch of
+straight-line code is still looked up on that branch only. Every access to a global in a
 function therefore shares one base pointer, which is what lets LLVM see `xs[j]` and `xs[j + 1]` as
 adjacent. Under `options solid_context` the address is instead `context->globals + stackTop`,
 computed once per function in the entry block.

modules/dasLLVM/CMakeLists.txt

@@ -186,6 +186,9 @@ if(NOT DAS_TOOLS_DISABLED)
                   modules/dasLLVM/tests/llvm_class_target_gates.das modules/dasLLVM/tests/llvm_gep_inbounds.das
                   modules/dasLLVM/tests/llvm_jit_baseline.das modules/dasLLVM/tests/llvm_jit_debug_info.das
                   modules/dasLLVM/tests/llvm_jit_global_lookup.das modules/dasLLVM/tests/llvm_jit_link.das
+                  modules/dasLLVM/tests/llvm_jit_ptr_diff_inline.das
+                  modules/dasLLVM/tests/llvm_jit_remarks.das
+                  modules/dasLLVM/tests/llvm_jit_pointer_aliasing.das
                   modules/dasLLVM/tests/llvm_jit_sanitize.das modules/dasLLVM/tests/llvm_layout_key.das
                   modules/dasLLVM/tests/llvm_second_context.das)
     das_lane_dastest_target(run_tests_llvm_tune COMMENT "Testing the llvm_tune framework"

modules/dasLLVM/REVIEW.md

@@ -5,18 +5,22 @@ docs: `ARCHITECTURE.md`, `ARCHITECTURE_CODEGEN_IDENTITY.md`, `ARCHITECTURE_DEBUG
 `ARCHITECTURE_EXE.md`, `ARCHITECTURE_JIT_ENTRY.md`, `ARCHITECTURE_LIB.md`,
 `ARCHITECTURE_TARGET_FEATURES.md`, `ARCHITECTURE_VECTOR_MATH.md`. Planned work: `LEDGER.md`.
 
-**A `[test]` file under this module that carries a `require dasllama/...` line of its own answers
-to `modules/dasLLAMA/REVIEW.md` (repo root) as well.**
+- **A `[test]` file under this module that carries a `require dasllama/...` line of its own
+  answers to `modules/dasLLAMA/REVIEW.md` (repo root) as well.**
 
-**A diff that changes how the compile handles a function because it carries `[tune]`,
-`[tune_perm]`, `[tune_companion]`, `[tune_scope]`, `[tune_policy]` or `[llvm_code]` - the body
-it emits for it, which permutation it picks for it - or what a read or a save of `<app>.tune.json`
-takes from a saved sidecar that is stale (its mtime is older than the running binary) or that
-another machine wrote, is reviewed with `skills/tune.md`.**
+- **A diff that changes how the compile handles a function because it carries `[tune]`,
+  `[tune_perm]`, `[tune_companion]`, `[tune_scope]`, `[tune_policy]` or `[llvm_code]` - the body
+  it emits for it, which permutation it picks for it - is reviewed with `skills/tune.md`.**
 
-**A change to the tune framework - `daslib/llvm_tune.das`, a test under `tests/` here that requires
-`llvm/daslib/llvm_tune` or spawns a child that does, or the descriptor and C++ rows that join it to
-a program (`.das_module`, `src/dasLLVM.cpp`) - is reviewed with `skills/internal/llvm_tune_internals.md`.**
+- **A diff that changes what a read or a save of `<app>.tune.json` takes from an
+  `<app>.tune.json` it finds stale (its mtime is older than the running binary) or written by
+  another machine is reviewed with `skills/tune.md`.**
+
+- **A change to the tune framework - `daslib/llvm_tune.das`, a test under `tests/` here that
+  requires `llvm/daslib/llvm_tune` or spawns a child that does, the
+  `registerModuleGroupMember("tune_framework", ...)` row in `src/dasLLVM.cpp`, or the
+  `register_module_group("tune_framework", ...)` row in `.das_module` - is reviewed with
+  `skills/internal/llvm_tune_internals.md`.**
 
 - **A change to any file under this module but a `.md` runs the module-owned suite** (command
   and build gate: `tests/README.md` here). CI runs some of its test files only nightly, so before
@@ -37,12 +41,12 @@ a program (`.das_module`, `src/dasLLVM.cpp`) - is reviewed with `skills/internal
   temp directory), and removes them** - a shared path under `build/` is one two concurrent runs
   collide on.
 
-- **A test under `tests/` here that spawns a daslang child keeps the child's artifacts inside a
-  directory it created uniquely for this process: `-output <dir>/...` for a `-exe` build,
-  `-no-module-cache` or `-module-cache <dir>/...` for a run that compiles through the front-end
-  cache, and `-jit-no-cache` or a pinned `jit_output_path` for a `-jit` run that executes the
-  script.** A child writes its caches relative to the cwd otherwise, which is the tree two
-  concurrent runs share.
+- **A test under `tests/` here that spawns a daslang child keeps every path the child writes
+  inside a directory the test created uniquely for this process: `-output <dir>/...` for a
+  `-exe` build; `-no-module-cache` or `-module-cache <dir>/...` for a run that compiles through
+  the front-end cache; `-jit-no-cache` or `-output <dir>/...` for a `-jit` run that executes the
+  script; a working directory inside `<dir>` covers the module cache and the JIT cache.** A child
+  writes its caches relative to the cwd otherwise, which is the tree two concurrent runs share.
 
 - **A diff that changes which arm a non-host triple selects in a branch keyed on the target
   triple (a key added, changed or removed), or changes code only a cross target's arm reaches,
@@ -202,10 +206,11 @@ a program (`.das_module`, `src/dasLLVM.cpp`) - is reviewed with `skills/internal
 - **A diff that changes the machine code a twinned emitter produces - its body, which of its
   arms a call selects, or the feature set its output is lowered under - leaves it covered by a
   cell comparing the emitted result with the interpreted result over the operand range that
-  emitter serves (every vector width for a vector emitter, every pair of int8 operands for a
-  dot), adding the cell in the same change when none covers that range.** A twinned emitter is
-  one the interpreter also computes, through a das body or a builtin: every `build_vector_*` in
-  `daslib/llvm_jit_intrin.das` and every `[llvm_code]` generator under this module.
+  emitter serves (every vector width for a vector emitter, every operand value for one over
+  8-bit operands, each operand class its arms branch on otherwise), adding the cell in the same
+  change when none covers that range.** A twinned emitter is a `build_vector_*` in
+  `daslib/llvm_jit_intrin.das` or an `[llvm_code]` generator under this module: the interpreter
+  computes the same answer through a das body or a builtin, so the two can drift apart.
 
 - **A test that searches the text of the LLVM IR the JIT emits matches an instruction by its
   opcode and flags (`fmul reassoc`), never by a numbered register (`%12`).** The numbering shifts

modules/dasLLVM/daslib/llvm_jit.das

@@ -4535,11 +4535,15 @@ class public LlvmJitVisitor : AstVisitor {
             }
         }
         let stride = expr.makeType.stride
-        let emptyEmbeddedTuple = expr.makeType.baseType == Type.tTuple && empty(expr.structs)
+        var mkBaseT = expr.makeType
+        while (mkBaseT.baseType == Type.tFixedArray && mkBaseT.firstType != null) {
+            mkBaseT = mkBaseT.firstType
+        }
+        let emptyEmbeddedTuple = mkBaseT.baseType == Type.tTuple && empty(expr.structs)
         let partiallyInitStruct = !expr.makeFlags.doesNotNeedInit && !expr.makeFlags.initAllFields
         if ((partiallyInitStruct || emptyEmbeddedTuple) && stride != 0) {
             let total = expr.structs |> length
-            let bytes = max(total, 1) * stride
+            let bytes = total != 0 ? total * stride : expr.makeType.sizeOf
             let align = uint(expr.makeType.alignOf)
             LLVMBuildMemSet(g_builder, mks_ptr, 0ul, uint64(bytes), align)
         }
@@ -5781,7 +5785,7 @@ class public LlvmJitVisitor : AstVisitor {
             res = LLVMBuildPointerCast(g_builder, subE, type_to_llvm_type(ect), "cast_p")
         } elif (ect.baseType == Type.tFixedArray && subT.isPointer) {
             res = LLVMBuildPointerCast(g_builder, subE, LLVMPointerType(type_to_llvm_type(ect), 0u), "")
-        } elif (is_int64_or_uint64(ect.baseType) && subT.isPointer) {
+        } elif (ect.isInteger && subT.isPointer) {
             res = LLVMBuildPtrToInt(g_builder, subE, type_to_llvm_type(ect), "cast_p_i")
         } elif (subT.baseType == Type.tFunction && (ect.isPointer || is_int64_or_uint64(ect.baseType))) {
             // reinterpret<uint64>(f)
@@ -5800,7 +5804,14 @@ class public LlvmJitVisitor : AstVisitor {
         } elif ((ect.isPointer || ect.isString) && subT.isInteger) {
             res = LLVMBuildIntToPtr(g_builder, subE, type_to_llvm_type(ect), "cast_i_p")
         } elif (!ect.isRefType && !subT.isRefType) {
-            if (ect.sizeOf == subT.sizeOf) {
+            if (ect.isBool && subT.isInteger) {
+                let low_byte = (subT.sizeOf > 1
+                    ? LLVMBuildTrunc(g_builder, subE, types.t_int8, "trunc_cast") : subE)
+                res = LLVMBuildICmp(g_builder, LLVMIntPredicate.LLVMIntNE, low_byte,
+                    LLVMConstNull(LLVMTypeOf(low_byte)), "cast_bool")
+            } elif (subT.isBool && ect.isInteger) {
+                res = LLVMBuildZExt(g_builder, subE, type_to_llvm_type(ect), "zext_cast")
+            } elif (ect.sizeOf == subT.sizeOf) {
                 res = LLVMBuildBitCast(g_builder, subE, type_to_llvm_type(ect), "cast_r")
             } elif (ect.sizeOf > subT.sizeOf) {
                 if (subT.isSignedInteger) {
@@ -6527,7 +6538,9 @@ class public LlvmJitVisitor : AstVisitor {
     def override preVisitExprReturn(expr : ExprReturn?) {
         if (thisBlock != null) {
             if (expr.returnFunc != null) {
-                to_log(LOG_WARNING, "LLVM JIT: return inside try/recover leaves the try arm, not the function. at {describe(expr.at)}\n")
+                let fix = "move the return after the try"
+                failed_E(expr,
+                    "a return inside a try/recover arm is not supported under -jit; {fix}")
             }
             pre_visit_expr_return(expr, isCMRESType(thisBlock.returnType))
         } elif (thisFunc != null) {