Merge branch 'main' into jahnvi/perf-fetch-optimization

Author: jahnvi480

OneBranchPipelines/stages/build-odbc-all-stage.yml

@@ -133,7 +133,7 @@ stages:
           # wheel that packaged the wrong (or an incomplete) subtree would still pass
           # `twine check`. This is the single-host cross-build's safety net: it proves
           # the ODBC_TARGET_* overrides selected the right libs/ subtree per tag and
-          # that no foreign-platform binaries leaked in.
+          # that no foreign-platform, foreign-architecture, or foreign-libc binaries leaked in.
           - powershell: |
               $ErrorActionPreference = "Stop"
               Add-Type -AssemblyName System.IO.Compression.FileSystem
@@ -149,10 +149,10 @@ stages:
                 'win_amd64'              = @{ Must = @('libs/windows/x64/', 'msodbcsql18.dll', 'mssql-auth.dll', 'libs/LICENSING'); Forbid = @('libs/linux/', 'libs/macos/', 'libs/windows/arm64/') }
                 'win_arm64'              = @{ Must = @('libs/windows/arm64/', 'msodbcsql18.dll', 'mssql-auth.dll', 'libs/LICENSING'); Forbid = @('libs/linux/', 'libs/macos/', 'libs/windows/x64/') }
                 'macosx_15_0_universal2' = @{ Must = @('libs/macos/arm64/', 'libs/macos/x86_64/', 'libmsodbcsql.18.dylib', 'libs/LICENSING'); Forbid = @('libs/windows/', 'libs/linux/') }
-                'manylinux_2_28_x86_64'  = @{ Must = @('libs/linux/debian_ubuntu/x86_64/', 'libmsodbcsql-18', 'libs/LICENSING'); Forbid = @('libs/windows/', 'libs/macos/', '/arm64/') }
-                'manylinux_2_28_aarch64' = @{ Must = @('libs/linux/debian_ubuntu/arm64/', 'libmsodbcsql-18', 'libs/LICENSING'); Forbid = @('libs/windows/', 'libs/macos/', '/x86_64/') }
-                'musllinux_1_2_x86_64'   = @{ Must = @('libs/linux/alpine/x86_64/', 'libmsodbcsql-18', 'libs/LICENSING'); Forbid = @('libs/windows/', 'libs/macos/', '/arm64/') }
-                'musllinux_1_2_aarch64'  = @{ Must = @('libs/linux/alpine/arm64/', 'libmsodbcsql-18', 'libs/LICENSING'); Forbid = @('libs/windows/', 'libs/macos/', '/x86_64/') }
+                'manylinux_2_28_x86_64'  = @{ Must = @('libs/linux/debian_ubuntu/x86_64/', 'libs/linux/rhel/x86_64/', 'libs/linux/suse/x86_64/', 'libmsodbcsql-18', 'libs/LICENSING'); Forbid = @('libs/windows/', 'libs/macos/', '/arm64/', 'libs/linux/alpine/') }
+                'manylinux_2_28_aarch64' = @{ Must = @('libs/linux/debian_ubuntu/arm64/', 'libs/linux/rhel/arm64/', 'libmsodbcsql-18', 'libs/LICENSING'); Forbid = @('libs/windows/', 'libs/macos/', '/x86_64/', 'libs/linux/alpine/') }
+                'musllinux_1_2_x86_64'   = @{ Must = @('libs/linux/alpine/x86_64/', 'libmsodbcsql-18', 'libs/LICENSING'); Forbid = @('libs/windows/', 'libs/macos/', '/arm64/', 'libs/linux/debian_ubuntu/', 'libs/linux/rhel/', 'libs/linux/suse/') }
+                'musllinux_1_2_aarch64'  = @{ Must = @('libs/linux/alpine/arm64/', 'libmsodbcsql-18', 'libs/LICENSING'); Forbid = @('libs/windows/', 'libs/macos/', '/x86_64/', 'libs/linux/debian_ubuntu/', 'libs/linux/rhel/', 'libs/linux/suse/') }
               }
 
               $failed = $false

PyPI_Description_ODBC.md

@@ -7,6 +7,11 @@ This package ships the platform-specific **Microsoft ODBC Driver 18 for SQL Serv
 (and the supporting runtime libraries they depend on) as a standalone, pure-data wheel, so that
 `mssql-python` does not have to bundle them inside its own wheel.
 
+Linux wheels are specific to both CPU architecture and libc family: `manylinux` wheels
+include the glibc-based Debian/Ubuntu, RHEL, and available SUSE driver variants, while
+`musllinux` wheels include only the Alpine/musl variant. Each retains the supporting
+libraries and resources for its included drivers.
+
 ## Not intended for direct use
 
 Do **not** install this package directly. Install

eng/conda_tools/__main__.py

@@ -37,8 +37,8 @@
     caught too;
   * no ``krb5``/``openssl``/``libltdl`` is VENDORED inside the payload (they are
     serviced by conda, never bundled).
-    * the complete supported driver inventory is present: alpine/debian_ubuntu/rhel/suse
-        for x86_64 and alpine/debian_ubuntu/rhel for arm64 (no SUSE ARM64 driver is shipped).
+  * the complete glibc driver inventory is present: debian_ubuntu/rhel/suse for x86_64
+    and debian_ubuntu/rhel for arm64 (no SUSE ARM64 driver is shipped).
 
 Non-Linux packages (``win-*`` / ``osx-*``) have no such ELF payload and are skipped.
 An unreadable/malformed package FAILS (it is never silently treated as non-Linux).

eng/conda_tools/contracts.py

@@ -42,13 +42,11 @@
 _MACHINE_NAME = {_EM_X86_64: "x86_64", _EM_AARCH64: "aarch64"}
 _REQUIRED_DRIVER_TREES = {
     "linux-64": {
-        ("alpine", "x86_64"),
         ("debian_ubuntu", "x86_64"),
         ("rhel", "x86_64"),
         ("suse", "x86_64"),
     },
     "linux-aarch64": {
-        ("alpine", "arm64"),
         ("debian_ubuntu", "arm64"),
         ("rhel", "arm64"),
     },
@@ -655,9 +653,9 @@ def validate_elf(
             )
         # musl/alpine variants (NEEDED libc.musl*) link differently -- their libodbcinst
         # statically resolves libltdl, so the glibc DT_NEEDED requirements below do not
-        # apply. There is no musl conda subdir (conda Linux is glibc-only); these variants
-        # ride along in the payload but are never the conda load target. The climb /
-        # presence / no-vendored checks still apply to them.
+        # apply. Older ODBC wheels also carry these variants, but they are never a
+        # conda load target (conda Linux is glibc-only). The climb / presence /
+        # no-vendored checks still apply when auditing those older payloads.
         is_musl = any("libc.musl" in n for n in needed)
         want = expected_climb_entry(name)
 
@@ -725,8 +723,8 @@ def validate_elf(
         )
     # Require the supported distro inventory for this architecture, then require EVERY
     # discovered driver lib dir to ship BOTH a driver and libodbcinst.so.2. The x86_64
-    # ODBC wheel supports alpine/debian_ubuntu/rhel/suse; the arm64 wheel supports
-    # alpine/debian_ubuntu/rhel (Microsoft does not ship a SUSE ARM64 driver tree).
+    # manylinux ODBC wheel supports debian_ubuntu/rhel/suse; the arm64 wheel supports
+    # debian_ubuntu/rhel (Microsoft does not ship a SUSE ARM64 driver tree).
     missing_trees = _REQUIRED_DRIVER_TREES[subdir] - driver_trees
     if missing_trees:
         errors.append(

setup_odbc.py

@@ -16,7 +16,8 @@
 full release matrix.
 
 Each wheel ships ONLY its own platform's ``libs/`` subtree (see
-``_target_libs_globs``). Because the package contains no compiled extension, a
+``_target_libs_globs``); manylinux and musllinux wheels also select only their
+own libc family. Because the package contains no compiled extension, a
 single build host can produce EVERY platform's wheel by setting
 ``ODBC_TARGET_PLATFORM_TAG`` / ``ODBC_TARGET_ARCH`` (see ``get_platform_info``),
 e.g. build all 7 release wheels on one Windows agent.
@@ -151,7 +152,8 @@ def _target_libs_globs(platform_tag: str, arch: str) -> list:
     the minimal set of ``libs/`` globs. Combined with ``include_package_data=False``
     this guarantees a Windows wheel never carries Linux/macOS binaries (and vice
     versa), whether the build runs on the native OS or is cross-built on a single
-    host via the ``ODBC_TARGET_*`` overrides.
+    host via the ``ODBC_TARGET_*`` overrides. Linux's manylinux and musllinux
+    tags additionally restrict the payload to glibc and musl, respectively.
     """
     # LICENSING travels with every wheel (it is a file at the libs/ root).
     globs = ["libs/LICENSING"]
@@ -173,9 +175,14 @@ def _subtree(root: str) -> None:
             if arch in ("x86_64", "amd64")
             else "arm64" if arch in ("aarch64", "arm64") else arch
         )
-        # A single Linux wheel serves all distro families for its arch; the driver
-        # is selected at runtime via /etc/*-release detection.
-        for distro in ("alpine", "debian_ubuntu", "rhel", "suse"):
+        # Keep the distro-specific layout used by the runtime resolver, but do
+        # not ship a different libc's binaries in manylinux/musllinux wheels.
+        distros = ("alpine", "debian_ubuntu", "rhel", "suse")
+        if tag.startswith("manylinux"):
+            distros = ("debian_ubuntu", "rhel", "suse")
+        elif tag.startswith("musllinux"):
+            distros = ("alpine",)
+        for distro in distros:
             _subtree(f"libs/linux/{distro}/{libs_arch}")
     else:
         raise OSError(f"Cannot determine libs subtree for platform tag {platform_tag!r}")

tests/test_029_bundled_binary_audit.py

@@ -138,8 +138,8 @@ def add_str(s):
 _CORE = "lib/python3.12/site-packages/mssql_py_core/mssql_py_core.cpython-312-x86_64-linux-gnu.so"
 _CORE_INIT = "lib/python3.12/site-packages/mssql_py_core/__init__.py"
 _DISTROS_BY_SUBDIR = {
-    "linux-64": ("alpine", "debian_ubuntu", "rhel", "suse"),
-    "linux-aarch64": ("alpine", "debian_ubuntu", "rhel"),
+    "linux-64": ("debian_ubuntu", "rhel", "suse"),
+    "linux-aarch64": ("debian_ubuntu", "rhel"),
 }
 
 
@@ -512,14 +512,22 @@ def test_malformed_required_core_elf_cannot_pass(tmp_path, damage):
 
 
 def test_audit_fails_when_entire_required_distro_tree_is_missing(tmp_path):
-    package = _make_pkg(tmp_path, distros=("debian_ubuntu", "rhel", "suse"))
+    package = _make_pkg(tmp_path, distros=("debian_ubuntu", "suse"))
     errors = audit.audit_package(package, "elf").violations
     assert any(
-        "missing required Linux driver trees" in error and "alpine/x86_64" in error
+        "missing required Linux driver trees" in error and "rhel/x86_64" in error
         for error in errors
     )
 
 
+@pytest.mark.parametrize("subdir,machine", [("linux-64", 62), ("linux-aarch64", 183)])
+def test_audit_accepts_glibc_only_driver_trees(tmp_path, subdir, machine):
+    package = _make_pkg(
+        tmp_path, subdir=subdir, machine=machine, distros=_DISTROS_BY_SUBDIR[subdir]
+    )
+    assert audit.audit_package(package, "elf").violations == []
+
+
 # --- N1: wrong climb variants must all FAIL --------------------------------
 
 
@@ -733,9 +741,8 @@ def add(name, data):
 
 
 def test_audit_allows_musl_variant_without_libltdl(tmp_path):
-    # The alpine/musl libodbcinst NEEDs libc.musl* and statically links ltdl, so the
-    # glibc libltdl DT_NEEDED requirement must NOT fail it. Package has a complete glibc
-    # debian_ubuntu variant plus an alpine/musl variant.
+    # Legacy manylinux wheels also carry Alpine. Its libodbcinst statically links
+    # libltdl, so glibc-specific DT_NEEDED requirements must not reject that payload.
     alpine_lib = "lib/python3.12/site-packages/mssql_python_odbc/libs/linux/alpine/x86_64/lib"
     p = tmp_path / "mssql-python-1.13.0-py312_0.tar.bz2"
     with tarfile.open(p, "w:bz2") as tf: