Update to v2.1.0-alpha.1 and mache v5.2.0

.github/workflows/build_workflow.yml

@@ -95,7 +95,6 @@ jobs:
         run: |
           git config --global url."https://github.com/".insteadOf "git@github.com:"
           ./deploy.py \
-            --mache-fork xylar/mache --mache-branch add-target-software-cli-flags \
             --no-spack \
             --python ${{ matrix.python-version }} \
             --recreate

.github/workflows/gis_coarse_meshgen.yml

@@ -41,7 +41,6 @@ jobs:
         run: |
           git config --global url."https://github.com/".insteadOf "git@github.com:"
           ./deploy.py \
-            --mache-fork xylar/mache --mache-branch add-target-software-cli-flags \
             --no-spack \
             --python 3.14 \
             --recreate
@@ -96,6 +95,13 @@ jobs:
           testDir=$root/gis_mesh_gen
           compass setup -w $testDir -t landice/greenland/mesh_gen -f system.cfg
 
-          # run the test case
+          # Some runners have an Azure (MANA) RDMA device that UCX cannot
+          # open, which makes MPI_Init fail, so stick to non-RDMA transports
+          export UCX_TLS=self,sm,tcp
+
+          # run the test case, printing its log if it fails
           cd $testDir
-          compass run
+          if ! compass run; then
+            cat case_outputs/*.log
+            exit 1
+          fi

compass/machines/pm-cpu.cfg

@@ -36,6 +36,11 @@ spack = /global/cfs/cdirs/e3sm/software/compass/pm-cpu/spack
 # pnetcdf as E3SM (spack modules are used otherwise)
 use_e3sm_hdf5_netcdf = True
 
+# the number of parallel jobs for spack builds on login nodes, which cap each
+# user's memory at 30 GiB (Albany exceeds that with 16 jobs); ignored inside
+# a Slurm job
+spack_build_jobs = 8
+
 # The parallel section describes options related to running jobs in parallel.
 # Most options in this section come from mache so here we just add or override
 # some defaults

compass/machines/pm-gpu.cfg

@@ -39,6 +39,11 @@ spack = /global/cfs/cdirs/e3sm/software/compass/pm-gpu/spack
 # pnetcdf as E3SM (spack modules are used otherwise)
 use_e3sm_hdf5_netcdf = True
 
+# the number of parallel jobs for spack builds on login nodes, which cap each
+# user's memory at 30 GiB (Albany's CUDA build exceeds that with 8 jobs);
+# ignored inside a Slurm job
+spack_build_jobs = 4
+
 # spack variants for Albany and Trilinos
 albany_variants = +mpas~py+unit_tests+cuda+sfad sfadsize=12
 trilinos_variants = +cuda+ampere80+zen3

compass/version.py

@@ -1 +1 @@
-__version__ = '2.0.0-alpha.3'
+__version__ = '2.1.0-alpha.1'

deploy/cli_spec.json

@@ -1,7 +1,7 @@
 {
   "meta": {
     "software": "compass",
-    "mache_version": "3.7.1",
+    "mache_version": "5.2.0",
     "description": "Deploy compass environment"
   },
   "arguments": [
@@ -57,6 +57,12 @@
       "help": "Path to the Spack checkout to use (overrides spack.spack_path in deploy/config.yaml.j2).",
       "route": ["deploy", "run"]
     },
+    {
+      "flags": ["--spack-pins"],
+      "dest": "spack_pins",
+      "help": "YAML file overriding the pinned Spack, spack-packages and e3sm-spack-packages sources (a tag, commit or branch per repository), for testing unreleased recipes.",
+      "route": ["deploy", "run"]
+    },
     {
       "flags": ["--recreate"],
       "dest": "recreate",

deploy/config.yaml.j2

@@ -164,6 +164,15 @@ spack:
   # excluding `hdf5_netcdf`.
   exclude_packages: []
 
+  # Overrides for the Spack sources pinned by mache (mache/spack/pins.yaml).
+  # e3sm-spack-packages is pinned past mache 5.2.0's v2026.06.2 for the
+  # trilinos-for-albany and albany fixes in E3SM-Project/e3sm-spack-packages#4,
+  # until a mache release pins a tag that includes them.
+  pins:
+    repos:
+      e3sm:
+        commit: 65739a6a79f45acffbf0ac24b754f4841ed2065a
+
 jigsaw:
   # If true, build/install JIGSAW + JIGSAW-Python into the pixi env
   enabled: true

deploy/hooks.py

@@ -41,6 +41,9 @@ def pre_spack(ctx: DeployContext) -> dict[str, Any] | None:
         updates['spack'] = {'spack_path': spack_path}
     if exclude_packages:
         updates.setdefault('spack', {})['exclude_packages'] = exclude_packages
+    build_jobs = _get_spack_build_jobs(ctx.machine_config)
+    if build_jobs is not None:
+        updates.setdefault('spack', {})['build_jobs'] = build_jobs
 
     if _with_albany(ctx):
         _check_albany_support(ctx.machine, toolchain_pairs)
@@ -262,6 +265,20 @@ def _maybe_exclude_e3sm_hdf5_netcdf(
         exclude_packages.append('hdf5_netcdf')
 
 
+def _get_spack_build_jobs(machine_config) -> int | None:
+    # the limit is for login nodes; a Slurm job has the node's memory
+    if os.environ.get('SLURM_JOB_ID'):
+        return None
+    if not machine_config.has_section('deploy'):
+        return None
+    value = machine_config.get(
+        'deploy', 'spack_build_jobs', fallback=''
+    ).strip()
+    if not value:
+        return None
+    return int(value)
+
+
 def _maybe_exclude_cmake(
     exclude_packages: list[str], machine_config
 ) -> None:

deploy/pins.cfg

@@ -4,8 +4,8 @@ bootstrap_python = 3.14
 python = 3.14
 esmf = 8.9.1
 geometric_features = 1.6.3
-mache = 3.7.1
-mpas_tools = 1.5.1
+mache = 5.2.0
+mpas_tools = 2.1.0
 otps = 2021.10
 parallelio = 2.6.9
 
@@ -24,6 +24,6 @@ trilinos_variants =
 
 # pins for both pixi and spack environments
 [all]
-netcdf_c = 4.10.0
-netcdf_fortran = 4.6.2
-pnetcdf = 1.14.1
+netcdf_c = 4.10.1
+netcdf_fortran = 4.6.3
+pnetcdf = 1.15.0

deploy/spack/katara_gnu_openmpi.yaml

@@ -1,42 +1,84 @@
-{%- set compiler = "gcc@13.3.0" %}
-{%- set mpi = "openmpi@4.1.6" %}
+{%- set compiler = "gcc@15.2.0" %}
+{%- set mpi = "openmpi@5.0.10" %}
 spack:
   specs:
-  - {{ compiler }}
-  - {{ mpi }}%{{ compiler }}
+  - {{ mpi }}
 {%- for spec in specs %}
-  - "{{ spec }}%{{ compiler }}"
+  - "{{ spec }}"
 {%- endfor %}
   concretizer:
     unify: true
+  toolchains:
+    mache:
+    - spec: "%c={{ compiler }}"
+      when: "%c"
+    - spec: "%cxx={{ compiler }}"
+      when: "%cxx"
+    - spec: "%fortran={{ compiler }}"
+      when: "%fortran"
   packages:
     all:
-      compiler: [{{ compiler }}]
+      require: ["%mache"]
       providers:
-        mpi: [{{ mpi }}%{{ compiler }}]
+        mpi: [{{ mpi }}]
+    autoconf:
+      externals:
+      - spec: autoconf@2.72
+        prefix: /usr
+      buildable: false
+    automake:
+      externals:
+      - spec: automake@1.18.1
+        prefix: /usr
+      buildable: false
+    diffutils:
+      externals:
+      - spec: diffutils@3.12
+        prefix: /usr
+      buildable: false
+    git:
+      externals:
+      - spec: git@2.53.0
+        prefix: /usr
+      buildable: false
+    gmake:
+      externals:
+      - spec: gmake@4.4.1
+        prefix: /usr
+      buildable: false
+    m4:
+      externals:
+      - spec: m4@1.4.21
+        prefix: /usr
+      buildable: false
+    perl:
+      externals:
+      - spec: perl@5.40.1
+        prefix: /usr
+      buildable: false
+    tar:
+      externals:
+      - spec: tar@1.35
+        prefix: /usr
+      buildable: false
     gcc:
       externals:
-      - spec: {{ compiler }}
+      - spec: {{ compiler }} languages='c,c++,fortran'
         prefix: /usr
+        extra_attributes:
+          compilers:
+            c: /usr/bin/gcc
+            cxx: /usr/bin/g++
+            fortran: /usr/bin/gfortran
       buildable: false
     openmpi:
       externals:
-      - spec: {{ mpi }}%{{ compiler }}
-        prefix: /usr
-      buildable: false
-  config:
-    install_missing_compilers: false
-  compilers:
-  - compiler:
-      spec: {{ compiler }}
-      paths:
-        cc: /usr/bin/gcc
-        cxx: /usr/bin/g++
-        f77: /usr/bin/gfortran
-        fc: /usr/bin/gfortran
-      flags: {}
-      operating_system: ubuntu24.04
-      target: x86_64
-      modules: []
-      environment: {}
-      extra_rpaths: []
+      - spec: {{ mpi }}
+        prefix: /usr
+        # mache builds the environment under `env -i`, and the Open MPI 5
+        # compiler wrappers fail when HOME is unset
+        extra_attributes:
+          environment:
+            set:
+              HOME: /home/xylar
+      buildable: false

docs/developers_guide/deploying_spack.rst

@@ -99,6 +99,13 @@ Useful flags
 If you need a non-default temporary directory for Spack builds, set
 ``spack.tmpdir`` in ``deploy/config.yaml.j2`` before running deployment.
 
+Where login nodes limit each user's memory, set ``spack_build_jobs`` in the
+``[deploy]`` section of the machine config file to limit the number of
+parallel Spack build jobs.  It is ignored inside a Slurm job, where the
+deployment has a whole node's memory.  Perlmutter uses 8 on pm-cpu and 4 on
+pm-gpu, where Albany's CUDA build needs more memory (see
+:ref:`dev_deploying_spack_albany`).
+
 Testing against a mache branch
 ------------------------------
 
@@ -115,6 +122,8 @@ and branch explicitly:
 This keeps the Compass deployment workflow the same while swapping in the
 requested ``mache`` branch during bootstrap and deployment.
 
+.. _dev_deploying_spack_albany:
+
 Albany deployments
 ------------------
 
@@ -130,6 +139,23 @@ If you are testing MALI workflows that require Albany, add
 Albany support is restricted to the machine/compiler/MPI combinations allowed
 by the deployment hooks and machine configuration.
 
+.. important::
+
+    On Perlmutter, deploy Albany-enabled Spack environments for ``pm-gpu``
+    from a compute node.  Login nodes cap each user's memory at 30 GiB, and
+    Albany's CUDA build only fits with 4 parallel build jobs, which takes
+    hours.  An interactive allocation usually starts within minutes.  Point
+    pixi's cache at ``$SCRATCH`` first, since pixi cannot lock files in its
+    default cache in your home directory from a compute node:
+
+    .. code-block:: bash
+
+        export PIXI_CACHE_DIR=${SCRATCH}/pixi-cache
+        salloc -A e3sm -q interactive -C cpu -N 1 -t 4:00:00 \
+            srun -N 1 -n 1 -c 128 ./deploy.py --machine pm-gpu \
+            --compiler gnugpu --mpi mpich --with-albany --deploy-spack \
+            --spack-path ${SCRATCH_SPACK}
+
 Validating the deployment
 =========================