diff --git a/.github/workflows/build_workflow.yml b/.github/workflows/build_workflow.yml index cafc30c184..63abb8d4d0 100644 --- a/.github/workflows/build_workflow.yml +++ b/.github/workflows/build_workflow.yml @@ -95,7 +95,6 @@ jobs: run: | git config --global url."https://github.com/".insteadOf "git@github.com:" ./deploy.py \ - --mache-fork xylar/mache --mache-branch add-target-software-cli-flags \ --no-spack \ --python ${{ matrix.python-version }} \ --recreate diff --git a/.github/workflows/gis_coarse_meshgen.yml b/.github/workflows/gis_coarse_meshgen.yml index 17dca534f5..88eff34223 100644 --- a/.github/workflows/gis_coarse_meshgen.yml +++ b/.github/workflows/gis_coarse_meshgen.yml @@ -41,7 +41,6 @@ jobs: run: | git config --global url."https://github.com/".insteadOf "git@github.com:" ./deploy.py \ - --mache-fork xylar/mache --mache-branch add-target-software-cli-flags \ --no-spack \ --python 3.14 \ --recreate @@ -96,6 +95,13 @@ jobs: testDir=$root/gis_mesh_gen compass setup -w $testDir -t landice/greenland/mesh_gen -f system.cfg - # run the test case + # Some runners have an Azure (MANA) RDMA device that UCX cannot + # open, which makes MPI_Init fail, so stick to non-RDMA transports + export UCX_TLS=self,sm,tcp + + # run the test case, printing its log if it fails cd $testDir - compass run + if ! compass run; then + cat case_outputs/*.log + exit 1 + fi diff --git a/compass/machines/pm-cpu.cfg b/compass/machines/pm-cpu.cfg index e6c26be01d..5cecd4395c 100644 --- a/compass/machines/pm-cpu.cfg +++ b/compass/machines/pm-cpu.cfg @@ -36,6 +36,11 @@ spack = /global/cfs/cdirs/e3sm/software/compass/pm-cpu/spack # pnetcdf as E3SM (spack modules are used otherwise) use_e3sm_hdf5_netcdf = True +# the number of parallel jobs for spack builds on login nodes, which cap each +# user's memory at 30 GiB (Albany exceeds that with 16 jobs); ignored inside +# a Slurm job +spack_build_jobs = 8 + # The parallel section describes options related to running jobs in parallel. # Most options in this section come from mache so here we just add or override # some defaults diff --git a/compass/machines/pm-gpu.cfg b/compass/machines/pm-gpu.cfg index 3ea5f74768..9a47b32aa6 100644 --- a/compass/machines/pm-gpu.cfg +++ b/compass/machines/pm-gpu.cfg @@ -39,6 +39,11 @@ spack = /global/cfs/cdirs/e3sm/software/compass/pm-gpu/spack # pnetcdf as E3SM (spack modules are used otherwise) use_e3sm_hdf5_netcdf = True +# the number of parallel jobs for spack builds on login nodes, which cap each +# user's memory at 30 GiB (Albany's CUDA build exceeds that with 8 jobs); +# ignored inside a Slurm job +spack_build_jobs = 4 + # spack variants for Albany and Trilinos albany_variants = +mpas~py+unit_tests+cuda+sfad sfadsize=12 trilinos_variants = +cuda+ampere80+zen3 diff --git a/compass/version.py b/compass/version.py index fa98dd1c47..168a408c50 100644 --- a/compass/version.py +++ b/compass/version.py @@ -1 +1 @@ -__version__ = '2.0.0-alpha.3' +__version__ = '2.1.0-alpha.1' diff --git a/deploy/cli_spec.json b/deploy/cli_spec.json index 544058c09c..38446dc507 100644 --- a/deploy/cli_spec.json +++ b/deploy/cli_spec.json @@ -1,7 +1,7 @@ { "meta": { "software": "compass", - "mache_version": "3.7.1", + "mache_version": "5.2.0", "description": "Deploy compass environment" }, "arguments": [ @@ -57,6 +57,12 @@ "help": "Path to the Spack checkout to use (overrides spack.spack_path in deploy/config.yaml.j2).", "route": ["deploy", "run"] }, + { + "flags": ["--spack-pins"], + "dest": "spack_pins", + "help": "YAML file overriding the pinned Spack, spack-packages and e3sm-spack-packages sources (a tag, commit or branch per repository), for testing unreleased recipes.", + "route": ["deploy", "run"] + }, { "flags": ["--recreate"], "dest": "recreate", diff --git a/deploy/config.yaml.j2 b/deploy/config.yaml.j2 index 0ae885a3fa..1df2fdcc46 100644 --- a/deploy/config.yaml.j2 +++ b/deploy/config.yaml.j2 @@ -164,6 +164,15 @@ spack: # excluding `hdf5_netcdf`. exclude_packages: [] + # Overrides for the Spack sources pinned by mache (mache/spack/pins.yaml). + # e3sm-spack-packages is pinned past mache 5.2.0's v2026.06.2 for the + # trilinos-for-albany and albany fixes in E3SM-Project/e3sm-spack-packages#4, + # until a mache release pins a tag that includes them. + pins: + repos: + e3sm: + commit: 65739a6a79f45acffbf0ac24b754f4841ed2065a + jigsaw: # If true, build/install JIGSAW + JIGSAW-Python into the pixi env enabled: true diff --git a/deploy/hooks.py b/deploy/hooks.py index df4c30b743..06948eb87c 100644 --- a/deploy/hooks.py +++ b/deploy/hooks.py @@ -41,6 +41,9 @@ def pre_spack(ctx: DeployContext) -> dict[str, Any] | None: updates['spack'] = {'spack_path': spack_path} if exclude_packages: updates.setdefault('spack', {})['exclude_packages'] = exclude_packages + build_jobs = _get_spack_build_jobs(ctx.machine_config) + if build_jobs is not None: + updates.setdefault('spack', {})['build_jobs'] = build_jobs if _with_albany(ctx): _check_albany_support(ctx.machine, toolchain_pairs) @@ -262,6 +265,20 @@ def _maybe_exclude_e3sm_hdf5_netcdf( exclude_packages.append('hdf5_netcdf') +def _get_spack_build_jobs(machine_config) -> int | None: + # the limit is for login nodes; a Slurm job has the node's memory + if os.environ.get('SLURM_JOB_ID'): + return None + if not machine_config.has_section('deploy'): + return None + value = machine_config.get( + 'deploy', 'spack_build_jobs', fallback='' + ).strip() + if not value: + return None + return int(value) + + def _maybe_exclude_cmake( exclude_packages: list[str], machine_config ) -> None: diff --git a/deploy/pins.cfg b/deploy/pins.cfg index 8343386311..1dfb3e3ffa 100644 --- a/deploy/pins.cfg +++ b/deploy/pins.cfg @@ -4,8 +4,8 @@ bootstrap_python = 3.14 python = 3.14 esmf = 8.9.1 geometric_features = 1.6.3 -mache = 3.7.1 -mpas_tools = 1.5.1 +mache = 5.2.0 +mpas_tools = 2.1.0 otps = 2021.10 parallelio = 2.6.9 @@ -24,6 +24,6 @@ trilinos_variants = # pins for both pixi and spack environments [all] -netcdf_c = 4.10.0 -netcdf_fortran = 4.6.2 -pnetcdf = 1.14.1 +netcdf_c = 4.10.1 +netcdf_fortran = 4.6.3 +pnetcdf = 1.15.0 diff --git a/deploy/spack/katara_gnu_openmpi.yaml b/deploy/spack/katara_gnu_openmpi.yaml index 2a99a0a046..c1977a8cff 100644 --- a/deploy/spack/katara_gnu_openmpi.yaml +++ b/deploy/spack/katara_gnu_openmpi.yaml @@ -1,42 +1,84 @@ -{%- set compiler = "gcc@13.3.0" %} -{%- set mpi = "openmpi@4.1.6" %} +{%- set compiler = "gcc@15.2.0" %} +{%- set mpi = "openmpi@5.0.10" %} spack: specs: - - {{ compiler }} - - {{ mpi }}%{{ compiler }} + - {{ mpi }} {%- for spec in specs %} - - "{{ spec }}%{{ compiler }}" + - "{{ spec }}" {%- endfor %} concretizer: unify: true + toolchains: + mache: + - spec: "%c={{ compiler }}" + when: "%c" + - spec: "%cxx={{ compiler }}" + when: "%cxx" + - spec: "%fortran={{ compiler }}" + when: "%fortran" packages: all: - compiler: [{{ compiler }}] + require: ["%mache"] providers: - mpi: [{{ mpi }}%{{ compiler }}] + mpi: [{{ mpi }}] + autoconf: + externals: + - spec: autoconf@2.72 + prefix: /usr + buildable: false + automake: + externals: + - spec: automake@1.18.1 + prefix: /usr + buildable: false + diffutils: + externals: + - spec: diffutils@3.12 + prefix: /usr + buildable: false + git: + externals: + - spec: git@2.53.0 + prefix: /usr + buildable: false + gmake: + externals: + - spec: gmake@4.4.1 + prefix: /usr + buildable: false + m4: + externals: + - spec: m4@1.4.21 + prefix: /usr + buildable: false + perl: + externals: + - spec: perl@5.40.1 + prefix: /usr + buildable: false + tar: + externals: + - spec: tar@1.35 + prefix: /usr + buildable: false gcc: externals: - - spec: {{ compiler }} + - spec: {{ compiler }} languages='c,c++,fortran' prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran buildable: false openmpi: externals: - - spec: {{ mpi }}%{{ compiler }} - prefix: /usr - buildable: false - config: - install_missing_compilers: false - compilers: - - compiler: - spec: {{ compiler }} - paths: - cc: /usr/bin/gcc - cxx: /usr/bin/g++ - f77: /usr/bin/gfortran - fc: /usr/bin/gfortran - flags: {} - operating_system: ubuntu24.04 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] + - spec: {{ mpi }} + prefix: /usr + # mache builds the environment under `env -i`, and the Open MPI 5 + # compiler wrappers fail when HOME is unset + extra_attributes: + environment: + set: + HOME: /home/xylar + buildable: false diff --git a/docs/developers_guide/deploying_spack.rst b/docs/developers_guide/deploying_spack.rst index 8a34971a4a..9cebe9452f 100644 --- a/docs/developers_guide/deploying_spack.rst +++ b/docs/developers_guide/deploying_spack.rst @@ -99,6 +99,13 @@ Useful flags If you need a non-default temporary directory for Spack builds, set ``spack.tmpdir`` in ``deploy/config.yaml.j2`` before running deployment. +Where login nodes limit each user's memory, set ``spack_build_jobs`` in the +``[deploy]`` section of the machine config file to limit the number of +parallel Spack build jobs. It is ignored inside a Slurm job, where the +deployment has a whole node's memory. Perlmutter uses 8 on pm-cpu and 4 on +pm-gpu, where Albany's CUDA build needs more memory (see +:ref:`dev_deploying_spack_albany`). + Testing against a mache branch ------------------------------ @@ -115,6 +122,8 @@ and branch explicitly: This keeps the Compass deployment workflow the same while swapping in the requested ``mache`` branch during bootstrap and deployment. +.. _dev_deploying_spack_albany: + Albany deployments ------------------ @@ -130,6 +139,23 @@ If you are testing MALI workflows that require Albany, add Albany support is restricted to the machine/compiler/MPI combinations allowed by the deployment hooks and machine configuration. +.. important:: + + On Perlmutter, deploy Albany-enabled Spack environments for ``pm-gpu`` + from a compute node. Login nodes cap each user's memory at 30 GiB, and + Albany's CUDA build only fits with 4 parallel build jobs, which takes + hours. An interactive allocation usually starts within minutes. Point + pixi's cache at ``$SCRATCH`` first, since pixi cannot lock files in its + default cache in your home directory from a compute node: + + .. code-block:: bash + + export PIXI_CACHE_DIR=${SCRATCH}/pixi-cache + salloc -A e3sm -q interactive -C cpu -N 1 -t 4:00:00 \ + srun -N 1 -n 1 -c 128 ./deploy.py --machine pm-gpu \ + --compiler gnugpu --mpi mpich --with-albany --deploy-spack \ + --spack-path ${SCRATCH_SPACK} + Validating the deployment =========================