From 77795df769a8573c47f24186ed437dda069c06a5 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 23 Oct 2026 08:01:10 +0200 Subject: [PATCH 01/12] Update to v2.1.0-alpha.1 --- compass/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compass/version.py b/compass/version.py index fa98dd1c47..168a408c50 100644 --- a/compass/version.py +++ b/compass/version.py @@ -1 +1 @@ -__version__ = '2.0.0-alpha.3' +__version__ = '2.1.0-alpha.1' From b88a0ba039ed8e3fbe272aa502901bbfa7ca836d Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 23 Oct 2026 08:01:32 +0200 Subject: [PATCH 02/12] Update to mache v5.2.0 --- deploy/cli_spec.json | 8 +++++++- deploy/pins.cfg | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/deploy/cli_spec.json b/deploy/cli_spec.json index 544058c09c..38446dc507 100644 --- a/deploy/cli_spec.json +++ b/deploy/cli_spec.json @@ -1,7 +1,7 @@ { "meta": { "software": "compass", - "mache_version": "3.7.1", + "mache_version": "5.2.0", "description": "Deploy compass environment" }, "arguments": [ @@ -57,6 +57,12 @@ "help": "Path to the Spack checkout to use (overrides spack.spack_path in deploy/config.yaml.j2).", "route": ["deploy", "run"] }, + { + "flags": ["--spack-pins"], + "dest": "spack_pins", + "help": "YAML file overriding the pinned Spack, spack-packages and e3sm-spack-packages sources (a tag, commit or branch per repository), for testing unreleased recipes.", + "route": ["deploy", "run"] + }, { "flags": ["--recreate"], "dest": "recreate", diff --git a/deploy/pins.cfg b/deploy/pins.cfg index 8343386311..a3c4169271 100644 --- a/deploy/pins.cfg +++ b/deploy/pins.cfg @@ -4,7 +4,7 @@ bootstrap_python = 3.14 python = 3.14 esmf = 8.9.1 geometric_features = 1.6.3 -mache = 3.7.1 +mache = 5.2.0 mpas_tools = 1.5.1 otps = 2021.10 parallelio = 2.6.9 From 77b7b426ce738c753beb09fdfda0fdef97a613a2 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 23 Oct 2026 08:01:51 +0200 Subject: [PATCH 03/12] Update katara spack template for spack 1.x --- deploy/spack/katara_gnu_openmpi.yaml | 96 ++++++++++++++++++++-------- 1 file changed, 69 insertions(+), 27 deletions(-) diff --git a/deploy/spack/katara_gnu_openmpi.yaml b/deploy/spack/katara_gnu_openmpi.yaml index 2a99a0a046..c1977a8cff 100644 --- a/deploy/spack/katara_gnu_openmpi.yaml +++ b/deploy/spack/katara_gnu_openmpi.yaml @@ -1,42 +1,84 @@ -{%- set compiler = "gcc@13.3.0" %} -{%- set mpi = "openmpi@4.1.6" %} +{%- set compiler = "gcc@15.2.0" %} +{%- set mpi = "openmpi@5.0.10" %} spack: specs: - - {{ compiler }} - - {{ mpi }}%{{ compiler }} + - {{ mpi }} {%- for spec in specs %} - - "{{ spec }}%{{ compiler }}" + - "{{ spec }}" {%- endfor %} concretizer: unify: true + toolchains: + mache: + - spec: "%c={{ compiler }}" + when: "%c" + - spec: "%cxx={{ compiler }}" + when: "%cxx" + - spec: "%fortran={{ compiler }}" + when: "%fortran" packages: all: - compiler: [{{ compiler }}] + require: ["%mache"] providers: - mpi: [{{ mpi }}%{{ compiler }}] + mpi: [{{ mpi }}] + autoconf: + externals: + - spec: autoconf@2.72 + prefix: /usr + buildable: false + automake: + externals: + - spec: automake@1.18.1 + prefix: /usr + buildable: false + diffutils: + externals: + - spec: diffutils@3.12 + prefix: /usr + buildable: false + git: + externals: + - spec: git@2.53.0 + prefix: /usr + buildable: false + gmake: + externals: + - spec: gmake@4.4.1 + prefix: /usr + buildable: false + m4: + externals: + - spec: m4@1.4.21 + prefix: /usr + buildable: false + perl: + externals: + - spec: perl@5.40.1 + prefix: /usr + buildable: false + tar: + externals: + - spec: tar@1.35 + prefix: /usr + buildable: false gcc: externals: - - spec: {{ compiler }} + - spec: {{ compiler }} languages='c,c++,fortran' prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran buildable: false openmpi: externals: - - spec: {{ mpi }}%{{ compiler }} - prefix: /usr - buildable: false - config: - install_missing_compilers: false - compilers: - - compiler: - spec: {{ compiler }} - paths: - cc: /usr/bin/gcc - cxx: /usr/bin/g++ - f77: /usr/bin/gfortran - fc: /usr/bin/gfortran - flags: {} - operating_system: ubuntu24.04 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] + - spec: {{ mpi }} + prefix: /usr + # mache builds the environment under `env -i`, and the Open MPI 5 + # compiler wrappers fail when HOME is unset + extra_attributes: + environment: + set: + HOME: /home/xylar + buildable: false From ea797b3d4026d60f0ab31d0d457ef733e85527ee Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 23 Oct 2026 08:06:13 +0200 Subject: [PATCH 04/12] Update dependency versions --- deploy/pins.cfg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/pins.cfg b/deploy/pins.cfg index a3c4169271..1dfb3e3ffa 100644 --- a/deploy/pins.cfg +++ b/deploy/pins.cfg @@ -5,7 +5,7 @@ python = 3.14 esmf = 8.9.1 geometric_features = 1.6.3 mache = 5.2.0 -mpas_tools = 1.5.1 +mpas_tools = 2.1.0 otps = 2021.10 parallelio = 2.6.9 @@ -24,6 +24,6 @@ trilinos_variants = # pins for both pixi and spack environments [all] -netcdf_c = 4.10.0 -netcdf_fortran = 4.6.2 -pnetcdf = 1.14.1 +netcdf_c = 4.10.1 +netcdf_fortran = 4.6.3 +pnetcdf = 1.15.0 From 329cc9443c05a6e2603932a891e9b74194408dce Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 23 Oct 2026 09:29:30 +0200 Subject: [PATCH 05/12] Use the pinned mache in CI This reverts commit 0a8c5dadf, which pointed CI at an old mache branch whose jigsaw build pins libnetcdf 4.10.0, conflicting with the 4.10.1 pin in deploy/pins.cfg. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/build_workflow.yml | 1 - .github/workflows/gis_coarse_meshgen.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/build_workflow.yml b/.github/workflows/build_workflow.yml index cafc30c184..63abb8d4d0 100644 --- a/.github/workflows/build_workflow.yml +++ b/.github/workflows/build_workflow.yml @@ -95,7 +95,6 @@ jobs: run: | git config --global url."https://github.com/".insteadOf "git@github.com:" ./deploy.py \ - --mache-fork xylar/mache --mache-branch add-target-software-cli-flags \ --no-spack \ --python ${{ matrix.python-version }} \ --recreate diff --git a/.github/workflows/gis_coarse_meshgen.yml b/.github/workflows/gis_coarse_meshgen.yml index 17dca534f5..5fea8a2e27 100644 --- a/.github/workflows/gis_coarse_meshgen.yml +++ b/.github/workflows/gis_coarse_meshgen.yml @@ -41,7 +41,6 @@ jobs: run: | git config --global url."https://github.com/".insteadOf "git@github.com:" ./deploy.py \ - --mache-fork xylar/mache --mache-branch add-target-software-cli-flags \ --no-spack \ --python 3.14 \ --recreate From d7f8fb34c0ba25ebf0539d6a34d71ac889d65abd Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 23 Oct 2026 09:37:42 +0200 Subject: [PATCH 06/12] Print the GIS mesh_gen log when it fails in CI Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/gis_coarse_meshgen.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gis_coarse_meshgen.yml b/.github/workflows/gis_coarse_meshgen.yml index 5fea8a2e27..98d0f2055f 100644 --- a/.github/workflows/gis_coarse_meshgen.yml +++ b/.github/workflows/gis_coarse_meshgen.yml @@ -95,6 +95,9 @@ jobs: testDir=$root/gis_mesh_gen compass setup -w $testDir -t landice/greenland/mesh_gen -f system.cfg - # run the test case + # run the test case, printing its log if it fails cd $testDir - compass run + if ! compass run; then + cat case_outputs/*.log + exit 1 + fi From 6a64b80f34d148d374c4b5b9d8e28e4fa75e83f6 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 23 Oct 2026 10:11:59 +0200 Subject: [PATCH 07/12] Keep MPI off RDMA transports in the GIS mesh CI Some GitHub runners have a Microsoft Azure Network Adapter whose RDMA device UCX cannot open, so MPI_Init fails in ESMF_RegridWeightGen on those runners only. Limit UCX to self, shared-memory and TCP transports. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/gis_coarse_meshgen.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/gis_coarse_meshgen.yml b/.github/workflows/gis_coarse_meshgen.yml index 98d0f2055f..88eff34223 100644 --- a/.github/workflows/gis_coarse_meshgen.yml +++ b/.github/workflows/gis_coarse_meshgen.yml @@ -95,6 +95,10 @@ jobs: testDir=$root/gis_mesh_gen compass setup -w $testDir -t landice/greenland/mesh_gen -f system.cfg + # Some runners have an Azure (MANA) RDMA device that UCX cannot + # open, which makes MPI_Init fail, so stick to non-RDMA transports + export UCX_TLS=self,sm,tcp + # run the test case, printing its log if it fails cd $testDir if ! compass run; then From 99bb9daad72693f00241cf2e1d63c8b4f4976986 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 23 Oct 2026 11:52:15 +0200 Subject: [PATCH 08/12] Limit Spack build jobs on Perlmutter Perlmutter login nodes cap each user's memory at 30 GiB, and Albany's build was OOM-killed with Spack's default of 16 jobs. A new spack_build_jobs option in a machine config's [deploy] section sets spack.build_jobs through the pre_spack hook; Perlmutter uses 8. Co-Authored-By: Claude Opus 5.5 (1M context) --- compass/machines/pm-cpu.cfg | 4 ++++ compass/machines/pm-gpu.cfg | 4 ++++ deploy/hooks.py | 14 ++++++++++++++ docs/developers_guide/deploying_spack.rst | 4 ++++ 4 files changed, 26 insertions(+) diff --git a/compass/machines/pm-cpu.cfg b/compass/machines/pm-cpu.cfg index e6c26be01d..be5cc3c74d 100644 --- a/compass/machines/pm-cpu.cfg +++ b/compass/machines/pm-cpu.cfg @@ -36,6 +36,10 @@ spack = /global/cfs/cdirs/e3sm/software/compass/pm-cpu/spack # pnetcdf as E3SM (spack modules are used otherwise) use_e3sm_hdf5_netcdf = True +# the number of parallel jobs for spack builds, limited because login nodes +# cap each user's memory at 30 GiB, which Albany exceeds with 16 jobs +spack_build_jobs = 8 + # The parallel section describes options related to running jobs in parallel. # Most options in this section come from mache so here we just add or override # some defaults diff --git a/compass/machines/pm-gpu.cfg b/compass/machines/pm-gpu.cfg index 3ea5f74768..0312de195c 100644 --- a/compass/machines/pm-gpu.cfg +++ b/compass/machines/pm-gpu.cfg @@ -39,6 +39,10 @@ spack = /global/cfs/cdirs/e3sm/software/compass/pm-gpu/spack # pnetcdf as E3SM (spack modules are used otherwise) use_e3sm_hdf5_netcdf = True +# the number of parallel jobs for spack builds, limited because login nodes +# cap each user's memory at 30 GiB, which Albany exceeds with 16 jobs +spack_build_jobs = 8 + # spack variants for Albany and Trilinos albany_variants = +mpas~py+unit_tests+cuda+sfad sfadsize=12 trilinos_variants = +cuda+ampere80+zen3 diff --git a/deploy/hooks.py b/deploy/hooks.py index df4c30b743..395bc74b7b 100644 --- a/deploy/hooks.py +++ b/deploy/hooks.py @@ -41,6 +41,9 @@ def pre_spack(ctx: DeployContext) -> dict[str, Any] | None: updates['spack'] = {'spack_path': spack_path} if exclude_packages: updates.setdefault('spack', {})['exclude_packages'] = exclude_packages + build_jobs = _get_spack_build_jobs(ctx.machine_config) + if build_jobs is not None: + updates.setdefault('spack', {})['build_jobs'] = build_jobs if _with_albany(ctx): _check_albany_support(ctx.machine, toolchain_pairs) @@ -262,6 +265,17 @@ def _maybe_exclude_e3sm_hdf5_netcdf( exclude_packages.append('hdf5_netcdf') +def _get_spack_build_jobs(machine_config) -> int | None: + if not machine_config.has_section('deploy'): + return None + value = machine_config.get( + 'deploy', 'spack_build_jobs', fallback='' + ).strip() + if not value: + return None + return int(value) + + def _maybe_exclude_cmake( exclude_packages: list[str], machine_config ) -> None: diff --git a/docs/developers_guide/deploying_spack.rst b/docs/developers_guide/deploying_spack.rst index 8a34971a4a..cc5395609c 100644 --- a/docs/developers_guide/deploying_spack.rst +++ b/docs/developers_guide/deploying_spack.rst @@ -99,6 +99,10 @@ Useful flags If you need a non-default temporary directory for Spack builds, set ``spack.tmpdir`` in ``deploy/config.yaml.j2`` before running deployment. +Where login nodes limit each user's memory, set ``spack_build_jobs`` in the +``[deploy]`` section of the machine config file to limit the number of +parallel Spack build jobs. Perlmutter uses 8. + Testing against a mache branch ------------------------------ From fc147fd21a0a1d67245aa74bc11b6a0d99c94e9f Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 23 Oct 2026 15:09:43 +0200 Subject: [PATCH 09/12] Use 4 Spack build jobs on pm-gpu Albany's CUDA build was still OOM-killed on a Perlmutter login node with 8 jobs. Co-Authored-By: Claude Opus 5.5 (1M context) --- compass/machines/pm-gpu.cfg | 5 +++-- docs/developers_guide/deploying_spack.rst | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/compass/machines/pm-gpu.cfg b/compass/machines/pm-gpu.cfg index 0312de195c..4db26d5692 100644 --- a/compass/machines/pm-gpu.cfg +++ b/compass/machines/pm-gpu.cfg @@ -40,8 +40,9 @@ spack = /global/cfs/cdirs/e3sm/software/compass/pm-gpu/spack use_e3sm_hdf5_netcdf = True # the number of parallel jobs for spack builds, limited because login nodes -# cap each user's memory at 30 GiB, which Albany exceeds with 16 jobs -spack_build_jobs = 8 +# cap each user's memory at 30 GiB, which Albany's CUDA build exceeds with 8 +# jobs +spack_build_jobs = 4 # spack variants for Albany and Trilinos albany_variants = +mpas~py+unit_tests+cuda+sfad sfadsize=12 diff --git a/docs/developers_guide/deploying_spack.rst b/docs/developers_guide/deploying_spack.rst index cc5395609c..6a9d2f2eda 100644 --- a/docs/developers_guide/deploying_spack.rst +++ b/docs/developers_guide/deploying_spack.rst @@ -101,7 +101,8 @@ If you need a non-default temporary directory for Spack builds, set Where login nodes limit each user's memory, set ``spack_build_jobs`` in the ``[deploy]`` section of the machine config file to limit the number of -parallel Spack build jobs. Perlmutter uses 8. +parallel Spack build jobs. Perlmutter uses 8 on pm-cpu and 4 on pm-gpu, +where Albany's CUDA build needs more memory. Testing against a mache branch ------------------------------ From 3058dc8884b9113f9455a5f86e37741f380f7bb3 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 23 Oct 2026 15:25:55 +0200 Subject: [PATCH 10/12] Only limit Spack build jobs on login nodes Inside a Slurm job the deployment has a whole node's memory, so spack_build_jobs no longer applies there. The Spack deployment guide now says to deploy pm-gpu's Albany environment from a compute node, since its CUDA build only fits on a login node with 4 jobs. Co-Authored-By: Claude Opus 5.5 (1M context) --- compass/machines/pm-cpu.cfg | 5 +++-- compass/machines/pm-gpu.cfg | 6 +++--- deploy/hooks.py | 3 +++ docs/developers_guide/deploying_spack.rst | 22 ++++++++++++++++++++-- 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/compass/machines/pm-cpu.cfg b/compass/machines/pm-cpu.cfg index be5cc3c74d..5cecd4395c 100644 --- a/compass/machines/pm-cpu.cfg +++ b/compass/machines/pm-cpu.cfg @@ -36,8 +36,9 @@ spack = /global/cfs/cdirs/e3sm/software/compass/pm-cpu/spack # pnetcdf as E3SM (spack modules are used otherwise) use_e3sm_hdf5_netcdf = True -# the number of parallel jobs for spack builds, limited because login nodes -# cap each user's memory at 30 GiB, which Albany exceeds with 16 jobs +# the number of parallel jobs for spack builds on login nodes, which cap each +# user's memory at 30 GiB (Albany exceeds that with 16 jobs); ignored inside +# a Slurm job spack_build_jobs = 8 # The parallel section describes options related to running jobs in parallel. diff --git a/compass/machines/pm-gpu.cfg b/compass/machines/pm-gpu.cfg index 4db26d5692..9a47b32aa6 100644 --- a/compass/machines/pm-gpu.cfg +++ b/compass/machines/pm-gpu.cfg @@ -39,9 +39,9 @@ spack = /global/cfs/cdirs/e3sm/software/compass/pm-gpu/spack # pnetcdf as E3SM (spack modules are used otherwise) use_e3sm_hdf5_netcdf = True -# the number of parallel jobs for spack builds, limited because login nodes -# cap each user's memory at 30 GiB, which Albany's CUDA build exceeds with 8 -# jobs +# the number of parallel jobs for spack builds on login nodes, which cap each +# user's memory at 30 GiB (Albany's CUDA build exceeds that with 8 jobs); +# ignored inside a Slurm job spack_build_jobs = 4 # spack variants for Albany and Trilinos diff --git a/deploy/hooks.py b/deploy/hooks.py index 395bc74b7b..06948eb87c 100644 --- a/deploy/hooks.py +++ b/deploy/hooks.py @@ -266,6 +266,9 @@ def _maybe_exclude_e3sm_hdf5_netcdf( def _get_spack_build_jobs(machine_config) -> int | None: + # the limit is for login nodes; a Slurm job has the node's memory + if os.environ.get('SLURM_JOB_ID'): + return None if not machine_config.has_section('deploy'): return None value = machine_config.get( diff --git a/docs/developers_guide/deploying_spack.rst b/docs/developers_guide/deploying_spack.rst index 6a9d2f2eda..2ed5fa8f60 100644 --- a/docs/developers_guide/deploying_spack.rst +++ b/docs/developers_guide/deploying_spack.rst @@ -101,8 +101,10 @@ If you need a non-default temporary directory for Spack builds, set Where login nodes limit each user's memory, set ``spack_build_jobs`` in the ``[deploy]`` section of the machine config file to limit the number of -parallel Spack build jobs. Perlmutter uses 8 on pm-cpu and 4 on pm-gpu, -where Albany's CUDA build needs more memory. +parallel Spack build jobs. It is ignored inside a Slurm job, where the +deployment has a whole node's memory. Perlmutter uses 8 on pm-cpu and 4 on +pm-gpu, where Albany's CUDA build needs more memory (see +:ref:`dev_deploying_spack_albany`). Testing against a mache branch ------------------------------ @@ -120,6 +122,8 @@ and branch explicitly: This keeps the Compass deployment workflow the same while swapping in the requested ``mache`` branch during bootstrap and deployment. +.. _dev_deploying_spack_albany: + Albany deployments ------------------ @@ -135,6 +139,20 @@ If you are testing MALI workflows that require Albany, add Albany support is restricted to the machine/compiler/MPI combinations allowed by the deployment hooks and machine configuration. +.. important:: + + On Perlmutter, deploy Albany-enabled Spack environments for ``pm-gpu`` + from a compute node. Login nodes cap each user's memory at 30 GiB, and + Albany's CUDA build only fits with 4 parallel build jobs, which takes + hours. An interactive allocation usually starts within minutes: + + .. code-block:: bash + + salloc -A e3sm -q interactive -C cpu -N 1 -t 4:00:00 \ + srun -N 1 -n 1 -c 128 ./deploy.py --machine pm-gpu \ + --compiler gnugpu --mpi mpich --with-albany --deploy-spack \ + --spack-path ${SCRATCH_SPACK} + Validating the deployment ========================= From f14883c42d8924b210dfa24dd5828f82fb34d44f Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 23 Oct 2026 15:56:28 +0200 Subject: [PATCH 11/12] Pin e3sm-spack-packages past v2026.06.2 The trilinos-for-albany and albany recipes need the Spack 1.x fixes from E3SM-Project/e3sm-spack-packages#4, which no tag or mache release includes yet. Co-Authored-By: Claude Opus 5.5 (1M context) --- deploy/config.yaml.j2 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/deploy/config.yaml.j2 b/deploy/config.yaml.j2 index 0ae885a3fa..1df2fdcc46 100644 --- a/deploy/config.yaml.j2 +++ b/deploy/config.yaml.j2 @@ -164,6 +164,15 @@ spack: # excluding `hdf5_netcdf`. exclude_packages: [] + # Overrides for the Spack sources pinned by mache (mache/spack/pins.yaml). + # e3sm-spack-packages is pinned past mache 5.2.0's v2026.06.2 for the + # trilinos-for-albany and albany fixes in E3SM-Project/e3sm-spack-packages#4, + # until a mache release pins a tag that includes them. + pins: + repos: + e3sm: + commit: 65739a6a79f45acffbf0ac24b754f4841ed2065a + jigsaw: # If true, build/install JIGSAW + JIGSAW-Python into the pixi env enabled: true From 83851f72519812e3e7d3bdac1eb0d7ec72c0ad13 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 23 Oct 2026 16:02:39 +0200 Subject: [PATCH 12/12] Say to move pixi's cache for compute-node deployments pixi cannot lock files in its default cache in the home directory from a Perlmutter compute node, so the compute-node example sets PIXI_CACHE_DIR under $SCRATCH first. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/developers_guide/deploying_spack.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/developers_guide/deploying_spack.rst b/docs/developers_guide/deploying_spack.rst index 2ed5fa8f60..9cebe9452f 100644 --- a/docs/developers_guide/deploying_spack.rst +++ b/docs/developers_guide/deploying_spack.rst @@ -144,10 +144,13 @@ by the deployment hooks and machine configuration. On Perlmutter, deploy Albany-enabled Spack environments for ``pm-gpu`` from a compute node. Login nodes cap each user's memory at 30 GiB, and Albany's CUDA build only fits with 4 parallel build jobs, which takes - hours. An interactive allocation usually starts within minutes: + hours. An interactive allocation usually starts within minutes. Point + pixi's cache at ``$SCRATCH`` first, since pixi cannot lock files in its + default cache in your home directory from a compute node: .. code-block:: bash + export PIXI_CACHE_DIR=${SCRATCH}/pixi-cache salloc -A e3sm -q interactive -C cpu -N 1 -t 4:00:00 \ srun -N 1 -n 1 -c 128 ./deploy.py --machine pm-gpu \ --compiler gnugpu --mpi mpich --with-albany --deploy-spack \