From 63d1992fa026bc75a806142b691dd61f3d6f5011 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 11 Sep 2026 17:45:11 -0600 Subject: [PATCH 1/6] EAMxx: rename oneapi->intel for ghci-snl --- .github/workflows/eamxx-sa-testing.yml | 12 ++++++------ components/eamxx/cmake/machine-files/aurora.cmake | 4 ++-- .../{ghci-snl-oneapi.cmake => ghci-snl-intel.cmake} | 2 +- components/eamxx/scripts/machines_specs.py | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) rename components/eamxx/cmake/machine-files/{ghci-snl-oneapi.cmake => ghci-snl-intel.cmake} (93%) diff --git a/.github/workflows/eamxx-sa-testing.yml b/.github/workflows/eamxx-sa-testing.yml index ad5f7daa25b7..54ddce671971 100644 --- a/.github/workflows/eamxx-sa-testing.yml +++ b/.github/workflows/eamxx-sa-testing.yml @@ -33,7 +33,7 @@ on: options: - gnu-openmp - gnu-cuda - - oneapi-openmp + - intel-openmp - all bless: description: 'Generate baselines' @@ -108,21 +108,21 @@ jobs: cmake-configs: Kokkos_ENABLE_OPENMP=ON ekat: ${{ env.ekat }} kokkos_develop: ${{ env.kokkos_dev_test }} - oneapi-openmp: + intel-openmp: if: | ${{ github.event_name != 'workflow_dispatch' || ( - github.event.inputs.job_to_run == 'oneapi-openmp' || + github.event.inputs.job_to_run == 'intel-openmp' || github.event.inputs.job_to_run == 'all' ) }} - runs-on: [self-hosted, ghci-snl, oneapi] + runs-on: [self-hosted, ghci-snl, intel] strategy: fail-fast: false matrix: build_type: [opt] - name: oneapi-openmp / ${{ matrix.build_type }} + name: intel-openmp / ${{ matrix.build_type }} steps: - name: Check out the repository uses: actions/checkout@v7 @@ -136,7 +136,7 @@ jobs: uses: ./.github/actions/test-all-eamxx with: build_type: ${{ matrix.build_type }} - machine: ghci-snl-oneapi + machine: ghci-snl-intel generate: ${{ env.generate }} submit: ${{ env.submit }} cmake-configs: Kokkos_ENABLE_OPENMP=ON diff --git a/components/eamxx/cmake/machine-files/aurora.cmake b/components/eamxx/cmake/machine-files/aurora.cmake index 015f6304a8ed..58d906d918d0 100644 --- a/components/eamxx/cmake/machine-files/aurora.cmake +++ b/components/eamxx/cmake/machine-files/aurora.cmake @@ -10,11 +10,11 @@ set(EKAT_MPI_THREAD_FLAG "-d" CACHE STRING "") SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") SET(SYCL_LINK_FLAGS "-fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device pvc\" -flink-huge-device-code") -if (COMPILER MATCHES ".*gpu.*") # oneapi-ifxgpu +if (COMPILER MATCHES ".*gpu.*") # intelgpu include (${EKAT_MACH_FILES_PATH}/kokkos/intel-pvc.cmake) set(CMAKE_CXX_FLAGS " -mlong-double-64 ${SYCL_COMPILE_FLAGS}" CACHE STRING "" FORCE) set(CMAKE_EXE_LINKER_FLAGS " -lifcore -mlong-double-64 ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "" FORCE) -else() # oneapi-ifx +else() # intel include (${EKAT_MACH_FILES_PATH}/kokkos/serial.cmake) set(CMAKE_CXX_FLAGS " -mlong-double-64" CACHE STRING "" FORCE) set(CMAKE_EXE_LINKER_FLAGS " -lifcore -mlong-double-64 -fortlib" CACHE STRING "" FORCE) diff --git a/components/eamxx/cmake/machine-files/ghci-snl-oneapi.cmake b/components/eamxx/cmake/machine-files/ghci-snl-intel.cmake similarity index 93% rename from components/eamxx/cmake/machine-files/ghci-snl-oneapi.cmake rename to components/eamxx/cmake/machine-files/ghci-snl-intel.cmake index 4c5fe99294ec..8b382a865520 100644 --- a/components/eamxx/cmake/machine-files/ghci-snl-oneapi.cmake +++ b/components/eamxx/cmake/machine-files/ghci-snl-intel.cmake @@ -2,7 +2,7 @@ include(${CMAKE_CURRENT_LIST_DIR}/ghci-snl.cmake) # Set SCREAM_MACHINE -set(SCREAM_MACHINE ghci-snl-oneapi CACHE STRING "") +set(SCREAM_MACHINE ghci-snl-intel CACHE STRING "") # We use MKL instead of blas here option(HOMME_USE_MKL "Whether to use Intel's MKL/oneMKL instead of blas/lapack" ON) diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index 3ae257548fdf..0d1465c0fd4f 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -350,13 +350,13 @@ def setup(cls): cls.env_setup = ["export GATOR_INITIAL_MB=4000MB"] ############################################################################### -class GHCISNLOneAPI(Machine): +class GHCISNLIntel(Machine): ############################################################################### concrete = True @classmethod def setup(cls): - super().setup_base("ghci-snl-oneapi") - cls.baselines_dir = "/projects/e3sm/data/baselines/scream/ghci-snl-oneapi" + super().setup_base("ghci-snl-intel") + cls.baselines_dir = "/projects/e3sm/data/baselines/scream/ghci-snl-intel" cls.env_setup = ["export GATOR_INITIAL_MB=4000MB", "export OMPI_MCA_io=romio321"] From 8191ba7a7924867349feb157b354dbc46ab4e03c Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Fri, 11 Sep 2026 17:45:27 -0600 Subject: [PATCH 2/6] EAMxx: nano-fix in ghci-snl-cuda eamxx mach file --- components/eamxx/cmake/machine-files/ghci-snl-cuda.cmake | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/eamxx/cmake/machine-files/ghci-snl-cuda.cmake b/components/eamxx/cmake/machine-files/ghci-snl-cuda.cmake index 42e7d97a1900..bf8a40d7d528 100644 --- a/components/eamxx/cmake/machine-files/ghci-snl-cuda.cmake +++ b/components/eamxx/cmake/machine-files/ghci-snl-cuda.cmake @@ -11,12 +11,14 @@ set(LAPACK_LIBRARIES "$ENV{BLAS_ROOT}/lib64/liblapack.so" CACHE STRING "Path to # Set SCREAM_MACHINE set(SCREAM_MACHINE ghci-snl-cuda CACHE STRING "") +# Set the path to BLAS/LAPACK libraries +set(BLAS_LIBRARIES "$ENV{BLAS_ROOT}/lib64/libblas.so" CACHE STRING "Path to BLAS library" FORCE) +set(LAPACK_LIBRARIES "$ENV{BLAS_ROOT}/lib64/liblapack.so" CACHE STRING "Path to LAPACK library" FORCE) + # Enable CUDA in kokkos set (EKAT_MACH_FILES_PATH ${CMAKE_CURRENT_LIST_DIR}/../../../../externals/ekat/cmake/machine-files) include (${EKAT_MACH_FILES_PATH}/kokkos/cuda.cmake) -set(EKAT_MPI_NP_FLAG "-n" CACHE STRING "The mpirun flag for designating the total number of ranks") - # TODO: rebuild cuda image with cuda-aware MPI, so we can set this to ON option(SCREAM_MPI_ON_DEVICE "Whether to use device pointers for MPI calls" OFF) From b96c75f721899abf2c4ce9bd980500aa3d59b8e9 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 18 Aug 2026 10:06:56 -0600 Subject: [PATCH 3/6] Machines: expand congig files for ghci-snl machine --- cime_config/machines/cmake_macros/ghci-snl_gnugpu.cmake | 1 + cime_config/machines/cmake_macros/ghci-snl_intel.cmake | 4 ++++ cime_config/machines/config_machines.xml | 5 +++-- 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 cime_config/machines/cmake_macros/ghci-snl_gnugpu.cmake create mode 100644 cime_config/machines/cmake_macros/ghci-snl_intel.cmake diff --git a/cime_config/machines/cmake_macros/ghci-snl_gnugpu.cmake b/cime_config/machines/cmake_macros/ghci-snl_gnugpu.cmake new file mode 100644 index 000000000000..38b4541abe2f --- /dev/null +++ b/cime_config/machines/cmake_macros/ghci-snl_gnugpu.cmake @@ -0,0 +1 @@ +string (APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_HOPPER90=ON -DKokkos_ENABLE_CUDA_LAMBDA=ON") diff --git a/cime_config/machines/cmake_macros/ghci-snl_intel.cmake b/cime_config/machines/cmake_macros/ghci-snl_intel.cmake new file mode 100644 index 000000000000..d0be6c3afd1c --- /dev/null +++ b/cime_config/machines/cmake_macros/ghci-snl_intel.cmake @@ -0,0 +1,4 @@ +string (APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_OPENMP=ON -DKokkos_ARCH_SPR=ON") + +# Help find_package to find the correct BLAS (i.e., the MKL installations) +set (BLA_VENDOR "Intel10_64lp" CACHE STRING "") diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index bb0563be1255..9d3999f02e39 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -2148,8 +2148,8 @@ ^[a-fA-F0-9]{12}$ LINUX proxy.sandia.gov:80 - gnu - openmpi + gnu,intel,gnugpu + openmpi,mpich /tmp /projects/e3sm/data/inputdata /projects/e3sm/data/inputdata/atm/datm7 @@ -2162,6 +2162,7 @@ lbertag at sandia dot gov 32 32 + 2 mpirun From 796696a6675c11f74ace603adbe4c53f3b362c8f Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Tue, 18 Aug 2026 10:07:38 -0600 Subject: [PATCH 4/6] Workflows: add eamxx-v1 testing with CUDA and intel compilers --- .github/workflows/eamxx-v1-testing.yml | 102 +++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/.github/workflows/eamxx-v1-testing.yml b/.github/workflows/eamxx-v1-testing.yml index 49d32feaddc1..084c34af43e2 100644 --- a/.github/workflows/eamxx-v1-testing.yml +++ b/.github/workflows/eamxx-v1-testing.yml @@ -31,6 +31,8 @@ on: type: choice options: - gnu + - intel + - gnugpu - all bless: description: 'Generate baselines' @@ -119,3 +121,103 @@ jobs: path: | /tmp/${{ matrix.test.full_name }}*/run/**/*.nc* retention-days: 14 + intel: + if: | + ${{ + github.event_name != 'workflow_dispatch' || + ( + github.event.inputs.job_to_run == 'intel' || + github.event.inputs.job_to_run == 'all' + ) + }} + runs-on: [self-hosted, intel, ghci-snl] + strategy: + matrix: + test: + - full_name: ERS.ne4pg2_ne4pg2.F2010-SCREAMv1.ghci-snl_intel.eamxx-prod + short_name: ERS.ne4pg2_ne4pg2.F2010-SCREAMv1.eamxx-prod + fail-fast: false + name: intel / ${{ matrix.test.short_name }} + steps: + - name: Check out the repository + uses: actions/checkout@v7 + with: + persist-credentials: false + show-progress: false + submodules: recursive + - name: Show action trigger + uses: ./.github/actions/show-workflow-trigger + - name: Run test + id: run-tests + run: | + ./cime/scripts/create_test ${{ matrix.test.full_name }} ${{ env.flags }} --wait + - name: Upload log files + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: logs.${{ matrix.test.short_name }} + path: | + /tmp/${{ matrix.test.full_name }}*/TestStatus + /tmp/${{ matrix.test.full_name }}*/TestStatus.log + /tmp/${{ matrix.test.full_name }}*/bld/**/*.bldlog.* + /tmp/${{ matrix.test.full_name }}*/run/**/*.log.* + /tmp/${{ matrix.test.full_name }}*/run/**/*.cprnc.out + retention-days: 14 + - name: Upload nc files + if: ${{ failure() && steps.run-tests.outcome == 'failure' }} + uses: actions/upload-artifact@v7 + with: + name: outputs.${{ matrix.test.short_name }} + path: | + /tmp/${{ matrix.test.full_name }}*/run/**/*.nc* + retention-days: 14 + gnugpu: + if: | + ${{ + github.event_name != 'workflow_dispatch' || + ( + github.event.inputs.job_to_run == 'gnugpu' || + github.event.inputs.job_to_run == 'all' + ) + }} + runs-on: [self-hosted, cuda, ghci-snl] + strategy: + matrix: + test: + - full_name: ERS.ne4pg2_ne4pg2.F2010-SCREAMv1.ghci-snl_gnugpu.eamxx-prod + short_name: ERS.ne4pg2_ne4pg2.F2010-SCREAMv1.eamxx-prod + fail-fast: false + name: gnugpu / ${{ matrix.test.short_name }} + steps: + - name: Check out the repository + uses: actions/checkout@v7 + with: + persist-credentials: false + show-progress: false + submodules: recursive + - name: Show action trigger + uses: ./.github/actions/show-workflow-trigger + - name: Run test + id: run-tests + run: | + ./cime/scripts/create_test ${{ matrix.test.full_name }} ${{ env.flags }} --wait + - name: Upload log files + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: logs.${{ matrix.test.short_name }} + path: | + /tmp/${{ matrix.test.full_name }}*/TestStatus + /tmp/${{ matrix.test.full_name }}*/TestStatus.log + /tmp/${{ matrix.test.full_name }}*/bld/**/*.bldlog.* + /tmp/${{ matrix.test.full_name }}*/run/**/*.log.* + /tmp/${{ matrix.test.full_name }}*/run/**/*.cprnc.out + retention-days: 14 + - name: Upload nc files + if: ${{ failure() && steps.run-tests.outcome == 'failure' }} + uses: actions/upload-artifact@v7 + with: + name: outputs.${{ matrix.test.short_name }} + path: | + /tmp/${{ matrix.test.full_name }}*/run/**/*.nc* + retention-days: 14 From 011914999820e90536228abe74e445cf66f30fd3 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Mon, 14 Sep 2026 14:32:36 -0600 Subject: [PATCH 5/6] Machines: fix fp-model flag for intel OneAPI The value 'source' is no longer supported in CXX, although it was kept for F90 for bwd compatibility. The new correct value is 'precise'. --- cime_config/machines/cmake_macros/intel.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/machines/cmake_macros/intel.cmake b/cime_config/machines/cmake_macros/intel.cmake index 3e66acb58671..9e1ab508df40 100644 --- a/cime_config/machines/cmake_macros/intel.cmake +++ b/cime_config/machines/cmake_macros/intel.cmake @@ -11,12 +11,12 @@ string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2 -debug minimal") string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g") string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g") string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -init=snan,arrays") -string(APPEND CMAKE_CXX_FLAGS " -fp-model source") +string(APPEND CMAKE_CXX_FLAGS " -fp-model precise") if (COMP_NAME STREQUAL cice) string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -init=nosnan,arrays") endif() string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRINTEL") -string(APPEND CMAKE_Fortran_FLAGS " -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model source") +string(APPEND CMAKE_Fortran_FLAGS " -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model precise") string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -fixed -132") string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -free") set(HAS_F2008_CONTIGUOUS "TRUE") From e4e60d0e3c8d417a979a53a9e9c890aec4149690 Mon Sep 17 00:00:00 2001 From: Luca Bertagna Date: Thu, 17 Sep 2026 13:57:50 -0600 Subject: [PATCH 6/6] Machines: disable cuda-aware MPI for ghci-snl_gnugpu --- cime_config/machines/cmake_macros/ghci-snl_gnugpu.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cime_config/machines/cmake_macros/ghci-snl_gnugpu.cmake b/cime_config/machines/cmake_macros/ghci-snl_gnugpu.cmake index 38b4541abe2f..20156c46a328 100644 --- a/cime_config/machines/cmake_macros/ghci-snl_gnugpu.cmake +++ b/cime_config/machines/cmake_macros/ghci-snl_gnugpu.cmake @@ -1 +1,4 @@ string (APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_HOPPER90=ON -DKokkos_ENABLE_CUDA_LAMBDA=ON") + +# Change to ON if/when our runners feature cuda-aware MPI +set(SCREAM_MPI_ON_DEVICE OFF CACHE STRING "")