Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions .github/workflows/eamxx-sa-testing.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ on:
options:
- gnu-openmp
- gnu-cuda
- oneapi-openmp
- intel-openmp
- all
bless:
description: 'Generate baselines'
Expand Down Expand Up @@ -108,21 +108,21 @@ jobs:
cmake-configs: Kokkos_ENABLE_OPENMP=ON
ekat: ${{ env.ekat }}
kokkos_develop: ${{ env.kokkos_dev_test }}
oneapi-openmp:
intel-openmp:
if: |
${{
github.event_name != 'workflow_dispatch' ||
(
github.event.inputs.job_to_run == 'oneapi-openmp' ||
github.event.inputs.job_to_run == 'intel-openmp' ||
github.event.inputs.job_to_run == 'all'
)
}}
runs-on: [self-hosted, ghci-snl, oneapi]
runs-on: [self-hosted, ghci-snl, intel]
strategy:
fail-fast: false
matrix:
build_type: [opt]
name: oneapi-openmp / ${{ matrix.build_type }}
name: intel-openmp / ${{ matrix.build_type }}
steps:
- name: Check out the repository
uses: actions/checkout@v7
Expand All @@ -136,7 +136,7 @@ jobs:
uses: ./.github/actions/test-all-eamxx
with:
build_type: ${{ matrix.build_type }}
machine: ghci-snl-oneapi
machine: ghci-snl-intel
generate: ${{ env.generate }}
submit: ${{ env.submit }}
cmake-configs: Kokkos_ENABLE_OPENMP=ON
Expand Down
102 changes: 102 additions & 0 deletions .github/workflows/eamxx-v1-testing.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ on:
type: choice
options:
- gnu
- intel
- gnugpu
- all
bless:
description: 'Generate baselines'
Expand Down Expand Up @@ -119,3 +121,103 @@ jobs:
path: |
/tmp/${{ matrix.test.full_name }}*/run/**/*.nc*
retention-days: 14
intel:
if: |
${{
github.event_name != 'workflow_dispatch' ||
(
github.event.inputs.job_to_run == 'intel' ||
github.event.inputs.job_to_run == 'all'
)
}}
runs-on: [self-hosted, intel, ghci-snl]
strategy:
matrix:
test:
- full_name: ERS.ne4pg2_ne4pg2.F2010-SCREAMv1.ghci-snl_intel.eamxx-prod
short_name: ERS.ne4pg2_ne4pg2.F2010-SCREAMv1.eamxx-prod
fail-fast: false
name: intel / ${{ matrix.test.short_name }}
steps:
- name: Check out the repository
uses: actions/checkout@v7
with:
persist-credentials: false
show-progress: false
submodules: recursive
- name: Show action trigger
uses: ./.github/actions/show-workflow-trigger
- name: Run test
id: run-tests
run: |
./cime/scripts/create_test ${{ matrix.test.full_name }} ${{ env.flags }} --wait
- name: Upload log files
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: logs.${{ matrix.test.short_name }}
path: |
/tmp/${{ matrix.test.full_name }}*/TestStatus
/tmp/${{ matrix.test.full_name }}*/TestStatus.log
/tmp/${{ matrix.test.full_name }}*/bld/**/*.bldlog.*
/tmp/${{ matrix.test.full_name }}*/run/**/*.log.*
/tmp/${{ matrix.test.full_name }}*/run/**/*.cprnc.out
retention-days: 14
- name: Upload nc files
if: ${{ failure() && steps.run-tests.outcome == 'failure' }}
uses: actions/upload-artifact@v7
with:
name: outputs.${{ matrix.test.short_name }}
path: |
/tmp/${{ matrix.test.full_name }}*/run/**/*.nc*
retention-days: 14
gnugpu:
if: |
${{
github.event_name != 'workflow_dispatch' ||
(
github.event.inputs.job_to_run == 'gnugpu' ||
github.event.inputs.job_to_run == 'all'
)
}}
runs-on: [self-hosted, cuda, ghci-snl]
strategy:
matrix:
test:
- full_name: ERS.ne4pg2_ne4pg2.F2010-SCREAMv1.ghci-snl_gnugpu.eamxx-prod
short_name: ERS.ne4pg2_ne4pg2.F2010-SCREAMv1.eamxx-prod
fail-fast: false
name: gnugpu / ${{ matrix.test.short_name }}
steps:
- name: Check out the repository
uses: actions/checkout@v7
with:
persist-credentials: false
show-progress: false
submodules: recursive
- name: Show action trigger
uses: ./.github/actions/show-workflow-trigger
- name: Run test
id: run-tests
run: |
./cime/scripts/create_test ${{ matrix.test.full_name }} ${{ env.flags }} --wait
- name: Upload log files
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
name: logs.${{ matrix.test.short_name }}
path: |
/tmp/${{ matrix.test.full_name }}*/TestStatus
/tmp/${{ matrix.test.full_name }}*/TestStatus.log
/tmp/${{ matrix.test.full_name }}*/bld/**/*.bldlog.*
/tmp/${{ matrix.test.full_name }}*/run/**/*.log.*
/tmp/${{ matrix.test.full_name }}*/run/**/*.cprnc.out
retention-days: 14
- name: Upload nc files
if: ${{ failure() && steps.run-tests.outcome == 'failure' }}
uses: actions/upload-artifact@v7
with:
name: outputs.${{ matrix.test.short_name }}
path: |
/tmp/${{ matrix.test.full_name }}*/run/**/*.nc*
retention-days: 14
4 changes: 4 additions & 0 deletions cime_config/machines/cmake_macros/ghci-snl_gnugpu.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
string (APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_HOPPER90=ON -DKokkos_ENABLE_CUDA_LAMBDA=ON")

# Change to ON if/when our runners feature cuda-aware MPI
set(SCREAM_MPI_ON_DEVICE OFF CACHE STRING "")
4 changes: 4 additions & 0 deletions cime_config/machines/cmake_macros/ghci-snl_intel.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
string (APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_OPENMP=ON -DKokkos_ARCH_SPR=ON")

# Help find_package to find the correct BLAS (i.e., the MKL installations)
set (BLA_VENDOR "Intel10_64lp" CACHE STRING "")
4 changes: 2 additions & 2 deletions cime_config/machines/cmake_macros/intel.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@ string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2 -debug minimal")
string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g")
string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -init=snan,arrays")
string(APPEND CMAKE_CXX_FLAGS " -fp-model source")
string(APPEND CMAKE_CXX_FLAGS " -fp-model precise")

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This changed answers for EAM cases on chrysalis.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I had to add this, as -fp-model source is no longer a valid intel flag. @amametjanov has a similar mod on his PR to fix aurora.

That said, I don't think it explains anything. The file <compiler>.cmake is included by cmake before <machine_compiler>.cmake. And chrysalis_intel.cmake already has

string(APPEND CMAKE_CXX_FLAGS " -fp-model precise")

So this change ONLY affects builds that did NOT set the fp model in <machine_compiler>.cmake. And if they didn't, oneapi should have errored out. Like it did on Aurora (hence, Az's PR).

if (COMP_NAME STREQUAL cice)
string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -init=nosnan,arrays")
endif()
string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRINTEL")
string(APPEND CMAKE_Fortran_FLAGS " -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model source")
string(APPEND CMAKE_Fortran_FLAGS " -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model precise")
string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -fixed -132")
string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -free")
set(HAS_F2008_CONTIGUOUS "TRUE")
Expand Down
5 changes: 3 additions & 2 deletions cime_config/machines/config_machines.xml
Original file line number Diff line number Diff line change
Expand Up @@ -2148,8 +2148,8 @@
<NODENAME_REGEX>^[a-fA-F0-9]{12}$</NODENAME_REGEX>
<OS>LINUX</OS>
<PROXY>proxy.sandia.gov:80</PROXY>
<COMPILERS>gnu</COMPILERS>
<MPILIBS>openmpi</MPILIBS>
<COMPILERS>gnu,intel,gnugpu</COMPILERS>
<MPILIBS>openmpi,mpich</MPILIBS>
<CIME_OUTPUT_ROOT>/tmp</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>/projects/e3sm/data/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/projects/e3sm/data/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
Expand All @@ -2162,6 +2162,7 @@
<SUPPORTED_BY>lbertag at sandia dot gov</SUPPORTED_BY>
<MAX_TASKS_PER_NODE>32</MAX_TASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE>32</MAX_MPITASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE compiler="gnugpu">2</MAX_MPITASKS_PER_NODE>

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why was this PR messing with aurora?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the ghci-snl entry of config_machines though. There's one mod to aurora.cmake (not sure why), but only in comments, which should not affect anything.

<mpirun mpilib="default">
<executable>mpirun</executable>
<arguments>
Expand Down
4 changes: 2 additions & 2 deletions components/eamxx/cmake/machine-files/aurora.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@ set(EKAT_MPI_THREAD_FLAG "-d" CACHE STRING "")
SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda")
SET(SYCL_LINK_FLAGS "-fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device pvc\" -flink-huge-device-code")

if (COMPILER MATCHES ".*gpu.*") # oneapi-ifxgpu
if (COMPILER MATCHES ".*gpu.*") # intelgpu
include (${EKAT_MACH_FILES_PATH}/kokkos/intel-pvc.cmake)
set(CMAKE_CXX_FLAGS " -mlong-double-64 ${SYCL_COMPILE_FLAGS}" CACHE STRING "" FORCE)
set(CMAKE_EXE_LINKER_FLAGS " -lifcore -mlong-double-64 ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "" FORCE)
else() # oneapi-ifx
else() # intel
include (${EKAT_MACH_FILES_PATH}/kokkos/serial.cmake)
set(CMAKE_CXX_FLAGS " -mlong-double-64" CACHE STRING "" FORCE)
set(CMAKE_EXE_LINKER_FLAGS " -lifcore -mlong-double-64 -fortlib" CACHE STRING "" FORCE)
Expand Down
6 changes: 4 additions & 2 deletions components/eamxx/cmake/machine-files/ghci-snl-cuda.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,14 @@ set(LAPACK_LIBRARIES "$ENV{BLAS_ROOT}/lib64/liblapack.so" CACHE STRING "Path to
# Set SCREAM_MACHINE
set(SCREAM_MACHINE ghci-snl-cuda CACHE STRING "")

# Set the path to BLAS/LAPACK libraries
set(BLAS_LIBRARIES "$ENV{BLAS_ROOT}/lib64/libblas.so" CACHE STRING "Path to BLAS library" FORCE)
set(LAPACK_LIBRARIES "$ENV{BLAS_ROOT}/lib64/liblapack.so" CACHE STRING "Path to LAPACK library" FORCE)

# Enable CUDA in kokkos
set (EKAT_MACH_FILES_PATH ${CMAKE_CURRENT_LIST_DIR}/../../../../externals/ekat/cmake/machine-files)
include (${EKAT_MACH_FILES_PATH}/kokkos/cuda.cmake)

set(EKAT_MPI_NP_FLAG "-n" CACHE STRING "The mpirun flag for designating the total number of ranks")

# TODO: rebuild cuda image with cuda-aware MPI, so we can set this to ON
option(SCREAM_MPI_ON_DEVICE "Whether to use device pointers for MPI calls" OFF)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
include(${CMAKE_CURRENT_LIST_DIR}/ghci-snl.cmake)

# Set SCREAM_MACHINE
set(SCREAM_MACHINE ghci-snl-oneapi CACHE STRING "")
set(SCREAM_MACHINE ghci-snl-intel CACHE STRING "")

# We use MKL instead of blas here
option(HOMME_USE_MKL "Whether to use Intel's MKL/oneMKL instead of blas/lapack" ON)
Expand Down
6 changes: 3 additions & 3 deletions components/eamxx/scripts/machines_specs.py
Original file line number Diff line number Diff line change
Expand Up @@ -350,13 +350,13 @@ def setup(cls):
cls.env_setup = ["export GATOR_INITIAL_MB=4000MB"]

###############################################################################
class GHCISNLOneAPI(Machine):
class GHCISNLIntel(Machine):
###############################################################################
concrete = True
@classmethod
def setup(cls):
super().setup_base("ghci-snl-oneapi")
cls.baselines_dir = "/projects/e3sm/data/baselines/scream/ghci-snl-oneapi"
super().setup_base("ghci-snl-intel")
cls.baselines_dir = "/projects/e3sm/data/baselines/scream/ghci-snl-intel"
cls.env_setup = ["export GATOR_INITIAL_MB=4000MB",
"export OMPI_MCA_io=romio321"]

Expand Down
Loading