Skip to content

Commit 35d2f06

Browse files
Rohit-KakodkarRohit-Kakodkar
authored
Merge pull request #97 from PrincetonUniversity/code-along
Code along Known issues - 1. memory misallocation with intel compilers 2. composite boundaries implementation doesn't work with CUDA compiler
2 parents 4bdfac1 + 6520a9e commit 35d2f06

511 files changed

Lines changed: 46742 additions & 3700 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/unittests.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ jobs:
1414
run: git submodule init
1515
&& git submodule update
1616
- name: Configure
17-
run: cmake -S . -B build
17+
run: cmake -S . -B build -DBUILD_TESTS=ON
1818
- name: Build
1919
run: cmake --build build
2020
- name: Run all tests

‎.gitignore‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,12 @@ examples/*/*_config.yaml
1515
tests/regression-tests/*.yaml
1616
tests/regression-tests/*/cpu/*.yaml
1717
tests/regression-tests/*/gpu/*.yaml
18+
*.mod
19+
*.o
20+
*/OUTPUT_FILES/*
21+
*.gnu
22+
examples/*/Par_File
23+
*OUTPUT_FILES*
24+
*.sqlite
25+
*.nsys-rep
26+
*.ncu-rep

‎.jenkins/compiler_checks.gvy‎

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -51,18 +51,19 @@ pipeline{
5151
steps {
5252
echo "Building ${CMAKE_HOST_FLAGS} ${CMAKE_DEVICE_FLAGS}"
5353
sh """
54+
module load boost/1.73.0
5455
module load ${CUDA_MODULE}
55-
cmake3 -S . -B build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME} -DCMAKE_BUILD_TYPE=Release ${CMAKE_HOST_FLAGS} ${CMAKE_DEVICE_FLAGS}
56-
cmake3 --build build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}
56+
cmake3 -S . -B build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT} -DCMAKE_BUILD_TYPE=Release ${CMAKE_HOST_FLAGS} ${CMAKE_DEVICE_FLAGS}
57+
cmake3 --build build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT}
5758
"""
5859
echo ' Build completed '
5960
}
6061
}
61-
stage (' Clean '){
62-
steps {
63-
echo ' Cleaning '
64-
sh "rm -rf build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}"
65-
}
62+
}
63+
post {
64+
always {
65+
echo ' Cleaning '
66+
sh "rm -rf build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT}"
6667
}
6768
}
6869
}
@@ -107,20 +108,21 @@ pipeline{
107108
steps {
108109
echo "Building ${CMAKE_HOST_FLAGS} ${CMAKE_DEVICE_FLAGS}"
109110
sh """
111+
module load boost/1.73.0
110112
module load intel/2022.2.0
111113
export CC=icx
112114
export CXX=icpx
113-
cmake3 -S . -B build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME} -DCMAKE_BUILD_TYPE=Release ${CMAKE_HOST_FLAGS} ${CMAKE_DEVICE_FLAGS}
114-
cmake3 --build build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}
115+
cmake3 -S . -B build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT} -DCMAKE_BUILD_TYPE=Release ${CMAKE_HOST_FLAGS} ${CMAKE_DEVICE_FLAGS}
116+
cmake3 --build build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT}
115117
"""
116118
echo ' Build completed '
117119
}
118120
}
119-
stage (' Clean '){
120-
steps {
121-
echo ' Cleaning '
122-
sh "rm -rf build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}"
123-
}
121+
}
122+
post {
123+
always {
124+
echo ' Cleaning '
125+
sh "rm -rf build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT}"
124126
}
125127
}
126128
}

‎.jenkins/regression_tests.gvy‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ pipeline {
1313
// Screen is needed since the sessions need to remain active even when this stage exits
1414
sh """
1515
screen -dm salloc -J jenkins_cpu -N 1 -n 1 -t 00:30:00 --constraint=broadwell
16-
screen -dm salloc -J jenkins_gpu -N 1 -c 10 -t 00:30:00 --gres=gpu:1 --constraint=a100 &
16+
screen -dm salloc -J jenkins_gpu -N 1 -c 10 -t 00:30:00 --gres=gpu:1 --constraint=a100
1717
"""
1818
}
1919
}
@@ -145,9 +145,9 @@ pipeline {
145145
stage (' Checkout main branch '){
146146
steps {
147147
checkout([$class: 'GitSCM',
148-
branches: [[name: 'devel']],
148+
branches: [[name: 'code-along']],
149149
extensions: [lfs()],
150-
userRemoteConfigs: [[url: 'https://github.com/PrincetonUniversity/specfem2d_kokkos']]])
150+
userRemoteConfigs: [[url: 'https://github.com/PrincetonUniversity/specfempp']]])
151151
}
152152
}
153153

‎.jenkins/unittests.gvy‎

Lines changed: 21 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -61,9 +61,10 @@ pipeline {
6161
steps {
6262
echo "Building ${CMAKE_HOST_FLAGS} ${CMAKE_DEVICE_FLAGS}"
6363
sh """
64+
module load boost/1.73.0
6465
module load ${CUDA_MODULE}
65-
cmake3 -S . -B build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME} -DCMAKE_BUILD_TYPE=Release ${CMAKE_HOST_FLAGS} ${CMAKE_DEVICE_FLAGS}
66-
cmake3 --build build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}
66+
cmake3 -S . -B build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT} -DCMAKE_BUILD_TYPE=Release ${CMAKE_HOST_FLAGS} ${CMAKE_DEVICE_FLAGS} -DBUILD_TESTS=ON
67+
cmake3 --build build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT}
6768
"""
6869
echo ' Build completed '
6970
}
@@ -72,16 +73,17 @@ pipeline {
7273
steps {
7374
echo " Running Unittests "
7475
sh """
75-
cd build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}/tests/unit-tests
76-
srun -N 1 -t 00:10:00 ${HOST_RUN_FLAGS} ${DEVICE_RUN_FLAGS} bash -c 'export OMP_PROC_BIND=spread; export OMP_THREADS=places; ctest'
76+
module load boost/1.73.0
77+
cd build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT}/tests/unit-tests
78+
srun -N 1 -t 00:10:00 ${HOST_RUN_FLAGS} ${DEVICE_RUN_FLAGS} bash -c 'export OMP_PROC_BIND=spread; export OMP_THREADS=places; ctest; ctest --rerun-failed --output-on-failure;'
7779
"""
7880
}
7981
}
80-
stage (' Clean '){
81-
steps {
82-
echo ' Cleaning '
83-
sh "rm -rf build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}"
84-
}
82+
}
83+
post {
84+
always {
85+
echo ' Cleaning '
86+
sh "rm -rf build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT}"
8587
}
8688
}
8789
}
@@ -138,8 +140,8 @@ pipeline {
138140
module load intel/2022.2.0
139141
export CC=icx
140142
export CXX=icpx
141-
cmake3 -S . -B build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME} -DCMAKE_BUILD_TYPE=Release ${CMAKE_HOST_FLAGS} ${CMAKE_DEVICE_FLAGS}
142-
cmake3 --build build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}
143+
cmake3 -S . -B build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT} -DCMAKE_BUILD_TYPE=Release ${CMAKE_HOST_FLAGS} ${CMAKE_DEVICE_FLAGS} -DBUILD_TESTS=ON
144+
cmake3 --build build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT}
143145
"""
144146
echo ' Build completed '
145147
}
@@ -148,17 +150,18 @@ pipeline {
148150
steps {
149151
echo " Running Unittests "
150152
sh """
153+
module load boost/1.73.0
151154
module load intel/2022.2.0
152-
cd build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}/tests/unit-tests
153-
srun -N 1 -t 00:10:00 ${HOST_RUN_FLAGS} ${DEVICE_RUN_FLAGS} bash -c 'export OMP_PROC_BIND=spread; export OMP_THREADS=places; ctest'
155+
cd build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT}/tests/unit-tests
156+
srun -N 1 -t 00:10:00 ${HOST_RUN_FLAGS} ${DEVICE_RUN_FLAGS} bash -c 'export OMP_PROC_BIND=spread; export OMP_THREADS=places; ctest; ctest --rerun-failed --output-on-failure;'
154157
"""
155158
}
156159
}
157-
stage (' Clean '){
158-
steps {
159-
echo ' Cleaning '
160-
sh "rm -rf build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}"
161-
}
160+
}
161+
post {
162+
always {
163+
echo ' Cleaning '
164+
sh "rm -rf build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT}"
162165
}
163166
}
164167
}

‎CMakeLists.txt‎

Lines changed: 40 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ project(specfem2d_kokkos VERSION 0.1.0)
44

55
set(CMAKE_CXX_STANDARD 17)
66
option(MPI_PARALLEL "MPI enabled" OFF)
7+
option(BUILD_TESTS "Tests included" OFF)
8+
option(BUILD_EXAMPLES "Examples included" OFF)
79

810
if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM")
911
set(CMAKE_CXX_FLAGS "-fp-model=precise")
@@ -25,25 +27,37 @@ endif()
2527
## TODO: Add options for on utilizing in house builds
2628
include(FetchContent)
2729
FetchContent_Declare(
28-
kokkos
29-
URL https://github.com/kokkos/kokkos/archive/refs/tags/4.0.00.zip
30+
kokkos
31+
URL https://github.com/kokkos/kokkos/archive/refs/tags/4.0.00.zip
3032
)
3133
FetchContent_MakeAvailable(kokkos)
3234

3335
FetchContent_Declare(
3436
yaml
35-
URL https://github.com/jbeder/yaml-cpp/archive/refs/tags/yaml-cpp-0.7.0.zip
37+
URL https://github.com/jbeder/yaml-cpp/archive/refs/tags/0.8.0.tar.gz
3638
)
3739
FetchContent_MakeAvailable(yaml)
38-
include_directories(BEFORE SYSTEM ${yaml_SOURCE_DIR} ${yaml_BINARY_DIR}/include)
3940

40-
add_subdirectory(boost-cmake)
41+
include_directories(BEFORE SYSTEM ${yaml_BINARY_DIR} ${yaml_SOURCE_DIR}/include)
42+
43+
# Try finding boost and if not found install.
44+
find_package(Boost 1.73.0 COMPONENTS program_options filesystem system)
45+
46+
if (NOT ${Boost_FOUND})
47+
add_subdirectory(boost-cmake)
48+
else ()
49+
message(STATUS " LIB: ${Boost_LIBRARY_DIRS}")
50+
message(STATUS " INC: ${Boost_INCLUDE_DIRS}")
51+
message(STATUS " LIBSO: ${Boost_LIBRARIES}")
52+
endif()
4153

4254
configure_file(constants.hpp.in constants.hpp)
4355

4456
include_directories(include)
4557
include_directories(${CMAKE_BINARY_DIR})
4658

59+
add_subdirectory(meshfem2d)
60+
4761
# Build specfem2d libraries
4862
add_library(
4963
quadrature
@@ -85,16 +99,6 @@ else()
8599
message("-- Compiling SPECFEM without MPI")
86100
endif(MPI_PARALLEL)
87101

88-
add_library(
89-
operators
90-
src/mathematical_operators/mathematical_operators.cpp
91-
)
92-
93-
target_link_libraries(
94-
operators
95-
Kokkos::kokkos
96-
)
97-
98102
add_library(
99103
material_class
100104
src/material/elastic_material.cpp
@@ -113,12 +117,15 @@ add_library(
113117
src/mesh/IO/fortran/read_material_properties.cpp
114118
src/mesh/boundaries/forcing_boundaries.cpp
115119
src/mesh/boundaries/absorbing_boundaries.cpp
120+
src/mesh/boundaries/acoustic_free_surface.cpp
116121
src/mesh/elements/tangential_elements.cpp
117122
src/mesh/elements/axial_elements.cpp
118123
src/mesh/properties/properties.cpp
119124
src/mesh/mpi_interfaces/mpi_interfaces.cpp
120125
src/mesh/material_indic/material_indic.cpp
121-
src/mesh/surfaces/acoustic_free_surface.cpp
126+
src/mesh/coupled_interfaces/elastic_acoustic.cpp
127+
src/mesh/coupled_interfaces/elastic_poroelastic.cpp
128+
src/mesh/coupled_interfaces/acoustic_poroelastic.cpp
122129
src/mesh/mesh.cpp
123130
)
124131

@@ -157,6 +164,7 @@ target_link_libraries(
157164
add_library(
158165
source_time_function
159166
src/source_time_function/dirac.cpp
167+
src/source_time_function/ricker.cpp
160168
)
161169

162170
target_link_libraries(
@@ -206,6 +214,8 @@ add_library(
206214
src/compute/compute_properties.cpp
207215
src/compute/compute_sources.cpp
208216
src/compute/compute_receivers.cpp
217+
src/compute/coupled_interfaces.cpp
218+
src/compute/compute_boundaries.cpp
209219
)
210220

211221
target_link_libraries(
@@ -218,22 +228,6 @@ target_link_libraries(
218228
Kokkos::kokkos
219229
)
220230

221-
# there seems to be a bug when compiling compute with -O3 mode using the icpx compiler.
222-
# set_target_properties(compute PROPERTIES COMPILE_OPTIONS "$<$<CONFIG:RELEASE>:-fp-model=precise>")
223-
224-
add_library(
225-
domain
226-
src/domain/elastic_domain.cpp
227-
)
228-
229-
target_link_libraries(
230-
domain
231-
compute
232-
quadrature
233-
operators
234-
Kokkos::kokkos
235-
)
236-
237231
add_library(
238232
timescheme
239233
src/timescheme/timescheme.cpp
@@ -242,7 +236,9 @@ add_library(
242236

243237
target_link_libraries(
244238
timescheme
245-
domain
239+
Kokkos::kokkos
240+
yaml-cpp
241+
compute
246242
)
247243

248244
add_library(
@@ -256,18 +252,6 @@ target_link_libraries(
256252
receiver_class
257253
)
258254

259-
add_library(
260-
solver
261-
src/solver/time_marching.cpp
262-
)
263-
264-
target_link_libraries(
265-
solver
266-
domain
267-
timescheme
268-
writer
269-
)
270-
271255
add_library(
272256
parameter_reader
273257
src/parameter_parser/run_setup.cpp
@@ -288,6 +272,7 @@ target_link_libraries(
288272
timescheme
289273
receiver_class
290274
yaml-cpp
275+
writer
291276
Boost::filesystem
292277
)
293278

@@ -306,17 +291,22 @@ target_link_libraries(
306291
compute
307292
source_class
308293
parameter_reader
309-
domain
310-
solver
311294
receiver_class
312295
writer
313296
Boost::program_options
314297
)
315298

316299
# Include tests
317-
add_subdirectory(tests/unit-tests)
318-
add_subdirectory(tests/regression-tests)
319-
add_subdirectory(examples)
300+
if (BUILD_TESTS)
301+
message("-- Including tests.")
302+
add_subdirectory(tests/unit-tests)
303+
add_subdirectory(tests/regression-tests)
304+
endif()
305+
306+
if (BUILD_EXAMPLES)
307+
message("-- Including examples.")
308+
add_subdirectory(examples)
309+
endif()
320310

321311
# Doxygen
322312

0 commit comments

Comments
 (0)