Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions programs/MHDTurbulence/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,23 @@ case "$system" in
echo "Executable is "${BIN}" and copied to "${artdir}
cp ../exe/$BIN ../../${artdir}
;;
RIKYU)
module load nvhpc-hpcx-cuda13/26.5
# Portability fixes; skipped when already upstream
grep -q "integer,dimension(8) :: seed" src_f90_omp_host/main.f90 || \
patch -p1 < ../programs/${code}/patches/random_seed_put_size.patch
grep -q "acc_init(acc_device_nvidia)" src_f90_acc_device/main.f90 || \
patch -p1 < ../programs/${code}/patches/acc_init_before_mpi.patch
# ntiles is compile-time: one binary per list.csv rank count
cd src_f90_acc_device
for tiles in 1 4 8; do
make clean >/dev/null 2>&1 || true
sed -i "s/ntiles(3) = \[.*,.*,.*\]/ntiles(3) = [ ${tiles},1,1 ]/" config.f90
make > /dev/null
echo "Executable is "${BIN}".t"${tiles}" and copied to "${artdir}
cp ../exe/$BIN ../../${artdir}/${BIN}.t${tiles}
done
;;
# in the future, we may add this
# MiyabiG/OpenMP)
# cd src_f90_omp_device
Expand Down
3 changes: 3 additions & 0 deletions programs/MHDTurbulence/list.csv
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,6 @@ system,enable,nodes,numproc_node,nthreads,elapse
Fugaku,no,1,4,12,0:10:00
MiyabiC,yes,1,96,1,0:10:00
MiyabiG,yes,1,1,1,0:10:00
RIKYU,yes,1,1,1,0:10:00
RIKYU,yes,1,4,1,0:10:00
RIKYU,yes,2,4,1,0:10:00
22 changes: 22 additions & 0 deletions programs/MHDTurbulence/patches/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Patches

Applied by `build.sh` only when the fix is not already present in the
fetched source (grep-guarded), so they become no-ops once upstream
merges them. Both originate from `william-dawson/MHDTurbulence`
branch `gfortran-port` (candidate upstream PRs):

- `random_seed_put_size.patch` — commit `d9a19d9`: `random_seed(PUT=)`
requires the array to match the compiler's seed size (8 for gfortran);
the shipped size-2 array breaks strict compilers. Physically inert
(the seeded RNG feeds a disabled perturbation, `rrv=0`).
- `acc_init_before_mpi.patch` — commit `4db3adb`: initialize the CUDA
context before `MPI_Init`. Without it, multi-node runs hang silently
in the first `BoundaryCondition` (UCX "cannot find remote protocol"
on the device-buffer `MPI_ISEND`, then `MPI_WAITALL` on a dead
request; the app's `MPI_ERRORS_RETURN` handler means the failed ISEND
returns an error code that is never checked).

Launch side (in `run.sh`, RIKYU case): each rank pins one GPU via
`CUDA_VISIBLE_DEVICES=$OMPI_COMM_WORLD_LOCAL_RANK`; combined with the
patch this is required for multi-node GPU runs. Validated 2026-09-03 on
Rikyu (8 GPUs / 2 nodes, results bit-consistent with 1-GPU runs).
19 changes: 19 additions & 0 deletions programs/MHDTurbulence/patches/acc_init_before_mpi.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
diff --git a/src_f90_acc_device/main.f90 b/src_f90_acc_device/main.f90
index c808716..3356a5e 100644
--- a/src_f90_acc_device/main.f90
+++ b/src_f90_acc_device/main.f90
@@ -1,5 +1,6 @@
program main
use omp_lib
+ use openacc
use basicmod
use mpimod
use boundarymod
@@ -9,6 +10,7 @@ program main
logical::is_final
logical,parameter:: forceoutput=.true., usualoutput=.false.
data is_final /.false./
+ call acc_init(acc_device_nvidia)
call InitializeMPI
if(myid_w == 0) print *, "setup grids and fields"
if(myid_w == 0) print *, "grid size for x y z",ngrid1*ntiles(1),ngrid2*ntiles(2),ngrid3*ntiles(3)
50 changes: 50 additions & 0 deletions programs/MHDTurbulence/patches/random_seed_put_size.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
diff --git a/src_f90_acc_device/main.f90 b/src_f90_acc_device/main.f90
index 669188c..c808716 100644
--- a/src_f90_acc_device/main.f90
+++ b/src_f90_acc_device/main.f90
@@ -123,7 +123,7 @@ subroutine GenerateProblem
real(8):: pi
real(8):: den, B0, rho1, rho2, dv, wid, sig

- integer,dimension(2) :: seed
+ integer,dimension(8) :: seed
real(8),dimension(1) :: rnum
real(8),parameter :: rrv =0.0d-2
real(8):: dv_harm
@@ -169,9 +169,10 @@ subroutine GenerateProblem

if(myid_w == 0) write(6,*) rrv*100.0d0 &
& , "% of Randam Perturbation imposed on velocity"
+ seed = 0
seed(1) = 1
seed(2) = 1 + myid_w*in*jn*kn
- call random_seed(PUT=seed(1:2))
+ call random_seed(PUT=seed)

! pert
do k=ks,ke
diff --git a/src_f90_omp_host/main.f90 b/src_f90_omp_host/main.f90
index 62cc30b..d27d395 100644
--- a/src_f90_omp_host/main.f90
+++ b/src_f90_omp_host/main.f90
@@ -120,7 +120,7 @@ subroutine GenerateProblem
real(8):: pi
real(8):: den, B0, rho1, rho2, dv, wid, sig

- integer,dimension(2) :: seed
+ integer,dimension(8) :: seed
real(8),dimension(1) :: rnum
real(8),parameter :: rrv =0.0d-2

@@ -163,9 +163,10 @@ subroutine GenerateProblem

if(myid_w == 0) write(6,*) rrv*100.0d0 &
& , "% of Randam Perturbation imposed on velocity"
+ seed = 0
seed(1) = 1
seed(2) = 1 + myid_w*in*jn*kn
- call random_seed(PUT=seed(1:2))
+ call random_seed(PUT=seed)

! pert
do k=ks,ke
14 changes: 14 additions & 0 deletions programs/MHDTurbulence/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,20 @@ case "$system" in
elapsed=$(grep "sim time \[s\]:" ${LOG} | awk '{print $4}' | tail -n 1)
tcc=$(grep "time/count/cell" ${LOG} | awk '{print $2}' | tail -n 1)
;;
RIKYU)
module load nvhpc-hpcx-cuda13/26.5
exedir=exe
mkdir -p $code/$exedir/
NP=$((nodes * numproc_node))
cp $artdir/${BIN}.t${NP} $code/$exedir/${BIN}
cd $code/$exedir/
echo "code is executed in "$code/$exedir/
# CUDA context must exist before MPI_Init and each rank must pin one
# GPU, or the inter-node device exchange hangs (UCX protocol failure)
mpirun -np "$NP" bash -c 'export CUDA_VISIBLE_DEVICES=$((OMPI_COMM_WORLD_LOCAL_RANK % 4)); exec ./Simulation.x' > ${LOG} 2>&1
elapsed=$(grep "sim time \[s\]:" ${LOG} | awk '{print $4}' | tail -n 1)
tcc=$(grep "time/count/cell" ${LOG} | awk '{print $2}' | tail -n 1)
;;
*)
echo "Unknown system: $system"
exit 1
Expand Down
Loading