diff --git a/programs/MHDTurbulence/build.sh b/programs/MHDTurbulence/build.sh index 54341e7..2893ef0 100644 --- a/programs/MHDTurbulence/build.sh +++ b/programs/MHDTurbulence/build.sh @@ -36,6 +36,23 @@ case "$system" in echo "Executable is "${BIN}" and copied to "${artdir} cp ../exe/$BIN ../../${artdir} ;; + RIKYU) + module load nvhpc-hpcx-cuda13/26.5 + # Portability fixes; skipped when already upstream + grep -q "integer,dimension(8) :: seed" src_f90_omp_host/main.f90 || \ + patch -p1 < ../programs/${code}/patches/random_seed_put_size.patch + grep -q "acc_init(acc_device_nvidia)" src_f90_acc_device/main.f90 || \ + patch -p1 < ../programs/${code}/patches/acc_init_before_mpi.patch + # ntiles is compile-time: one binary per list.csv rank count + cd src_f90_acc_device + for tiles in 1 4 8; do + make clean >/dev/null 2>&1 || true + sed -i "s/ntiles(3) = \[.*,.*,.*\]/ntiles(3) = [ ${tiles},1,1 ]/" config.f90 + make > /dev/null + echo "Executable is "${BIN}".t"${tiles}" and copied to "${artdir} + cp ../exe/$BIN ../../${artdir}/${BIN}.t${tiles} + done + ;; # in the future, we may add this # MiyabiG/OpenMP) # cd src_f90_omp_device diff --git a/programs/MHDTurbulence/list.csv b/programs/MHDTurbulence/list.csv index 0c8d53d..6d2aac2 100644 --- a/programs/MHDTurbulence/list.csv +++ b/programs/MHDTurbulence/list.csv @@ -2,3 +2,6 @@ system,enable,nodes,numproc_node,nthreads,elapse Fugaku,no,1,4,12,0:10:00 MiyabiC,yes,1,96,1,0:10:00 MiyabiG,yes,1,1,1,0:10:00 +RIKYU,yes,1,1,1,0:10:00 +RIKYU,yes,1,4,1,0:10:00 +RIKYU,yes,2,4,1,0:10:00 diff --git a/programs/MHDTurbulence/patches/README.md b/programs/MHDTurbulence/patches/README.md new file mode 100644 index 0000000..7b0cc7f --- /dev/null +++ b/programs/MHDTurbulence/patches/README.md @@ -0,0 +1,22 @@ +# Patches + +Applied by `build.sh` only when the fix is not already present in the +fetched source (grep-guarded), so they become no-ops once upstream +merges them. Both originate from `william-dawson/MHDTurbulence` +branch `gfortran-port` (candidate upstream PRs): + +- `random_seed_put_size.patch` — commit `d9a19d9`: `random_seed(PUT=)` + requires the array to match the compiler's seed size (8 for gfortran); + the shipped size-2 array breaks strict compilers. Physically inert + (the seeded RNG feeds a disabled perturbation, `rrv=0`). +- `acc_init_before_mpi.patch` — commit `4db3adb`: initialize the CUDA + context before `MPI_Init`. Without it, multi-node runs hang silently + in the first `BoundaryCondition` (UCX "cannot find remote protocol" + on the device-buffer `MPI_ISEND`, then `MPI_WAITALL` on a dead + request; the app's `MPI_ERRORS_RETURN` handler means the failed ISEND + returns an error code that is never checked). + +Launch side (in `run.sh`, RIKYU case): each rank pins one GPU via +`CUDA_VISIBLE_DEVICES=$OMPI_COMM_WORLD_LOCAL_RANK`; combined with the +patch this is required for multi-node GPU runs. Validated 2026-09-03 on +Rikyu (8 GPUs / 2 nodes, results bit-consistent with 1-GPU runs). diff --git a/programs/MHDTurbulence/patches/acc_init_before_mpi.patch b/programs/MHDTurbulence/patches/acc_init_before_mpi.patch new file mode 100644 index 0000000..56f042c --- /dev/null +++ b/programs/MHDTurbulence/patches/acc_init_before_mpi.patch @@ -0,0 +1,19 @@ +diff --git a/src_f90_acc_device/main.f90 b/src_f90_acc_device/main.f90 +index c808716..3356a5e 100644 +--- a/src_f90_acc_device/main.f90 ++++ b/src_f90_acc_device/main.f90 +@@ -1,5 +1,6 @@ + program main + use omp_lib ++ use openacc + use basicmod + use mpimod + use boundarymod +@@ -9,6 +10,7 @@ program main + logical::is_final + logical,parameter:: forceoutput=.true., usualoutput=.false. + data is_final /.false./ ++ call acc_init(acc_device_nvidia) + call InitializeMPI + if(myid_w == 0) print *, "setup grids and fields" + if(myid_w == 0) print *, "grid size for x y z",ngrid1*ntiles(1),ngrid2*ntiles(2),ngrid3*ntiles(3) diff --git a/programs/MHDTurbulence/patches/random_seed_put_size.patch b/programs/MHDTurbulence/patches/random_seed_put_size.patch new file mode 100644 index 0000000..d156919 --- /dev/null +++ b/programs/MHDTurbulence/patches/random_seed_put_size.patch @@ -0,0 +1,50 @@ +diff --git a/src_f90_acc_device/main.f90 b/src_f90_acc_device/main.f90 +index 669188c..c808716 100644 +--- a/src_f90_acc_device/main.f90 ++++ b/src_f90_acc_device/main.f90 +@@ -123,7 +123,7 @@ subroutine GenerateProblem + real(8):: pi + real(8):: den, B0, rho1, rho2, dv, wid, sig + +- integer,dimension(2) :: seed ++ integer,dimension(8) :: seed + real(8),dimension(1) :: rnum + real(8),parameter :: rrv =0.0d-2 + real(8):: dv_harm +@@ -169,9 +169,10 @@ subroutine GenerateProblem + + if(myid_w == 0) write(6,*) rrv*100.0d0 & + & , "% of Randam Perturbation imposed on velocity" ++ seed = 0 + seed(1) = 1 + seed(2) = 1 + myid_w*in*jn*kn +- call random_seed(PUT=seed(1:2)) ++ call random_seed(PUT=seed) + + ! pert + do k=ks,ke +diff --git a/src_f90_omp_host/main.f90 b/src_f90_omp_host/main.f90 +index 62cc30b..d27d395 100644 +--- a/src_f90_omp_host/main.f90 ++++ b/src_f90_omp_host/main.f90 +@@ -120,7 +120,7 @@ subroutine GenerateProblem + real(8):: pi + real(8):: den, B0, rho1, rho2, dv, wid, sig + +- integer,dimension(2) :: seed ++ integer,dimension(8) :: seed + real(8),dimension(1) :: rnum + real(8),parameter :: rrv =0.0d-2 + +@@ -163,9 +163,10 @@ subroutine GenerateProblem + + if(myid_w == 0) write(6,*) rrv*100.0d0 & + & , "% of Randam Perturbation imposed on velocity" ++ seed = 0 + seed(1) = 1 + seed(2) = 1 + myid_w*in*jn*kn +- call random_seed(PUT=seed(1:2)) ++ call random_seed(PUT=seed) + + ! pert + do k=ks,ke diff --git a/programs/MHDTurbulence/run.sh b/programs/MHDTurbulence/run.sh index 8eda70e..ae38532 100644 --- a/programs/MHDTurbulence/run.sh +++ b/programs/MHDTurbulence/run.sh @@ -40,6 +40,20 @@ case "$system" in elapsed=$(grep "sim time \[s\]:" ${LOG} | awk '{print $4}' | tail -n 1) tcc=$(grep "time/count/cell" ${LOG} | awk '{print $2}' | tail -n 1) ;; + RIKYU) + module load nvhpc-hpcx-cuda13/26.5 + exedir=exe + mkdir -p $code/$exedir/ + NP=$((nodes * numproc_node)) + cp $artdir/${BIN}.t${NP} $code/$exedir/${BIN} + cd $code/$exedir/ + echo "code is executed in "$code/$exedir/ + # CUDA context must exist before MPI_Init and each rank must pin one + # GPU, or the inter-node device exchange hangs (UCX protocol failure) + mpirun -np "$NP" bash -c 'export CUDA_VISIBLE_DEVICES=$((OMPI_COMM_WORLD_LOCAL_RANK % 4)); exec ./Simulation.x' > ${LOG} 2>&1 + elapsed=$(grep "sim time \[s\]:" ${LOG} | awk '{print $4}' | tail -n 1) + tcc=$(grep "time/count/cell" ${LOG} | awk '{print $2}' | tail -n 1) + ;; *) echo "Unknown system: $system" exit 1