module m_rhs
use m_eos
implicit none
real(wp), allocatable :: blkmod(:, :, :)
!$acc declare create(blkmod)
contains
subroutine by_element_idx(q, n) ! MFC's form: field elements indexed through a device-resident type
type(vector_field), intent(in) :: q
integer, intent(in) :: n
integer :: k, l, m
!$acc parallel loop collapse(3) private(k, l, m)
do m = 0, 0
do l = 0, 0
do k = 0, n
call bulk_modulus(q%vf(eqn%e)%sf(k, l, m), q%vf(eqn%adv)%sf(k, l, m), q%vf(eqn%cont)%sf(k, l, m), 1, blkmod(k, l, m))
end do
end do
end do
end subroutine
subroutine by_element_const(q, n) ! same with literal indices
type(vector_field), intent(in) :: q
integer, intent(in) :: n
integer :: k, l, m
!$acc parallel loop collapse(3) private(k, l, m)
do m = 0, 0
do l = 0, 0
do k = 0, n
call bulk_modulus(q%vf(1)%sf(k, l, m), q%vf(2)%sf(k, l, m), q%vf(3)%sf(k, l, m), 1, blkmod(k, l, m))
end do
end do
end do
end subroutine
subroutine elements_in_scalar_out(q, n)
type(vector_field), intent(in) :: q
integer, intent(in) :: n
integer :: k, l, m
real(wp) :: b
!$acc parallel loop collapse(3) private(k, l, m, b)
do m = 0, 0
do l = 0, 0
do k = 0, n
call bulk_modulus(q%vf(1)%sf(k, l, m), q%vf(2)%sf(k, l, m), q%vf(3)%sf(k, l, m), 1, b)
blkmod(k, l, m) = b
end do
end do
end do
end subroutine
subroutine scalars_in_element_out(q, n)
type(vector_field), intent(in) :: q
integer, intent(in) :: n
integer :: k, l, m
real(wp) :: p, a, ar
!$acc parallel loop collapse(3) private(k, l, m, p, a, ar)
do m = 0, 0
do l = 0, 0
do k = 0, n
p = q%vf(1)%sf(k, l, m)
a = q%vf(2)%sf(k, l, m)
ar = q%vf(3)%sf(k, l, m)
call bulk_modulus(p, a, ar, 1, blkmod(k, l, m))
end do
end do
end do
end subroutine
subroutine by_scalar(q, n) ! the fix: scalars in, scalar out
type(vector_field), intent(in) :: q
integer, intent(in) :: n
integer :: k, l, m
real(wp) :: p, a, ar, b
!$acc parallel loop collapse(3) private(k, l, m, p, a, ar, b)
do m = 0, 0
do l = 0, 0
do k = 0, n
p = q%vf(eqn%e)%sf(k, l, m)
a = q%vf(eqn%adv)%sf(k, l, m)
ar = q%vf(eqn%cont)%sf(k, l, m)
call bulk_modulus(p, a, ar, 1, b)
blkmod(k, l, m) = b
end do
end do
end do
end subroutine
end module
CCE 19.0.0 OpenACC (Frontier,
-hacc -h acc_model=auto_async_none -h acc_model=no_fast_addr -O2, MFC's flags) mishandles a by-reference actual argument that is an element of a device-resident array when the callee is a!$acc routine seqthat does not get inlined. Reads through such an argument return garbage; writes through one are lost. Scalars and locals are correct. Index kind (literal or device-resident derived-type member) makes no difference. CCE OpenMP offload, amdflang and nvfortran are correct on the same source.Found on #1811, where the new EOS helpers were called from kernels with
q_prim_vf%vf(i)%sf(j,k,l)inputs and ablkmod1(k,l,q)output; every test on those paths ended inNaN(s) in timestep outputon the Frontier CCE gpu-acc lanes only. An in-situ check recomputed the kernel's output on the host: device 0.0 in 300/300 cells, host 1.4 from the same fields. The parent PR passed because its only such callee was a leaf that CCE inlined at device link time; the rule holds for master too, which passes field elements into device routines at several sites that work only by inlining.Standalone reproducer (three files, one kernel per argument kind; output below is from a Frontier login node):
Workaround, applied in #1811 and recorded in
.claude/rules/common-pitfalls.md: copy the element to a scalar before the call and receive outputs into a scalar. A lint rule could enforce it (flag%sf(or aGPU_DECLAREd array element inside the argument list of acallin a kernel or device routine); it would also flag the currently-working master sites, which rely on inlining.build.sh
m_eos.f90
m_rhs.f90
main.f90