From e775a7bb056763be64907e3d29e6fc1eb241cd74 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Mon, 24 Aug 2026 09:43:32 +0200 Subject: [PATCH 01/10] Add the device-side hostcall protocol. Port LLVM libc's GPU RPC protocol: a warp-collective mailbox in pinned, device-mapped host memory, with per-lane packets and device-memory locks. Shared mailboxes use only loads, stores, and fences because system-scope RMW atomics are not atomic across PCIe. Expose raw ports plus @hostcall and hostcall/hostcall_async for by-value calls to statically identifiable host functions. Arguments may include compiler-relocated host constants; results must be isbits. A target's type hash is embedded in the kernel image, and the protocol descriptor shares the compact kernel state used by dynamic parallelism. --- CUDACore/src/compiler/exceptions.jl | 19 +- CUDACore/src/compiler/execution.jl | 5 +- CUDACore/src/device/intrinsics.jl | 1 + CUDACore/src/device/intrinsics/hostcall.jl | 587 +++++++++++++++++++++ CUDACore/src/device/quirks.jl | 2 +- CUDACore/src/device/runtime.jl | 16 +- test/core/device/hostcall.jl | 264 +++++++++ 7 files changed, 883 insertions(+), 11 deletions(-) create mode 100644 CUDACore/src/device/intrinsics/hostcall.jl create mode 100644 test/core/device/hostcall.jl diff --git a/CUDACore/src/compiler/exceptions.jl b/CUDACore/src/compiler/exceptions.jl index 76d6451a3f..29da71a036 100644 --- a/CUDACore/src/compiler/exceptions.jl +++ b/CUDACore/src/compiler/exceptions.jl @@ -14,15 +14,28 @@ end ## exception handling const exception_infos = Dict{CuContext, HostMemory}() +const exception_infos_lock = ReentrantLock() + +# A no-hostcall descriptor follows the exception flag in the same mapped allocation. This +# gives every kernel one runtime-state pointer, even when hostcalls are unavailable. +const exception_client_offset = cld(sizeof(ExceptionInfo_st), sizeof(UInt)) * sizeof(UInt) +const exception_state_size = exception_client_offset + sizeof(HostcallClient) # create a CPU/GPU exception flag for error signalling, and put it in the module function create_exceptions!(mod::CuModule) - mem = get!(exception_infos, mod.ctx) do - alloc(HostMemory, sizeof(ExceptionInfo_st), MEMHOSTALLOC_DEVICEMAP) + mem = @lock exception_infos_lock begin + get!(exception_infos, mod.ctx) do + alloc(HostMemory, exception_state_size, MEMHOSTALLOC_DEVICEMAP) + end end exception_info = convert(ExceptionInfo, mem) unsafe_store!(exception_info, ExceptionInfo_st()) - return exception_info + base = convert(Ptr{UInt8}, mem) + client_ptr = base + exception_client_offset + unsafe_store!(convert(Ptr{HostcallClient}, client_ptr), + HostcallClient(reinterpret(Ptr{Cvoid}, exception_info))) + client = reinterpret(HostcallClientPtr, client_ptr) + return exception_info, client end # check the exception flags on every API call, similarly to how CUDA handles errors diff --git a/CUDACore/src/compiler/execution.jl b/CUDACore/src/compiler/execution.jl index 7bda242d1b..5d3e70c0a3 100644 --- a/CUDACore/src/compiler/execution.jl +++ b/CUDACore/src/compiler/execution.jl @@ -560,7 +560,7 @@ end # add the kernel state, passing an instance with a unique seed pushfirst!(call_t, KernelState) - pushfirst!(call_args, :(KernelState(kernel.state.exception_info, make_seed(kernel)))) + pushfirst!(call_args, :(KernelState(kernel.state.client, make_seed(kernel)))) # finalize types call_tt = Base.to_tuple_type(call_t) @@ -710,7 +710,8 @@ function cufunction(f::F, tt::TT=Tuple{}; kwargs...) where {F,TT} kernel = get(_kernel_instances, key, nothing) if kernel === nothing # create the kernel state object - state = KernelState(create_exceptions!(fun.mod), UInt32(0)) + _, client = create_exceptions!(fun.mod) + state = KernelState(client, UInt32(0)) kernel = HostKernel{F,tt}(f, fun, state) _kernel_instances[key] = kernel diff --git a/CUDACore/src/device/intrinsics.jl b/CUDACore/src/device/intrinsics.jl index 4d26afbdae..6169f5e4bc 100644 --- a/CUDACore/src/device/intrinsics.jl +++ b/CUDACore/src/device/intrinsics.jl @@ -12,6 +12,7 @@ include("intrinsics/output.jl") include("intrinsics/assertion.jl") include("intrinsics/atomics.jl") include("intrinsics/misc.jl") +include("intrinsics/hostcall.jl") include("intrinsics/wmma.jl") # functionality from libdevice diff --git a/CUDACore/src/device/intrinsics/hostcall.jl b/CUDACore/src/device/intrinsics/hostcall.jl new file mode 100644 index 0000000000..ebc985f36f --- /dev/null +++ b/CUDACore/src/device/intrinsics/hostcall.jl @@ -0,0 +1,587 @@ +# Hostcall: calling host functions from device code. +# +# The protocol is a port of LLVM libc's GPU RPC ("mailbox"): a pinned, device-mapped host +# buffer holds, per port, an inbox word (written by the host), an outbox word (written by +# the device), a header, and one 64-byte packet per lane. Ownership of a port's buffer is +# encoded by the inbox and outbox bits: the device owns the port when they are equal, the +# host when they differ, and every send or receive is a single bit flip of the writer's own +# mailbox. Only loads, stores and fences touch the shared buffer (system-scope RMW atomics +# are not atomic across PCIe); intra-GPU arbitration uses a lock bitfield in device memory. +# Ports are claimed per warp: all lanes that reach a call site together share one port and +# each lane gets its own packet. + +export @hostcall, hostcall, hostcall_async +@public HostcallClient, HostcallPort, HostcallHeader, + hostcall_open, hostcall_send!, hostcall_recv!, hostcall_close!, + hostcall_lane_packet, HOSTCALL_PACKET_SIZE, hostcall_packet_layout + + +## shared data structures + +""" + HostcallHeader + +Per-port header written by the device when it opens a port: the mask of lanes that take +part in the call, a flag word, and the 64-bit target identifier. +""" +struct HostcallHeader + mask::UInt32 + flags::UInt32 + target::UInt64 +end + +const HOSTCALL_PACKET_SIZE = 64 # bytes per lane +const HOSTCALL_LANES = 32 # packets per port +const HOSTCALL_PORT_BYTES = HOSTCALL_PACKET_SIZE * HOSTCALL_LANES + +# Header flags used by the generic host service. Raw protocols may use the remaining bits. +const HOSTCALL_FLAG_ASYNC = UInt32(1) + +# Built-in targets occupy the low identifiers; static targets have the high bit set. +const HOSTCALL_BUILTIN_IDS = UInt64(256) +const HOSTCALL_STATIC_ID_BIT = UInt64(0x8000_0000_0000_0000) +const HC_EXCEPTION = UInt64(1) + +# inbox words carry a status in the bits above the ownership bit; the host sets these +# when it replies to a port. bit 0 is the ownership bit. +const HOSTCALL_STATUS_OK = UInt32(0) +const HOSTCALL_STATUS_ERROR = UInt32(1) # the handler threw, or the target is unknown + +""" + HostcallClient + +Device-side descriptor of a hostcall area: the number of ports and pointers to the +mailboxes, headers, packets (all in pinned host memory) and the lock bitfield (in device +memory). A client with `nports == 0` indicates that hostcalls are not available. +The descriptor also points to the context's exception state. The kernel state holds only +a pointer to this descriptor, keeping it small (it is passed to child launches during +dynamic parallelism). +""" +struct HostcallClient + nports::UInt32 + inbox::LLVMPtr{UInt32,AS.Global} # host-written, device-read + outbox::LLVMPtr{UInt32,AS.Global} # device-written, host-read + header::LLVMPtr{HostcallHeader,AS.Global} + packet::LLVMPtr{UInt8,AS.Global} # nports * 32 lanes * 64 bytes + lock::LLVMPtr{UInt32,AS.Global} # device memory, nports/32 words + exception_info::Ptr{Cvoid} +end + +HostcallClient(exception_info::Ptr{Cvoid}=C_NULL) = + HostcallClient(0, reinterpret(LLVMPtr{UInt32,AS.Global}, C_NULL), + reinterpret(LLVMPtr{UInt32,AS.Global}, C_NULL), + reinterpret(LLVMPtr{HostcallHeader,AS.Global}, C_NULL), + reinterpret(LLVMPtr{UInt8,AS.Global}, C_NULL), + reinterpret(LLVMPtr{UInt32,AS.Global}, C_NULL), exception_info) + +# protocol-only constructor, for raw-port users that do not carry exception state +HostcallClient(nports, inbox, outbox, header, packet, lock) = + HostcallClient(nports, inbox, outbox, header, packet, lock, C_NULL) + +const HostcallClientPtr = LLVMPtr{HostcallClient,AS.Global} +const null_hostcall_client = reinterpret(HostcallClientPtr, C_NULL) + +""" + hostcall_packet_layout(nports) + +Compute the byte layout of a hostcall area with `nports` ports. Returns a named tuple with +the offsets of the client descriptor, inbox, outbox, header and packet arrays, and the +total size. The layout is shared between the device-side client and the host-side area. +""" +function hostcall_packet_layout(nports::Integer) + nports >= 0 || throw(ArgumentError("the number of hostcall ports must be non-negative")) + nports <= typemax(UInt32) || throw(ArgumentError("too many hostcall ports: $nports")) + nports = Int(nports) + client = 0 + # Keep the immutable descriptor off the cache line containing the first mailboxes. + inbox = Base.checked_mul(cld(sizeof(HostcallClient), 128), 128) + outbox = Base.checked_add(inbox, Base.checked_mul(4, nports)) + header = Base.checked_add(outbox, Base.checked_mul(4, nports)) + packet = Base.checked_mul(cld(Base.checked_add(header, + Base.checked_mul(sizeof(HostcallHeader), nports)), + 128), 128) + total = Base.checked_add(packet, Base.checked_mul(nports, HOSTCALL_PORT_BYTES)) + return (; client, inbox, outbox, header, packet, total) +end + +""" + HostcallPort + +A claimed hostcall port: the client it belongs to, the mask of participating lanes, the +port index, the current value of the outbox bit and whether the device currently owns the +buffer. Ports are immutable values; [`hostcall_send!`](@ref) and [`hostcall_recv!`](@ref) +return the updated port. +""" +struct HostcallPort + client::HostcallClient + mask::UInt32 + index::UInt32 + out::UInt32 + owns::Bool +end + + +## memory model primitives + +# mailbox accesses are relaxed system-scope loads/stores (sm_70+), or volatile accesses on +# older hardware, which is what LLVM libc does as well. inline assembly keeps LLVM from +# hoisting or combining them. +@inline function mailbox_load(p::LLVMPtr{UInt32,AS.Global}) + if compute_capability() >= sv"7.0" + @asmcall("ld.relaxed.sys.global.u32 \$0, [\$1];", "=r,l", true, + UInt32, Tuple{LLVMPtr{UInt32,AS.Global}}, p) + else + @asmcall("ld.volatile.global.u32 \$0, [\$1];", "=r,l", true, + UInt32, Tuple{LLVMPtr{UInt32,AS.Global}}, p) + end +end + +@inline function mailbox_store!(p::LLVMPtr{UInt32,AS.Global}, v::UInt32) + if compute_capability() >= sv"7.0" + @asmcall("st.relaxed.sys.global.u32 [\$0], \$1;", "l,r", true, + Cvoid, Tuple{LLVMPtr{UInt32,AS.Global},UInt32}, p, v) + else + @asmcall("st.volatile.global.u32 [\$0], \$1;", "l,r", true, + Cvoid, Tuple{LLVMPtr{UInt32,AS.Global},UInt32}, p, v) + end +end + +# system-scope fence (the PTX fence is acquire-release; before sm_70 use membar.sys) +@inline function fence_sys() + if compute_capability() >= sv"7.0" + @asmcall("fence.acq_rel.sys;", "~{memory}", true, Cvoid, Tuple{}) + else + threadfence_system() + end +end + +# gpu-scope fence, for the lock bitfield in device memory +@inline function fence_gpu() + if compute_capability() >= sv"7.0" + @asmcall("fence.acq_rel.gpu;", "~{memory}", true, Cvoid, Tuple{}) + else + threadfence() + end +end + +# exponential backoff between polls; sleeping is not available before sm_70, in which case +# we simply re-probe (the load itself takes a few hundred nanoseconds). +const HOSTCALL_BACKOFF_MIN = UInt32(8) +const HOSTCALL_BACKOFF_MAX = UInt32(256) +@inline function hostcall_backoff(ns::UInt32) + if compute_capability() >= sv"7.0" + # Do not call `nanosleep` here: its static capability check runs before this + # target-dependent branch is eliminated on some Julia versions. + @asmcall("nanosleep.u32 \$0;", "r", true, Cvoid, Tuple{UInt32}, ns) + end + return min(ns << 1, HOSTCALL_BACKOFF_MAX) +end + + +## warp helpers + +@inline first_lane(mask::UInt32) = trailing_zeros(mask) % Int32 + Int32(1) # 1-based +@inline is_first_lane(mask::UInt32) = laneid() == first_lane(mask) +@inline broadcast_value(mask::UInt32, v) = shfl_sync(mask, v, first_lane(mask)) + +# spread concurrently active warps over the port space; starting every warp at port 0 +# heavily contends the first ports. +@inline function hostcall_start_index(nports::UInt32) + block = (blockIdx().z - 1i32) * gridDim().y * gridDim().x + + (blockIdx().y - 1i32) * gridDim().x + (blockIdx().x - 1i32) + thread = (threadIdx().z - 1i32) * blockDim().y * blockDim().x + + (threadIdx().y - 1i32) * blockDim().x + (threadIdx().x - 1i32) + warps_per_block = (blockDim().x * blockDim().y * blockDim().z + 31i32) ÷ 32i32 + warp = block * warps_per_block + thread ÷ 32i32 + return (warp % UInt32) % nports +end + + +## lock bitfield (device memory) + +@inline function try_lock!(c::HostcallClient, mask::UInt32, index::UInt32) + # every lane in the mask atomically sets the port's bit; the ballot tells whether any + # lane observed it clear, i.e. whether the warp took the lock. lanes outside the mask + # (under independent thread scheduling) are no-ops. + id = laneid() - Int32(1) + in_mask = (mask >> id) & UInt32(1) + word = c.lock + 4 * (index >> 5) + bit = UInt32(1) << (index & UInt32(31)) + before = atomic_or!(word, in_mask * bit) + failed = (before & bit) != 0 + packed = vote_ballot_sync(mask, failed) + holding = mask != packed + holding && fence_gpu() + return holding +end + +@inline function unlock!(c::HostcallClient, mask::UInt32, index::UInt32) + fence_gpu() + if is_first_lane(mask) + word = c.lock + 4 * (index >> 5) + bit = UInt32(1) << (index & UInt32(31)) + atomic_and!(word, ~bit) + end + sync_warp(mask) + return +end + + +## port protocol + +@inline hostcall_load_inbox(c::HostcallClient, mask::UInt32, index::UInt32) = + broadcast_value(mask, mailbox_load(c.inbox + 4index)) + +@inline hostcall_load_outbox(c::HostcallClient, mask::UInt32, index::UInt32) = + broadcast_value(mask, mailbox_load(c.outbox + 4index)) + +# whether the device owns the buffer, given the inbox and outbox words (bit 0 only; the +# remaining inbox bits carry a status) +@inline hostcall_owned(in::UInt32, out::UInt32) = (in & UInt32(1)) == out + +@inline function hostcall_invert_outbox!(c::HostcallClient, mask::UInt32, index::UInt32, + out::UInt32) + inverted = out ⊻ UInt32(1) + sync_warp(mask) + fence_sys() + if is_first_lane(mask) + mailbox_store!(c.outbox + 4index, inverted) + end + return inverted +end + +# spin until the inbox indicates that the device owns the buffer; returns the inbox word +@inline function hostcall_wait_for_ownership(c::HostcallClient, mask::UInt32, index::UInt32, + out::UInt32, in::UInt32) + ns = HOSTCALL_BACKOFF_MIN + while !hostcall_owned(in, out) + ns = hostcall_backoff(ns) + in = hostcall_load_inbox(c, mask, index) + end + fence_sys() + return in +end + +""" + hostcall_open(client::HostcallClient, target::UInt64, flags::UInt32=0) -> HostcallPort + +Claim a free port and label it with `target`. This is a warp-collective operation: all +lanes that are active when calling it share the port, and the active mask is captured for +the lifetime of the port. The device owns the buffer of the returned port. +""" +@inline function hostcall_open(c::HostcallClient, target::UInt64, flags::UInt32=UInt32(0)) + index = hostcall_start_index(c.nports) + while true + # under independent thread scheduling the lanes may reconverge with different + # indices, so re-read the mask and keep the index uniform + mask = active_mask() + index = broadcast_value(mask, index) + if try_lock!(c, mask, index) + # issue both loads before broadcasting either; they are independent + in_raw = mailbox_load(c.inbox + 4index) + out_raw = mailbox_load(c.outbox + 4index) + in = broadcast_value(mask, in_raw) + out = broadcast_value(mask, out_raw) + if hostcall_owned(in, out) + if is_first_lane(mask) + unsafe_store!(c.header + sizeof(HostcallHeader) * index, + HostcallHeader(mask, flags, target)) + end + sync_warp(mask) + return HostcallPort(c, mask, index, out, true) + end + # the port is free but its last call has not been serviced yet + unlock!(c, mask, index) + end + index += UInt32(1) + index >= c.nports && (index = UInt32(0)) + end +end + +""" + hostcall_lane_packet(port::HostcallPort) -> LLVMPtr{UInt8,AS.Global} + +Pointer to the calling lane's 64-byte packet of `port`. +""" +@inline hostcall_lane_packet(port::HostcallPort) = + port.client.packet + (port.index * HOSTCALL_LANES + (laneid() - 1i32)) * HOSTCALL_PACKET_SIZE + +""" + hostcall_send!(fill, port::HostcallPort) -> HostcallPort + +Wait until the device owns the buffer of `port`, call `fill(packet)` on every lane with a +pointer to the lane's packet, and hand the buffer to the host. Returns the updated port. + +Note that `fill` runs on the device; variables it captures must not be reassigned in the +enclosing function (use `let`), or they get boxed. +""" +@inline function hostcall_send!(fill::F, port::HostcallPort) where {F} + c = port.client + in = port.owns ? port.out : hostcall_load_inbox(c, port.mask, port.index) + hostcall_wait_for_ownership(c, port.mask, port.index, port.out, in) + fill(hostcall_lane_packet(port)) + out = hostcall_invert_outbox!(c, port.mask, port.index, port.out) + return HostcallPort(c, port.mask, port.index, out, false) +end + +""" + hostcall_recv!(use, port::HostcallPort) -> (HostcallPort, value, status) + +Wait until the host has handed the buffer of `port` back, and call `use(packet)` on every +lane with a pointer to the lane's packet. Returns the updated port, the value returned by +`use`, and the status word set by the host (`0` on success). +""" +@inline function hostcall_recv!(use::U, port::HostcallPort) where {U} + c = port.client + out = port.out + if port.owns + # consecutive receives: hand the buffer back first + out = hostcall_invert_outbox!(c, port.mask, port.index, out) + end + in = hostcall_load_inbox(c, port.mask, port.index) + in = hostcall_wait_for_ownership(c, port.mask, port.index, out, in) + val = use(hostcall_lane_packet(port)) + return HostcallPort(c, port.mask, port.index, out, true), val, in >> 1 +end + +""" + hostcall_close!(port::HostcallPort) + +Release `port`. If the host still owns the buffer (i.e. after a send without a matching +receive) the call is completed asynchronously by the host. +""" +@inline function hostcall_close!(port::HostcallPort) + sync_warp(port.mask) + unlock!(port.client, port.mask, port.index) + return +end + + +## value marshalling + +# values are shipped in their Julia layout, 64 bytes per packet; larger values are split +# over consecutive sends (the host knows the type, and thus the number of chunks) + +@inline function hostcall_send_value!(port::HostcallPort, x::T) where {T} + if sizeof(T) <= HOSTCALL_PACKET_SIZE + port = hostcall_send!(port) do pkt + unsafe_store!(reinterpret(LLVMPtr{T,AS.Global}, pkt), x) + end + else + # stream the value through local memory; all lanes send the same number of chunks + ref = Ref(x) + GC.@preserve ref begin + src = reinterpret(LLVMPtr{UInt8,AS.Generic}, Base.unsafe_convert(Ptr{T}, ref)) + nchunks = cld(sizeof(T), HOSTCALL_PACKET_SIZE) + i = 0 + while i < nchunks + offset = i * HOSTCALL_PACKET_SIZE + nbytes = min(HOSTCALL_PACKET_SIZE, sizeof(T) - offset) + port = let offset = offset, nbytes = nbytes, src = src + hostcall_send!(port) do pkt + j = 0 + while j < nbytes + unsafe_store!(pkt + j, unsafe_load(src + offset + j)) + j += 1 + end + end + end + i += 1 + end + end + end + return port +end + +@inline function hostcall_recv_value!(port::HostcallPort, ::Type{T}) where {T} + if sizeof(T) <= HOSTCALL_PACKET_SIZE + port, val, status = hostcall_recv!(port) do pkt + unsafe_load(reinterpret(LLVMPtr{T,AS.Global}, pkt)) + end + return port, val, status + else + ref = Ref{T}() + status = UInt32(0) + GC.@preserve ref begin + dst = reinterpret(LLVMPtr{UInt8,AS.Generic}, Base.unsafe_convert(Ptr{T}, ref)) + nchunks = cld(sizeof(T), HOSTCALL_PACKET_SIZE) + i = 0 + while i < nchunks + offset = i * HOSTCALL_PACKET_SIZE + nbytes = min(HOSTCALL_PACKET_SIZE, sizeof(T) - offset) + port, _, st = let offset = offset, nbytes = nbytes, dst = dst + hostcall_recv!(port) do pkt + j = 0 + while j < nbytes + unsafe_store!(dst + offset + j, unsafe_load(pkt + j)) + j += 1 + end + nothing + end + end + status |= st + # Error replies contain no result payload. In particular, the host may not + # know the result type of an unknown target and can only return one packet. + st == HOSTCALL_STATUS_OK || break + i += 1 + end + return port, ref[], status + end + end +end + + +## high-level API + +# Device-side argument conversion. Arguments travel in their Julia layout and may contain +# compiler-relocated host constants such as string literals. Device pointers are shipped as +# `CuPtr`, which is what the host expects. +hostconvert(x) = x +hostconvert(p::LLVMPtr{T}) where {T} = reinterpret(CuPtr{T}, p) + +# The hash is computed while compiling and travels with the image. The high bit keeps +# static targets disjoint from built-ins; the registry detects collisions. +function hostcall_target_id_value(@nospecialize(K::Type)) + return (hash(K) % UInt64) | HOSTCALL_STATIC_ID_BIT +end +@generated hostcall_target_id(::Type{K}) where {K} = :($(hostcall_target_id_value(K))) + +# the marker function the compiler scans for: `K` is the registry key +# (`Tuple{typeof(f), RT, AT}`). it must not be inlined so that its specializations show up +# in the compiled method instances, and must not throw. +@noinline function hostcall_impl(::Type{K}, ::Type{RT}, args::AT, + ::Val{async}) where {K, RT, AT<:Tuple, async} + client = hostcall_client() + flags = async ? HOSTCALL_FLAG_ASYNC : UInt32(0) + port = hostcall_open(client, hostcall_target_id(K), flags) + port = hostcall_send_value!(port, args) + if async + hostcall_close!(port) + return nothing + end + port, val, status = hostcall_recv_value!(port, RT) + hostcall_close!(port) + if status != HOSTCALL_STATUS_OK + # the handler failed; the host has recorded the error and will throw it at the next + # synchronization, so just stop this thread. + exit() + end + return val +end + +@inline hostcall_key(::F, ::Type{RT}, ::Type{AT}) where {F,RT,AT} = Tuple{F,RT,AT} + +# the values shipped for a call: the function itself unless it is a singleton +@inline hostcall_payload(f::F, args::Tuple) where {F} = + Base.issingletontype(F) ? args : (f, args...) + +""" + hostcall(f, R, args...) -> R + hostcall_async(f, args...) -> nothing + +Call the host function `f` with `args...` from device code, returning its result converted +to `R`. `hostcall_async` does not wait for the call to complete and implies `R === Nothing`. +`R` must be isbits or `Nothing`. Arguments may contain compiler-relocated host constants, +such as string literals, but arbitrary Julia references are unsupported. `f` must be +recoverable from its type (a named function or an isbits functor). See [`@hostcall`](@ref). +""" +@inline function hostcall(f::F, ::Type{RT}, args...) where {F,RT} + # Results are reconstructed on the device and cannot contain host references. Argument + # payloads may contain compiler-relocated constants such as string literals. + GPUCompiler.@static_assert(RT === Nothing || isbitstype(RT), + "hostcall return types must be isbits or Nothing") + payload = hostcall_payload(f, map(hostconvert, args)) + K = hostcall_key(f, RT, typeof(payload)) + return hostcall_impl(K, RT, payload, Val(false))::RT +end + +@inline function hostcall_async(f::F, args...) where {F} + payload = hostcall_payload(f, map(hostconvert, args)) + K = hostcall_key(f, Nothing, typeof(payload)) + hostcall_impl(K, Nothing, payload, Val(true)) + return nothing +end + +@doc (@doc hostcall) hostcall_async + +""" + @hostcall f(args...)::R + @hostcall async=true f(args...) + +Call the host function `f` from device code, `@ccall`-style: the return type annotation +`::R` is required (it is the one thing the device cannot know) unless `async=true`, in +which case the call returns immediately, `R` is `Nothing`, and the call is completed by the +time the next `synchronize()` returns. Individual arguments may be annotated (`a::T`) to +convert them before shipping. + +The call is warp-collective: all active lanes submit their own arguments and receive their +own results through a single port; divergent lanes simply form separate calls. Values are +shipped in their Julia layout. Arguments may contain compiler-relocated host constants such +as string literals, but arbitrary Julia references are unsupported; results must be isbits +or `Nothing`. Device memory is passed as explicit pointers (received as `CuPtr` on the host). +The handler runs on a dedicated host thread with the kernel's context active and a dedicated +non-blocking stream; handler exceptions surface as `HostcallException` at the next +`synchronize()`. + +```julia +function kernel(out, i) + y = @hostcall load_from_disk(i)::Float32 + @hostcall async=true println("thread ", i) + ... +end +``` +""" +macro hostcall(exprs...) + isempty(exprs) && throw(ArgumentError("@hostcall requires a call expression")) + call = exprs[end] + options = exprs[1:end-1] + + # options + async = false + for opt in options + Meta.isexpr(opt, :(=), 2) || + throw(ArgumentError("invalid @hostcall option `$opt`; expected `key=value`")) + key, val = opt.args + if key === :async + val isa Bool || + throw(ArgumentError("the `async` option of @hostcall requires a literal Bool")) + async = val + else + throw(ArgumentError("unknown @hostcall option `$key`")) + end + end + + # return type + rettype = nothing + if Meta.isexpr(call, :(::), 2) + rettype = call.args[2] + call = call.args[1] + end + if !async && rettype === nothing + throw(ArgumentError("@hostcall requires a return type annotation: `@hostcall f(args...)::R`, or `async=true`")) + end + if async && rettype !== nothing && rettype !== :Nothing + throw(ArgumentError("`@hostcall async=true` cannot return a value; drop the `::$rettype` annotation")) + end + Meta.isexpr(call, :call) || + throw(ArgumentError("@hostcall expects a function call, got `$call`")) + f = call.args[1] + args = call.args[2:end] + any(arg -> Meta.isexpr(arg, :parameters) || Meta.isexpr(arg, :kw), args) && + throw(ArgumentError("@hostcall does not support keyword arguments")) + + # per-argument conversions + argexprs = map(args) do arg + if Meta.isexpr(arg, :(::), 2) + :(convert($(esc(arg.args[2])), $(esc(arg.args[1])))) + else + esc(arg) + end + end + + if async + :(hostcall_async($(esc(f)), $(argexprs...))) + else + :(hostcall($(esc(f)), $(esc(rettype)), $(argexprs...))) + end +end diff --git a/CUDACore/src/device/quirks.jl b/CUDACore/src/device/quirks.jl index 7e8aedcf90..61a728e7b2 100644 --- a/CUDACore/src/device/quirks.jl +++ b/CUDACore/src/device/quirks.jl @@ -9,7 +9,7 @@ macro gputhrow(subtype, reason) quote - info = kernel_state().exception_info + info = exception_info() info.subtype = @strptr $subtype info.reason = @strptr $reason throw(nothing) diff --git a/CUDACore/src/device/runtime.jl b/CUDACore/src/device/runtime.jl index bd041651f6..d9ba82229e 100644 --- a/CUDACore/src/device/runtime.jl +++ b/CUDACore/src/device/runtime.jl @@ -143,7 +143,7 @@ end function report_exception(ex) # this is the first reporting function being called, so claim the exception - info = kernel_state().exception_info + info = exception_info() if lock_output!(info) # override the exception type GPUCompiler deduced if the user provided a subtype if info.subtype != C_NULL @@ -160,7 +160,7 @@ function report_exception(ex) end function report_exception_name(ex) - info = kernel_state().exception_info + info = exception_info() # this is the first reporting function being called, so claim the exception if lock_output!(info) @@ -179,7 +179,7 @@ function report_exception_name(ex) end function report_exception_frame(idx, func, file, line) - info = kernel_state().exception_info + info = exception_info() if lock_output!(info) @cuprintf(" [%d] %s at %s:%d\n", idx, func, file, line) @@ -188,7 +188,7 @@ function report_exception_frame(idx, func, file, line) end function signal_exception() - info = kernel_state().exception_info + info = exception_info() # finalize output if lock_output!(info) @@ -210,12 +210,18 @@ end ## kernel state struct KernelState - exception_info::ExceptionInfo + client::HostcallClientPtr random_seed::UInt32 end @inline @generated kernel_state() = GPUCompiler.kernel_state_value(KernelState) +@inline function hostcall_client() + return unsafe_load(kernel_state().client) +end + +@inline exception_info() = reinterpret(ExceptionInfo, hostcall_client().exception_info) + ## other diff --git a/test/core/device/hostcall.jl b/test/core/device/hostcall.jl new file mode 100644 index 0000000000..60112d260e --- /dev/null +++ b/test/core/device/hostcall.jl @@ -0,0 +1,264 @@ +using CUDA: HostcallClient, HostcallPort, HostcallHeader, + hostcall_open, hostcall_send!, hostcall_recv!, hostcall_close!, + hostcall_lane_packet, HOSTCALL_PACKET_SIZE, hostcall_packet_layout + +@testset "@hostcall syntax" begin + @test_throws ArgumentError hostcall_packet_layout(-1) + f(x) = x + # the return type is required unless async + @test_throws LoadError @eval @hostcall f(1) + @test_throws LoadError @eval @hostcall async=true f(1)::Int + @test_throws LoadError @eval @hostcall lanes=:all f(1)::Int + @test_throws LoadError @eval @hostcall f(1; y=2)::Int + ex = @macroexpand @hostcall f(1, 2.0)::Int + @test Meta.isexpr(ex, :call) && ex.args[1] == GlobalRef(CUDACore, :hostcall) + @test ex.args[3] == :Int + ex = @macroexpand @hostcall async=true f(1) + @test Meta.isexpr(ex, :call) && ex.args[1] == GlobalRef(CUDACore, :hostcall_async) + ex = @macroexpand @hostcall async=true f(1)::Nothing + @test Meta.isexpr(ex, :call) && ex.args[1] == GlobalRef(CUDACore, :hostcall_async) + ex = @macroexpand @hostcall f(x::Float32)::Int + @test Meta.isexpr(ex.args[4], :call) && ex.args[4].args[1].name == :convert + + # non-isbits return types are rejected when the kernel is compiled + badrt() = (@hostcall string(1)::String; nothing) + @test_throws "hostcall return types must be isbits" @cuda launch=false badrt() +end + +# a minimal host-side implementation of the port protocol, so that the device-side API can +# be tested without the hostcall service: the test polls the area on the main thread. +module TestPoller + using CUDA, CUDACore + using CUDA: HostcallClient, HostcallHeader, HOSTCALL_PACKET_SIZE, hostcall_packet_layout + using Core: LLVMPtr + const AS = CUDA.AS + + mutable struct Area + nports::Int + mem::CUDACore.HostMemory + base::Ptr{UInt8} + locks::CuVector{UInt32} + client::HostcallClient + layout::Any + end + + function Area(nports) + layout = hostcall_packet_layout(nports) + mem = CUDACore.alloc(CUDACore.HostMemory, layout.total, + CUDACore.MEMHOSTALLOC_DEVICEMAP | CUDACore.MEMHOSTALLOC_PORTABLE) + base = convert(Ptr{UInt8}, mem) + @assert UInt(base) == UInt(convert(CuPtr{UInt8}, mem)) + unsafe_wrap(Array, base, layout.total) .= 0 + locks = CUDA.zeros(UInt32, cld(nports, 32)) + dp(off, T) = reinterpret(LLVMPtr{T,AS.Global}, base + off) + client = HostcallClient(nports, dp(layout.inbox, UInt32), dp(layout.outbox, UInt32), + dp(layout.header, HostcallHeader), dp(layout.packet, UInt8), + reinterpret(LLVMPtr{UInt32,AS.Global}, pointer(locks))) + Area(nports, mem, base, locks, client, layout) + end + free(a::Area) = CUDACore.free(a.mem) + + inbox(a, i) = convert(Ptr{UInt32}, a.base + a.layout.inbox) + 4i + outbox(a, i) = convert(Ptr{UInt32}, a.base + a.layout.outbox) + 4i + header(a, i) = unsafe_load(convert(Ptr{HostcallHeader}, a.base + a.layout.header) + sizeof(HostcallHeader) * i) + packet(a, i, lane) = a.base + a.layout.packet + (i * 32 + lane) * HOSTCALL_PACKET_SIZE + + load(p::Ptr{UInt32}) = Core.Intrinsics.atomic_pointerref(p, :acquire) + store!(p::Ptr{UInt32}, v::UInt32) = Core.Intrinsics.atomic_pointerset(p, v, :release) + + # one sweep; `handle(target, lane, packet_ptr)` services every live lane of a pending + # port and returns the status to report. returns the number of ports serviced. + function sweep!(handle, a::Area) + n = 0 + for i in 0:a.nports-1 + out = load(outbox(a, i)) + in = load(inbox(a, i)) + (in & 1) == out && continue + hdr = header(a, i) + status = UInt32(0) + for lane in 0:31 + (hdr.mask >> lane) & 1 == 0 && continue + status |= UInt32(handle(hdr.target, lane, packet(a, i, lane))) + end + store!(inbox(a, i), out | (status << 1)) + n += 1 + end + return n + end + + # poll until the current stream is done and everything has been drained + function serve!(handle, a::Area; timeout=30) + t0 = time() + while true + n = sweep!(handle, a) + if n == 0 + CUDA.isdone(stream()) && break + ccall(:jl_cpu_pause, Cvoid, ()) + time() - t0 > timeout && error("timeout waiting for the kernel") + end + end + sweep!(handle, a) + return + end +end + +@testset "raw ports" begin + area = TestPoller.Area(64) + try + OP_ADD1 = UInt64(1) + OP_SUM4 = UInt64(2) + OP_RECORD = UInt64(3) + OP_FAIL = UInt64(4) + records = Threads.Atomic{Int}(0) + function handler(target, lane, pkt) + p = convert(Ptr{UInt64}, pkt) + if target == OP_ADD1 + unsafe_store!(p, unsafe_load(p) + 1) + elseif target == OP_SUM4 + unsafe_store!(p, sum(unsafe_load(p, i) for i in 1:4)) + elseif target == OP_RECORD + Threads.atomic_add!(records, 1) + elseif target == OP_FAIL + return 1 + end + return 0 + end + + # blocking round trips, every lane its own value + function roundtrip(client, n, out) + v = UInt64((blockIdx().x - 1) * blockDim().x + threadIdx().x) + i = 0 + while i < n + port = hostcall_open(client, OP_ADD1) + port = let v = v + hostcall_send!(port) do pkt + unsafe_store!(reinterpret(Core.LLVMPtr{UInt64,AS.Global}, pkt), v) + end + end + port, v, status = hostcall_recv!(port) do pkt + unsafe_load(reinterpret(Core.LLVMPtr{UInt64,AS.Global}, pkt)) + end + hostcall_close!(port) + i += 1 + end + out[(blockIdx().x - 1) * blockDim().x + threadIdx().x] = v + return + end + for (threads, blocks) in [(32, 1), (20, 2), (256, 4)] + out = CUDA.zeros(UInt64, threads * blocks) + @cuda threads=threads blocks=blocks roundtrip(area.client, 3, out) + TestPoller.serve!(handler, area) + synchronize() + @test Array(out) == UInt64.(1:threads*blocks) .+ 3 + end + + # divergent call sites within a warp + function divergent(client, out) + t = threadIdx().x + if isodd(t) + port = hostcall_open(client, OP_SUM4) + port = hostcall_send!(port) do pkt + p = reinterpret(Core.LLVMPtr{UInt64,AS.Global}, pkt) + unsafe_store!(p, UInt64(t), 1) + unsafe_store!(p, UInt64(10), 2) + unsafe_store!(p, UInt64(100), 3) + unsafe_store!(p, UInt64(1000), 4) + end + port, r, _ = hostcall_recv!(port) do pkt + unsafe_load(reinterpret(Core.LLVMPtr{UInt64,AS.Global}, pkt)) + end + hostcall_close!(port) + out[t] = r + else + port = hostcall_open(client, OP_ADD1) + port = hostcall_send!(port) do pkt + unsafe_store!(reinterpret(Core.LLVMPtr{UInt64,AS.Global}, pkt), UInt64(t)) + end + port, r, _ = hostcall_recv!(port) do pkt + unsafe_load(reinterpret(Core.LLVMPtr{UInt64,AS.Global}, pkt)) + end + hostcall_close!(port) + out[t] = r + end + return + end + out = CUDA.zeros(UInt64, 32) + @cuda threads=32 divergent(area.client, out) + TestPoller.serve!(handler, area) + synchronize() + @test Array(out) == [isodd(t) ? UInt64(t + 1110) : UInt64(t + 1) for t in 1:32] + + # asynchronous sends: the port is released without waiting for a reply + function record(client, n) + i = 0 + while i < n + port = hostcall_open(client, OP_RECORD) + port = let i = i + hostcall_send!(port) do pkt + unsafe_store!(reinterpret(Core.LLVMPtr{Int,AS.Global}, pkt), i) + end + end + hostcall_close!(port) + i += 1 + end + return + end + records[] = 0 + @cuda threads=64 blocks=8 record(area.client, 5) + TestPoller.serve!(handler, area) + synchronize() + @test records[] == 64 * 8 * 5 + + # a status word set by the host is returned by recv! + function failing(client, out) + port = hostcall_open(client, OP_FAIL) + port = hostcall_send!(port) do pkt end + port, _, status = hostcall_recv!(port) do pkt nothing end + hostcall_close!(port) + out[threadIdx().x] = status + return + end + out = CUDA.zeros(UInt32, 4) + @cuda threads=4 failing(area.client, out) + TestPoller.serve!(handler, area) + synchronize() + @test all(==(1), Array(out)) + + # all ports are released afterwards + @test all(==(0), Array(area.locks)) + finally + TestPoller.free(area) + end +end + +@testset "memory model" begin + # the mailbox accesses should be system-scope, the fences and sleeps present + function probe(client, out) + port = hostcall_open(client, UInt64(1)) + port = hostcall_send!(port) do pkt end + port, v, _ = hostcall_recv!(port) do pkt + unsafe_load(reinterpret(Core.LLVMPtr{UInt64,AS.Global}, pkt)) + end + hostcall_close!(port) + out[1] = v + return + end + tt = Tuple{HostcallClient, CuDeviceVector{UInt64,1}} + modern = sprint(io -> CUDA.code_ptx(io, probe, tt; arch=sm"70")) + @test occursin("ld.relaxed.sys.global.u32", modern) + @test occursin("st.relaxed.sys.global.u32", modern) + @test occursin("fence.acq_rel.sys", modern) + @test occursin("nanosleep", modern) + + legacy = sprint(io -> CUDA.code_ptx(io, probe, tt; arch=sm"60")) + @test occursin("ld.volatile.global.u32", legacy) + @test occursin("st.volatile.global.u32", legacy) + @test occursin("membar.sys", legacy) + @test !occursin("nanosleep", legacy) + + for ptx in (modern, legacy) + @test occursin(r"atom\.(global\.)?or\.b32", ptx) + @test occursin("activemask", ptx) + @test occursin("bar.warp.sync", ptx) + end +end From be1f1d870f6dd3971f3b93b33cbabae1d48cfb4a Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Mon, 24 Aug 2026 09:44:19 +0200 Subject: [PATCH 02/10] Add the hostcall host service. Service per-context hostcall areas from a foreign libuv thread so calls progress while Julia threads block in CUDA. The server polls while launches are armed, backs off when idle, and uses cuLaunchHostFunc only to post a semaphore. Accept compiler-described static targets and precompile handlers before the server runs them. Handlers use the calling context and a dedicated non-blocking stream; failures, deferred output and asynchronous completion surface at synchronization. --- CUDACore/lib/cudadrv/context.jl | 4 + CUDACore/lib/cudadrv/module.jl | 5 +- CUDACore/lib/cudadrv/synchronization.jl | 13 +- CUDACore/src/CUDACore.jl | 1 + CUDACore/src/compiler/exceptions.jl | 46 +- CUDACore/src/compiler/execution.jl | 3 +- CUDACore/src/hostcall.jl | 931 ++++++++++++++++++++++++ test/core/hostcall.jl | 8 + 8 files changed, 994 insertions(+), 17 deletions(-) create mode 100644 CUDACore/src/hostcall.jl create mode 100644 test/core/hostcall.jl diff --git a/CUDACore/lib/cudadrv/context.jl b/CUDACore/lib/cudadrv/context.jl index 54b8e099b7..86c88e3609 100644 --- a/CUDACore/lib/cudadrv/context.jl +++ b/CUDACore/lib/cudadrv/context.jl @@ -95,6 +95,8 @@ respect any users of the context, and might make other objects unusable. """ function unsafe_destroy!(ctx::CuContext) if isvalid(ctx) + hostcall_forget!(ctx) + forget_exceptions!(ctx) cuCtxDestroy_v2(ctx) end end @@ -226,6 +228,8 @@ in the current process. Note that this forcibly invalidates all contexts derived primary context, and as a result outstanding resources might become invalid. """ function unsafe_reset!(pctx::CuPrimaryContext) + hostcall_forget!(pctx.dev) + forget_exceptions!(pctx.dev) cuDevicePrimaryCtxReset_v2(pctx.dev) return end diff --git a/CUDACore/lib/cudadrv/module.jl b/CUDACore/lib/cudadrv/module.jl index e5ffa2ce37..34131c0a8f 100644 --- a/CUDACore/lib/cudadrv/module.jl +++ b/CUDACore/lib/cudadrv/module.jl @@ -13,7 +13,10 @@ function checked_cuModuleLoadDataEx(_module, image, numOptions, options, optionV # available, but cached by the allocator. by configuring the allocator with a # release threshold, we have it actually free up that memory, but that requires # synchronizing all streams to make sure pending frees are actually executed. - if !is_capturing() + # + # inside a hostcall handler we must not synchronize the device: it would wait for + # the kernel that is waiting for the handler. + if !is_capturing() && !hostcall_in_handler() device_synchronize() end diff --git a/CUDACore/lib/cudadrv/synchronization.jl b/CUDACore/lib/cudadrv/synchronization.jl index 60b02969c1..c1c0202a1f 100644 --- a/CUDACore/lib/cudadrv/synchronization.jl +++ b/CUDACore/lib/cudadrv/synchronization.jl @@ -183,7 +183,7 @@ function nonblocking_synchronize(val) end function device_synchronize(; blocking::Bool=false, spin::Bool=true) - if use_nonblocking_synchronization && !blocking + if use_nonblocking_synchronization && !blocking && !hostcall_in_handler() if spin && spinning_synchronization(isdone, legacy_stream()) cuCtxSynchronize() else @@ -199,7 +199,10 @@ function device_synchronize(; blocking::Bool=false, spin::Bool=true) end function synchronize(stream::CuStream=stream(); blocking::Bool=false, spin::Bool=true) - if use_nonblocking_synchronization && !blocking + # the special streams (default, legacy, per-thread) belong to the current context + ctx = something(stream.ctx, context()) + # hostcall handlers run on a foreign thread that must not wait on the Julia scheduler + if use_nonblocking_synchronization && !blocking && !hostcall_in_handler() if spin && spinning_synchronization(isdone, stream) cuStreamSynchronize(stream) else @@ -211,11 +214,11 @@ function synchronize(stream::CuStream=stream(); blocking::Bool=false, spin::Bool cuStreamSynchronize(stream) end - check_exceptions() + check_exceptions(ctx) end function synchronize(event::CuEvent; blocking::Bool=false, spin::Bool=true) - if use_nonblocking_synchronization && !blocking + if use_nonblocking_synchronization && !blocking && !hostcall_in_handler() if spin && spinning_synchronization(isdone, event) cuEventSynchronize(event) else @@ -226,4 +229,6 @@ function synchronize(event::CuEvent; blocking::Bool=false, spin::Bool=true) maybe_collect(true) cuEventSynchronize(event) end + + check_exceptions(event.ctx) end diff --git a/CUDACore/src/CUDACore.jl b/CUDACore/src/CUDACore.jl index 8919d33242..8120475e92 100644 --- a/CUDACore/src/CUDACore.jl +++ b/CUDACore/src/CUDACore.jl @@ -105,6 +105,7 @@ include("refpointer.jl") include("compiler/compilation.jl") include("compiler/execution.jl") include("compiler/exceptions.jl") +include("hostcall.jl") # array implementation include("utilities.jl") diff --git a/CUDACore/src/compiler/exceptions.jl b/CUDACore/src/compiler/exceptions.jl index 29da71a036..276f6e2c3f 100644 --- a/CUDACore/src/compiler/exceptions.jl +++ b/CUDACore/src/compiler/exceptions.jl @@ -38,18 +38,42 @@ function create_exceptions!(mod::CuModule) return exception_info, client end -# check the exception flags on every API call, similarly to how CUDA handles errors -function check_exceptions() - for (ctx,mem) in exception_infos - exception_info = convert(ExceptionInfo, mem) - if exception_info.status != 0 - # restore the structure - unsafe_store!(exception_info, ExceptionInfo_st()) - - # throw host-side - dev = device(ctx) - throw(KernelException(dev)) +# check the exception flags of a context after synchronizing it, similarly to how CUDA +# handles errors. exceptions are reported per context: synchronizing one device does not +# surface exceptions from kernels on another device, and the flag memory of a context that +# has been destroyed (`unsafe_reset!`) is never touched again. +function check_exceptions(ctx::CuContext=context()) + hostcall_synchronize(ctx) + mem = @lock exception_infos_lock get(exception_infos, ctx, nothing) + mem === nothing && return + exception_info = convert(ExceptionInfo, mem) + if exception_info.status != 0 + # restore the structure + unsafe_store!(exception_info, ExceptionInfo_st()) + + # throw host-side + dev = device(ctx) + throw(KernelException(dev)) + end + return +end + +# Drop exception state before destroying a context. The flag is raw pinned memory and must +# be released explicitly. +function forget_exceptions!(pred) + forgotten = @lock exception_infos_lock begin + entries = Pair{CuContext,HostMemory}[] + for (ctx, mem) in collect(exception_infos) + pred(ctx) || continue + push!(entries, ctx => mem) + delete!(exception_infos, ctx) end + entries + end + for (_, mem) in forgotten + free(mem) end return end +forget_exceptions!(ctx::CuContext) = forget_exceptions!(candidate -> candidate == ctx) +forget_exceptions!(dev::CuDevice) = forget_exceptions!(ctx -> device(ctx) == dev) diff --git a/CUDACore/src/compiler/execution.jl b/CUDACore/src/compiler/execution.jl index 5d3e70c0a3..aa43a913bb 100644 --- a/CUDACore/src/compiler/execution.jl +++ b/CUDACore/src/compiler/execution.jl @@ -710,7 +710,8 @@ function cufunction(f::F, tt::TT=Tuple{}; kwargs...) where {F,TT} kernel = get(_kernel_instances, key, nothing) if kernel === nothing # create the kernel state object - _, client = create_exceptions!(fun.mod) + exception_info, fallback = create_exceptions!(fun.mod) + client = hostcall_client(ctx, cuda.device, exception_info, fallback) state = KernelState(client, UInt32(0)) kernel = HostKernel{F,tt}(f, fun, state) diff --git a/CUDACore/src/hostcall.jl b/CUDACore/src/hostcall.jl new file mode 100644 index 0000000000..f3c69b6547 --- /dev/null +++ b/CUDACore/src/hostcall.jl @@ -0,0 +1,931 @@ +# Hostcall: host-side service +# +# The device-side protocol lives in `device/intrinsics/hostcall.jl`. This file provides the +# pinned memory area that backs it (one per context, created lazily), the target registry, +# and the server: a single foreign libuv thread that sweeps every area while kernels that +# may hostcall are in flight, and wakes up periodically otherwise. The server thread is +# independent of Julia's scheduler, so hostcalls keep being serviced while the launching +# thread is blocked in a CUDA call. + +@public HostcallException, HostcallArea, hostcall_area, hostcall_drain, hostcall_available + +using Preferences: @load_preference + + +## preferences + +# whether hostcalls are enabled at all +const hostcall_enabled = @load_preference("hostcall", true)::Bool + +# the number of ports per area; defaults to the number of resident warps of the device +const hostcall_ports_pref = @load_preference("hostcall_ports", nothing) + +""" + hostcall_available([dev::CuDevice]) -> Bool + +Whether hostcalls (and thus hostcall-based functionality such as device exception +reporting) are available on `dev`. Controlled by the `hostcall` preference. Note that on +devices with a display watchdog (Windows WDDM, or a display attached), kernels blocked in +a hostcall remain subject to that watchdog, like any other running kernel. +""" +function hostcall_available(::CuDevice=device()) + hostcall_enabled || return false + get(ENV, "JULIA_CUDA_HOSTCALL", "true") == "false" && return false + return true +end + + +## area + +""" + HostcallArea + +Host-side owner of a hostcall area: pinned, device-mapped memory for the mailboxes, +headers and packets of `nports` ports, the lock bitfield in device memory, and the mapped +`HostcallClient` descriptor referenced by kernels. Exception-only areas are polled by the +server heartbeat; other areas are polled while an ordinary hostcall kernel is active. +Areas are created per context by [`hostcall_area`](@ref). +""" +mutable struct HostcallArea + const ctx::CuContext + const dev::CuDevice + const nports::Int + const mem::HostMemory + const base::Ptr{UInt8} + const layout::@NamedTuple{client::Int, inbox::Int, outbox::Int, header::Int, + packet::Int, total::Int} + const locks::DeviceMemory + const client::HostcallClientPtr + const heartbeat::Bool # poll while idle (exception-only kernels use this area) + + # host-side sweep state + const shadow::Vector{UInt32} # the inbox words we last wrote (host is the only writer) + const outbox::Vector{UInt32} # view of the pinned outbox array + cursor::Int # sweeps resume after the last serviced port + + # handlers run on a dedicated non-blocking stream, so that they never wait on the + # stream the calling kernel is running on + stream::Union{Nothing,CuStream} +end + +const HOSTCALL_SWEEP_CHUNK = 64 +const HOSTCALL_MIN_PORTS = 64 +const HOSTCALL_MAX_PORTS = 4096 + +function HostcallArea(ctx::CuContext, nports::Integer, exception_info::ExceptionInfo; + heartbeat::Bool=true) + nports > 0 || throw(ArgumentError("the number of hostcall ports must be positive")) + nports <= typemax(UInt32) - (HOSTCALL_SWEEP_CHUNK - 1) || + throw(ArgumentError("too many hostcall ports: $nports")) + nports = cld(Int(nports), HOSTCALL_SWEEP_CHUNK) * HOSTCALL_SWEEP_CHUNK + layout = hostcall_packet_layout(nports) + # look up the device through the context: `context!` does not activate a context that + # is already the task's current one, so `current_device()` may not work here yet + dev = device(ctx) + context!(ctx) do + mem = alloc(HostMemory, layout.total, MEMHOSTALLOC_DEVICEMAP | MEMHOSTALLOC_PORTABLE) + base = convert(Ptr{UInt8}, mem) + # the protocol assumes a unified address space: the same pointer on both sides + UInt(base) == UInt(convert(CuPtr{UInt8}, mem)) || + error("Hostcall requires unified addressing (host and device pointers to pinned memory differ)") + unsafe_wrap(Array, base, layout.total) .= 0 + locks = alloc(DeviceMemory, 4 * cld(nports, 32)) + cuMemsetD32_v2(locks, 0, cld(nports, 32)) + dp(off, T) = reinterpret(LLVMPtr{T,AS.Global}, base + off) + descriptor = HostcallClient(nports, dp(layout.inbox, UInt32), + dp(layout.outbox, UInt32), + dp(layout.header, HostcallHeader), + dp(layout.packet, UInt8), + reinterpret(LLVMPtr{UInt32,AS.Global}, + convert(CuPtr{UInt32}, locks)), + reinterpret(Ptr{Cvoid}, exception_info)) + unsafe_store!(convert(Ptr{HostcallClient}, base + layout.client), descriptor) + client = dp(layout.client, HostcallClient) + shadow = Base.zeros(UInt32, nports) + outbox = unsafe_wrap(Array, convert(Ptr{UInt32}, base + layout.outbox), nports) + HostcallArea(ctx, dev, nports, mem, base, layout, locks, client, heartbeat, + shadow, outbox, 0, nothing) + end +end + +inbox_ptr(a::HostcallArea, i) = convert(Ptr{UInt32}, a.base + a.layout.inbox) + 4i +outbox_ptr(a::HostcallArea, i) = convert(Ptr{UInt32}, a.base + a.layout.outbox) + 4i +header_ptr(a::HostcallArea, i) = + convert(Ptr{HostcallHeader}, a.base + a.layout.header) + sizeof(HostcallHeader) * i +packet_ptr(a::HostcallArea, i, lane) = + a.base + a.layout.packet + (i * HOSTCALL_LANES + lane) * HOSTCALL_PACKET_SIZE + +mailbox_load(p::Ptr{UInt32}) = Core.Intrinsics.atomic_pointerref(p, :acquire) +mailbox_store!(p::Ptr{UInt32}, v::UInt32) = Core.Intrinsics.atomic_pointerset(p, v, :release) + +function hostcall_stream(a::HostcallArea) + s = a.stream + if s === nothing + s = context!(a.ctx) do + CuStream(; flags=STREAM_NON_BLOCKING) + end + a.stream = s + end + return s +end + +# the default number of ports: enough for every resident warp, so that a warp never has to +# wait for another warp to release a port (GPUs have no forward-progress guarantee) +function hostcall_default_ports(dev::CuDevice) + if hostcall_ports_pref !== nothing + return Int(hostcall_ports_pref) + end + sms = attribute(dev, DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT) + warps = attribute(dev, DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR) ÷ 32 + return clamp(sms * warps, HOSTCALL_MIN_PORTS, HOSTCALL_MAX_PORTS) +end + +# all areas, as an immutable snapshot that the server thread can read without locking +mutable struct HostcallAreas + Base.@atomic list::Vector{HostcallArea} +end +const hostcall_areas = HostcallAreas(HostcallArea[]) +const hostcall_areas_lock = ReentrantLock() + +""" + hostcall_area(ctx, exception_info; ports=HOSTCALL_MIN_PORTS) -> HostcallArea + +Return the hostcall area of `ctx` with at least `ports` ports, creating it (and starting +the server thread) if necessary. Kernels that do not call host functions only need a small +area (for exception reporting), so contexts start with a minimal one and get a larger area +when a kernel that uses hostcalls is linked; earlier areas stay alive and keep being +serviced, because running kernels may still refer to them. +""" +function hostcall_area(ctx::CuContext, exception_info::ExceptionInfo; + ports::Integer=HOSTCALL_MIN_PORTS, heartbeat::Bool=true) + @lock hostcall_areas_lock begin + areas = Base.@atomic hostcall_areas.list + # newest first: the last area created for a context is the largest + for area in Iterators.reverse(areas) + area.ctx == ctx && area.nports >= ports && + (!heartbeat || area.heartbeat) && return area + end + area = HostcallArea(ctx, ports, exception_info; heartbeat) + hostcall_server_start() + Base.@atomic hostcall_areas.list = [areas..., area] + return area + end +end + +""" + hostcall_client(ctx, dev, exception_info, fallback; ports, heartbeat) + +Pointer to a suitable runtime descriptor for kernels launched in `ctx`. When hostcalls are +unavailable, `fallback` is returned so device exception handling remains usable. +""" +function hostcall_client(ctx::CuContext, dev::CuDevice, exception_info::ExceptionInfo, + fallback::HostcallClientPtr; + ports::Integer=HOSTCALL_MIN_PORTS, heartbeat::Bool=true) + hostcall_available(dev) || return fallback + # kernels compiled during precompilation are not launched; creating the area would + # start the server thread in the precompilation process + ccall(:jl_generating_output, Cint, ()) != 0 && return fallback + return hostcall_area(ctx, exception_info; ports, heartbeat).client +end + + +## target registry + +# a statically-known host function and how its arguments and result are encoded +struct HostcallTarget + key::Type # registry key `Tuple{F,RT,AT}` + f::Any + RT::Type # return type (`Nothing` for calls without a reply) + AT::Type # payload tuple type; includes `f` when it is not stored + stored::Bool # callable is stored in `f`, rather than included in the payload +end + +const hostcall_targets = Dict{UInt64,HostcallTarget}() +const hostcall_targets_lock = Threads.SpinLock() + +""" + register_hostcall_targets!(targets) + +Register the statically-known hostcall targets of a kernel (pairs of identifier and key +type `Tuple{F,RT,AT}`, as recorded by the compiler) so that the server can dispatch calls. +""" +function register_hostcall_targets!(targets) + isempty(targets) && return + for (id, K) in targets + F, RT, AT = K.parameters + stored = Base.issingletontype(F) + f = stored ? F.instance : nothing + target = HostcallTarget(K, f, RT, AT, stored) + + # Compile the handler on the registering thread, before taking the registry lock. + precompile(stored ? Tuple{F, AT.parameters...} : AT) + @lock hostcall_targets_lock begin + previous = get(hostcall_targets, id, nothing) + if previous === nothing + hostcall_targets[id] = target + elseif previous.key !== K + error("hostcall target identifier collision between $(previous.key) and $K") + end + end + end + return +end + +hostcall_target(id::UInt64) = @lock hostcall_targets_lock get(hostcall_targets, id, nothing) + +# conversion of values received from the device (pointers already arrive as `CuPtr`, see +# the device-side `hostconvert`) +hostconvert_host(@nospecialize(x)) = x +function hostconvert_host(@nospecialize(p::LLVMPtr)) + T = typeof(p).parameters[1] + return load_bits(CuPtr{T}, reinterpret(Ptr{UInt8}, ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), p))) +end + + +## exceptions and deferred output + +""" + HostcallException + +Thrown by `synchronize()` when a host function called from a kernel threw, when the +target of a call is unknown, or when its result could not be converted. The calling device +thread has been stopped. The original error and backtrace are available in the `error` +and `backtrace` fields, and the device the call came from in `device` (`nothing` for errors +in the server itself). Like device-side exceptions, these are reported by synchronizing the +context the kernel ran in. +""" +struct HostcallException <: Exception + target::Any + error::Any + backtrace::Any + device::Union{Nothing,CuDevice} +end +HostcallException(target, error, backtrace=nothing) = + HostcallException(target, error, backtrace, nothing) + +function Base.showerror(io::IO, err::HostcallException) + print(io, "HostcallException: error while servicing a hostcall to ", err.target) + err.device === nothing || print(io, " on device ", name(err.device)) + print(io, ":\n") + showerror(io, err.error) + if err.backtrace !== nothing + println(io) + Base.show_backtrace(io, err.backtrace) + end +end + +# pending exceptions, tagged with the context of the area they were recorded for (or +# `nothing` for errors in the server itself, which any context reports) +const hostcall_exceptions = Tuple{Union{Nothing,CuContext},HostcallException}[] +const hostcall_exceptions_lock = Threads.SpinLock() +const hostcall_exceptions_pending = Threads.Atomic{Int}(0) +# Aggregate work that must be observed at synchronization. Keeping the fast path to one +# atomic load avoids charging ordinary synchronization for each hostcall subsystem. +const hostcall_sync_pending = Threads.Atomic{Int}(0) + +function record_hostcall_exception!(area::Union{Nothing,HostcallArea}, target, err, bt=nothing) + ctx = area === nothing ? nothing : area.ctx + dev = area === nothing ? nothing : area.dev + @lock hostcall_exceptions_lock begin + push!(hostcall_exceptions, (ctx, HostcallException(target, err, bt, dev))) + Threads.atomic_add!(hostcall_exceptions_pending, 1) + Threads.atomic_add!(hostcall_sync_pending, 1) + end + return +end + +# throw the oldest pending exception of `ctx` (called from `check_exceptions`). exceptions +# are reported per context, like device-side exceptions: synchronizing one device does not +# surface the errors of kernels running on another. +function check_hostcall_exceptions(ctx::CuContext) + hostcall_exceptions_pending[] == 0 && return + err = @lock hostcall_exceptions_lock begin + i = findfirst(((c, _),) -> c === nothing || c == ctx, hostcall_exceptions) + i === nothing && return + Threads.atomic_sub!(hostcall_exceptions_pending, 1) + Threads.atomic_sub!(hostcall_sync_pending, 1) + popat!(hostcall_exceptions, i)[2] + end + throw(err) +end + +# output from the print family is not written by the server thread (libuv I/O from a +# foreign thread can hang while the main thread is blocked in a ccall, julia#55525), but +# queued and emitted by a Julia task, or by whoever drains the hostcall area next. +const hostcall_output = IOBuffer() +const hostcall_output_lock = Threads.SpinLock() +const hostcall_output_pending = Threads.Atomic{Int}(0) +const hostcall_output_cond = Ref{Union{Nothing,Base.AsyncCondition}}(nothing) +const hostcall_output_cond_lock = ReentrantLock() + +is_print_target(@nospecialize(f)) = + f === print || f === println || f === printstyled || f === show || f === display + +function queue_hostcall_output(f, args...) + io = IOBuffer() + if f === display + show(io, MIME"text/plain"(), args...) + println(io) + else + f(io, args...) + end + data = take!(io) + @lock hostcall_output_lock begin + write(hostcall_output, data) + Threads.atomic_add!(hostcall_output_pending, 1) + Threads.atomic_add!(hostcall_sync_pending, 1) + end + cond = hostcall_output_cond[] + if cond !== nothing + # wakes the printer task, if thread 1 is free to run it; otherwise the output is + # flushed by the next synchronization + ccall(:uv_async_send, Cint, (Ptr{Cvoid},), cond) + end + return +end + +function flush_hostcall_output() + hostcall_output_pending[] == 0 && return + data, n = @lock hostcall_output_lock begin + n = hostcall_output_pending[] + hostcall_output_pending[] = 0 + take!(hostcall_output), n + end + Threads.atomic_sub!(hostcall_sync_pending, n) + isempty(data) || write(stdout, data) + return +end + +function hostcall_output_printer() + @lock hostcall_output_cond_lock begin + if hostcall_output_cond[] === nothing + hostcall_output_cond[] = Base.AsyncCondition() do _ + flush_hostcall_output() + end + end + end +end + + +## servicing a port + +# host-side view of a single in-progress call on a port: the server owns the buffer +# whenever the outbox differs from the inbox it last wrote +mutable struct HostPort + area::HostcallArea + index::Int + out::UInt32 # last observed outbox value + in::UInt32 # last written inbox value (bit 0 is the ownership bit) +end + +# give the buffer to the device (optionally with data written to the packets) and report a +# status; the next device flip makes it ours again +function hport_flip!(p::HostPort, status::UInt32=HOSTCALL_STATUS_OK) + p.in = p.out | (status << 1) + @inbounds p.area.shadow[p.index + 1] = p.in + mailbox_store!(inbox_ptr(p.area, p.index), p.in) + return +end + +# wait for the device to flip its outbox after we handed the buffer back. the device is +# actively waiting on us when this is called, so this should be quick; time out in case +# the kernel died. +function hport_wait!(p::HostPort; timeout=10.0) + t0 = time() + while true + out = mailbox_load(outbox_ptr(p.area, p.index)) + if out != (p.in & UInt32(1)) + p.out = out + return true + end + ccall(:jl_cpu_pause, Cvoid, ()) + ccall(:jl_gc_safepoint, Cvoid, ()) + time() - t0 > timeout && return false + end +end + +lane_packet(p::HostPort, lane) = packet_ptr(p.area, p.index, lane) + +# everything below is deliberately type-erased (`@nospecialize`, `jl_new_bits`, +# `jl_value_ptr`): the server thread must not have to compile code for every new target +# type, only the handlers themselves, which are precompiled on registration. + +# load a value of type `T` from memory into a new boxed object +@inline load_bits(@nospecialize(T::Type), ptr::Ptr{UInt8}) = + ccall(:jl_new_bits, Any, (Any, Ptr{Cvoid}), T, ptr) + +# store the bits of a boxed isbits value +@inline function store_bits!(ptr::Ptr{UInt8}, @nospecialize(x)) + n = Core.sizeof(typeof(x)) + n == 0 && return + src = ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), x) + ccall(:memcpy, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, Csize_t), ptr, src, n) + return +end + +# read every live lane's value of type `T`, receiving additional chunks if needed +function read_lanes(p::HostPort, @nospecialize(T::Type), mask::UInt32) + values = Vector{Any}(undef, 32) + nbytes = Core.sizeof(T) + if nbytes <= HOSTCALL_PACKET_SIZE + for lane in 0:31 + (mask >> lane) & 1 == 0 && continue + @inbounds values[lane + 1] = load_bits(T, lane_packet(p, lane)) + end + else + nchunks = cld(nbytes, HOSTCALL_PACKET_SIZE) + bufs = [Vector{UInt8}(undef, nchunks * HOSTCALL_PACKET_SIZE) for _ in 1:32] + for chunk in 0:nchunks-1 + if chunk > 0 + hport_flip!(p) + hport_wait!(p) || error("timed out receiving hostcall arguments") + end + for lane in 0:31 + (mask >> lane) & 1 == 0 && continue + unsafe_copyto!(pointer(bufs[lane + 1]) + chunk * HOSTCALL_PACKET_SIZE, + lane_packet(p, lane), HOSTCALL_PACKET_SIZE) + end + end + for lane in 0:31 + (mask >> lane) & 1 == 0 && continue + @inbounds values[lane + 1] = load_bits(T, pointer(bufs[lane + 1])) + end + end + return values +end + +# write each lane's value (of type `T`) to its packets and hand the buffer back; for large +# values this involves multiple flips, for which the device must be receiving. +function write_lanes!(p::HostPort, @nospecialize(T::Type), values::Vector{Any}, mask::UInt32, + status::UInt32) + nbytes = Core.sizeof(T) + if nbytes <= HOSTCALL_PACKET_SIZE + for lane in 0:31 + (mask >> lane) & 1 == 0 && continue + @inbounds store_bits!(lane_packet(p, lane), values[lane + 1]) + end + hport_flip!(p, status) + else + nchunks = cld(nbytes, HOSTCALL_PACKET_SIZE) + bufs = [Vector{UInt8}(undef, nchunks * HOSTCALL_PACKET_SIZE) for _ in 1:32] + for lane in 0:31 + (mask >> lane) & 1 == 0 && continue + @inbounds store_bits!(pointer(bufs[lane + 1]), values[lane + 1]) + end + for chunk in 0:nchunks-1 + if chunk > 0 + hport_wait!(p) || error("timed out sending hostcall results") + end + for lane in 0:31 + (mask >> lane) & 1 == 0 && continue + unsafe_copyto!(lane_packet(p, lane), + pointer(bufs[lane + 1]) + chunk * HOSTCALL_PACKET_SIZE, + HOSTCALL_PACKET_SIZE) + end + hport_flip!(p, status) + end + end + return +end + +# the arguments of a call: the payload tuple without the callable (if shipped), converted +function call_arguments(@nospecialize(payload), skip::Int) + n = nfields(payload) - skip + args = Vector{Any}(undef, n) + for i in 1:n + @inbounds args[i] = hostconvert_host(getfield(payload, i + skip)) + end + return args +end + +# invoke a registered target for every live lane of a port +function service_target!(p::HostPort, target::HostcallTarget, hdr::HostcallHeader) + mask = hdr.mask + RT = target.RT + reply = (hdr.flags & HOSTCALL_FLAG_ASYNC) == 0 && RT !== Nothing + payloads = read_lanes(p, target.AT, mask) + results = Vector{Any}(undef, 32) + status = HOSTCALL_STATUS_OK + for lane in 0:31 + (mask >> lane) & 1 == 0 && continue + payload = @inbounds payloads[lane + 1] + f = target.stored ? target.f : getfield(payload, 1) + args = call_arguments(payload, target.stored ? 0 : 1) + try + if is_print_target(f) + queue_hostcall_output(f, args...) + rv = nothing + else + rv = Base.invokelatest(f, args...) + end + if reply + rv isa RT || (rv = convert(RT, rv)) + @inbounds results[lane + 1] = rv + end + catch err + record_hostcall_exception!(p.area, f, err, catch_backtrace()) + status = HOSTCALL_STATUS_ERROR + end + end + if status != HOSTCALL_STATUS_OK || !reply + hport_flip!(p, status) + else + write_lanes!(p, RT, results, mask, status) + end + return +end + +# built-in targets, implemented by the runtime library and serviced by these handlers +const hostcall_builtins = Dict{UInt64,Function}() + +function service_port!(a::HostcallArea, i::Int, out::UInt32) + hdr = unsafe_load(header_ptr(a, i)) + p = HostPort(a, i, out, @inbounds a.shadow[i + 1]) + context!(a.ctx) do + stream!(hostcall_stream(a)) do + if hdr.target < HOSTCALL_BUILTIN_IDS + handler = get(hostcall_builtins, hdr.target, nothing) + if handler === nothing + record_hostcall_exception!(a, hdr.target, ErrorException("unknown built-in hostcall target")) + hport_flip!(p, HOSTCALL_STATUS_ERROR) + else + try + handler(p, hdr) + catch err + record_hostcall_exception!(a, hdr.target, err, catch_backtrace()) + hport_flip!(p, HOSTCALL_STATUS_ERROR) + end + end + else + target = hostcall_target(hdr.target) + if target === nothing + record_hostcall_exception!(a, hdr.target, ErrorException("unknown hostcall target; was the kernel compiled in another session without its targets being registered?")) + hport_flip!(p, HOSTCALL_STATUS_ERROR) + else + service_target!(p, target, hdr) + end + end + end + end + return +end + +# one sweep over the ports of an area; returns the number of ports serviced. the outbox +# words are compared against our shadow of the inbox in chunks, which vectorizes and keeps +# the cost of scanning idle ports negligible. +function sweep!(a::HostcallArea) + # fast-path the all-idle case with a single scan over the whole area: the chunked walk + # below has per-chunk overhead that adds up on slower CPUs, and with one server thread + # sweeping every context's area, idle areas are the common case + pending_ports(a) || return 0 + n = 0 + nports = a.nports + outbox = a.outbox + shadow = a.shadow + start = (a.cursor ÷ HOSTCALL_SWEEP_CHUNK) * HOSTCALL_SWEEP_CHUNK + @inbounds for c0 in 0:HOSTCALL_SWEEP_CHUNK:nports-1 + cbase = (start + c0) % nports + cend = cbase + HOSTCALL_SWEEP_CHUNK - 1 + d = UInt32(0) + @simd for j in cbase:cend + d |= (outbox[j+1] ⊻ shadow[j+1]) & UInt32(1) + end + d == 0 && continue + for j in cbase:cend + out = mailbox_load(outbox_ptr(a, j)) + out == (shadow[j+1] & UInt32(1)) && continue + service_port!(a, j, out) + n += 1 + a.cursor = j + 1 >= nports ? 0 : j + 1 + end + end + return n +end + + +## server thread + +mutable struct HostcallServer + const mutex::Ptr{Cvoid} # uv_mutex_t, guards `cond` + const cond::Ptr{Cvoid} # uv_cond_t, signalled when a kernel is armed + const sem::Ptr{Cvoid} # uv_sem_t, posted by `cuLaunchHostFunc` after every armed kernel + const sweep_lock::Ptr{Cvoid} # uv_mutex_t, serializes sweeps (server thread vs. draining tasks) + const launches::Threads.Atomic{Int} + const serviced::Threads.Atomic{Int} # completed launches covered by a full sweep + const graphs::Threads.Atomic{Bool} # graph replays cannot be armed + const sweep_owner::Threads.Atomic{UInt} # identity of the task holding `sweep_lock`, or 0 + finished::Int # launches that completed; only touched by the server thread +end + +const hostcall_server = Ref{Union{Nothing,HostcallServer}}(nothing) +const hostcall_server_lock = ReentrantLock() + +# acquire the sweep lock without blocking the thread in a GC-unsafe state +function lock_sweeps(srv::HostcallServer) + while @ccall(uv_mutex_trylock(srv.sweep_lock::Ptr{Cvoid})::Cint) != 0 + ccall(:jl_cpu_pause, Cvoid, ()) + ccall(:jl_gc_safepoint, Cvoid, ()) + end + srv.sweep_owner[] = task_identity() + return +end +function unlock_sweeps(srv::HostcallServer) + srv.sweep_owner[] = UInt(0) + @ccall uv_mutex_unlock(srv.sweep_lock::Ptr{Cvoid})::Cvoid + return +end +# sweeps are owned by a task (the server thread's root task, or a draining task), not a +# thread: a draining task that migrates between threads keeps its ownership, and other +# tasks running on its thread do not inherit it. +task_identity() = UInt(pointer_from_objref(current_task())) +# whether the current task is in the middle of a sweep, i.e. running a handler +sweeping(srv::HostcallServer) = srv.sweep_owner[] == task_identity() +function hostcall_in_handler() + srv = hostcall_server[] + srv === nothing && return false + return sweeping(srv) +end + +function sweep_all!() + n = 0 + for area in Base.@atomic hostcall_areas.list + n += sweep!(area) + end + return n +end + +# Unarmed kernels can only use hostcalls for exception reporting. Poll just the areas +# assigned to those kernels; full-size areas are swept while their kernels are armed, or +# on every heartbeat once a hostcall graph has been captured. +function sweep_heartbeat!() + n = 0 + for area in Base.@atomic hostcall_areas.list + area.heartbeat || continue + n += sweep!(area) + end + return n +end + +# wait for a kernel to be armed, or for the heartbeat interval to pass. a thread blocked in +# a plain ccall holds up garbage collection, so the wait is GC-safe (and short anyway). +const HOSTCALL_HEARTBEAT_NS = 1_000_000 +function hostcall_server_wait(srv::HostcallServer) + @ccall uv_mutex_lock(srv.mutex::Ptr{Cvoid})::Cvoid + if srv.launches[] - srv.finished == 0 + @gcsafe_ccall uv_cond_timedwait(srv.cond::Ptr{Cvoid}, srv.mutex::Ptr{Cvoid}, + HOSTCALL_HEARTBEAT_NS::UInt64)::Cint + end + @ccall uv_mutex_unlock(srv.mutex::Ptr{Cvoid})::Cvoid + return +end + +const HOSTCALL_SPIN_BEFORE_SLEEP = 1 << 16 # empty sweeps before backing off to sleeps + +# a short GC-safe sleep for the armed-but-idle path. Windows has no `usleep`, so use +# libuv's millisecond sleep there. +@static if Sys.iswindows() + hostcall_backoff() = @gcsafe_ccall uv_sleep(1::Cuint)::Cvoid +else + hostcall_backoff() = @gcsafe_ccall usleep(20::Cuint)::Cint +end + +function hostcall_server_main(srv::HostcallServer) + idle = 0 + while true + try + # account for completed launches + while @ccall(uv_sem_trywait(srv.sem::Ptr{Cvoid})::Cint) == 0 + srv.finished += 1 + end + armed = srv.launches[] - srv.finished + full_sweep = armed > 0 || srv.finished != srv.serviced[] || srv.graphs[] + + lock_sweeps(srv) + found = try + full_sweep ? sweep_all!() : sweep_heartbeat!() + finally + unlock_sweeps(srv) + end + if full_sweep + completed = srv.finished - srv.serviced[] + srv.serviced[] = srv.finished + completed > 0 && Threads.atomic_sub!(hostcall_sync_pending, completed) + end + + if armed > 0 + # kernels that may hostcall are running: poll, backing off to short sleeps + # when nothing has happened for a while + if found > 0 + idle = 0 + else + idle += 1 + if idle < HOSTCALL_SPIN_BEFORE_SLEEP + ccall(:jl_cpu_pause, Cvoid, ()) + else + hostcall_backoff() + end + end + else + # idle: heartbeat sweeps, woken early by the next launch + idle = 0 + hostcall_server_wait(srv) + end + catch err + # errors in the server itself (not in handlers, which are caught separately) + # are reported at the next synchronization; keep the thread alive + record_hostcall_exception!(nothing, :server, err, catch_backtrace()) + end + ccall(:jl_gc_safepoint, Cvoid, ()) + end +end + +function hostcall_server_entry(::Ptr{Cvoid}) + srv = hostcall_server[]::HostcallServer + hostcall_server_main(srv) + return nothing +end + + +function hostcall_server_start() + @lock hostcall_server_lock begin + hostcall_server[] === nothing || return + # libuv synchronization objects; never freed + mutex = Libc.malloc(64) + cond = Libc.malloc(64) + sem = Libc.malloc(64) + sweep_lock = Libc.malloc(64) + @ccall(uv_mutex_init(mutex::Ptr{Cvoid})::Cint) == 0 || error("uv_mutex_init failed") + @ccall(uv_mutex_init(sweep_lock::Ptr{Cvoid})::Cint) == 0 || error("uv_mutex_init failed") + @ccall(uv_cond_init(cond::Ptr{Cvoid})::Cint) == 0 || error("uv_cond_init failed") + @ccall(uv_sem_init(sem::Ptr{Cvoid}, 0::Cuint)::Cint) == 0 || error("uv_sem_init failed") + srv = HostcallServer(mutex, cond, sem, sweep_lock, Threads.Atomic{Int}(0), + Threads.Atomic{Int}(0), Threads.Atomic{Bool}(false), + Threads.Atomic{UInt}(0), 0) + + # Compile the server and its dynamically-dispatched handlers on this thread. + precompile(hostcall_server_main, (HostcallServer,)) + precompile(service_port!, (HostcallArea, Int, UInt32)) + for handler in values(hostcall_builtins) + precompile(handler, (HostPort, HostcallHeader)) + end + + # the thread is adopted by Julia when it first calls into the @cfunction, and is + # never torn down + hostcall_server[] = srv + tid = Ref{NTuple{32, UInt8}}(ntuple(i -> 0x0, 32)) + cb = @cfunction(hostcall_server_entry, Cvoid, (Ptr{Cvoid},)) + err = @ccall uv_thread_create(tid::Ptr{Cvoid}, cb::Ptr{Cvoid}, C_NULL::Ptr{Cvoid})::Cint + if err != 0 + hostcall_server[] = nothing + Base.uv_error("uv_thread_create", err) + end + err = @ccall uv_thread_detach(tid::Ptr{Cvoid})::Cint + err == 0 || Base.uv_error("uv_thread_detach", err) + + # the printer task is created on a Julia thread + hostcall_output_printer() + end + return +end + + +## arming and draining + +# Captured kernels are replayed without passing through `hostcall_launch`, so keep idle +# heartbeat sweeps enabled for every area after the first hostcall graph is captured. +function hostcall_mark_graph!() + srv = hostcall_server[]::HostcallServer + if !Threads.atomic_cas!(srv.graphs, false, true) + Threads.atomic_add!(hostcall_sync_pending, 1) + end + return +end + +""" + hostcall_arm!() + +Announce an upcoming launch of a kernel that may hostcall: the server starts polling. +Balanced by [`hostcall_disarm!`](@ref) after the kernel, enqueued on its stream. +""" +function hostcall_arm!() + srv = hostcall_server[]::HostcallServer + Threads.atomic_add!(hostcall_sync_pending, 1) + Threads.atomic_add!(srv.launches, 1) + @ccall uv_mutex_lock(srv.mutex::Ptr{Cvoid})::Cvoid + @ccall uv_cond_signal(srv.cond::Ptr{Cvoid})::Cvoid + @ccall uv_mutex_unlock(srv.mutex::Ptr{Cvoid})::Cvoid + return +end + +""" + hostcall_disarm!(stream::CuStream) + hostcall_disarm!() + +Mark a launch as completed, either immediately or when `stream` reaches this point. The +latter enqueues a plain C callback (`uv_sem_post`) that only touches a semaphore, so it +neither calls into Julia nor needs the libuv event loop. +""" +function hostcall_disarm!(stream::CuStream) + srv = hostcall_server[]::HostcallServer + cuLaunchHostFunc(stream, cglobal(:uv_sem_post), srv.sem) + return +end +function hostcall_disarm!() + srv = hostcall_server[]::HostcallServer + @ccall uv_sem_post(srv.sem::Ptr{Cvoid})::Cvoid + return +end + +""" + hostcall_drain() + +Service every pending hostcall and flush deferred output. Called by `synchronize()` and +`device_synchronize()`, so that asynchronous hostcalls made by a kernel have completed by +the time those return; handlers may run on the calling task. +""" +# whether any port of an area is owned by the host, i.e. has a pending call; a lock-free +# read that may race with a sweep, in which case the lock-protected sweep decides +function pending_ports(a::HostcallArea) + outbox = a.outbox + shadow = a.shadow + d = UInt32(0) + @inbounds @simd for j in 1:a.nports + d |= (outbox[j] ⊻ shadow[j]) & UInt32(1) + end + return d != 0 +end + +function hostcall_drain() + srv = hostcall_server[] + srv === nothing && return + areas = Base.@atomic hostcall_areas.list + isempty(areas) && return + # fast path: nothing to service or flush + if !any(pending_ports, areas) && hostcall_output_pending[] == 0 + return + end + if sweeping(srv) + # called from a handler (e.g. a handler synchronizing the hostcall stream): the + # port being serviced is still pending, so a nested sweep would re-enter the + # handler. handlers cannot wait for other hostcalls anyway. + return + end + lock_sweeps(srv) + try + sweep_all!() + finally + unlock_sweeps(srv) + end + flush_hostcall_output() + return +end + +# Complete pending asynchronous work and surface handler failures. The aggregate counter +# keeps ordinary synchronization to one atomic load when no hostcall work is outstanding. +@inline function hostcall_synchronize(ctx::CuContext) + hostcall_sync_pending[] == 0 && return + hostcall_drain() + check_hostcall_exceptions(ctx) + return +end + +# Forget the areas of a context that is about to be destroyed. Stop the server from +# sweeping them before releasing their streams and raw allocations. +function hostcall_forget!(pred) + srv = hostcall_server[] + srv === nothing && return + nested = sweeping(srv) + nested || lock_sweeps(srv) + try + @lock hostcall_areas_lock begin + areas = Base.@atomic hostcall_areas.list + forgotten = filter(pred, areas) + isempty(forgotten) && return + Base.@atomic hostcall_areas.list = filter(a -> !pred(a), areas) + # Release resources while the context is still valid. These low-level memory + # wrappers do not own finalizers. + for a in forgotten + context!(a.ctx) do + a.stream === nothing || unsafe_destroy!(a.stream) + a.stream = nothing + free(a.locks) + free(a.mem) + end + end + # exceptions recorded for these contexts can no longer be reported + ctxs = Set(a.ctx for a in forgotten) + @lock hostcall_exceptions_lock begin + n = length(hostcall_exceptions) + filter!(((c, _),) -> !(c in ctxs), hostcall_exceptions) + removed = n - length(hostcall_exceptions) + Threads.atomic_sub!(hostcall_exceptions_pending, removed) + Threads.atomic_sub!(hostcall_sync_pending, removed) + end + end + finally + nested || unlock_sweeps(srv) + end + return +end +hostcall_forget!(ctx::CuContext) = hostcall_forget!(a -> a.ctx == ctx) +hostcall_forget!(dev::CuDevice) = hostcall_forget!(a -> a.dev == dev) diff --git a/test/core/hostcall.jl b/test/core/hostcall.jl new file mode 100644 index 0000000000..0a15e43e08 --- /dev/null +++ b/test/core/hostcall.jl @@ -0,0 +1,8 @@ +# the kernel state is part of the launch ABI and gets forwarded to dynamic parallelism +# child launches, so it must stay compact +@test sizeof(CUDACore.KernelState) == 2sizeof(UInt) + +@testset "idle backoff" begin + # This used to call `usleep` unconditionally, which is unavailable on Windows. + @test_nowarn CUDACore.hostcall_backoff() +end From 418b12ae45bd85a4830462cc2e0cbdcf9a2aced2 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Mon, 24 Aug 2026 09:58:29 +0200 Subject: [PATCH 03/10] Integrate hostcalls with the compiler and launcher. Recover the statically-known hostcall targets of a kernel from its compiled method instances and store them in the compile results, so that they travel with cached kernel images and get registered at link time, including for kernels compiled during package precompilation. Kernels that may hostcall get a full-size area (one port per resident warp) and arm the server around every launch, with the disarm enqueued on the stream; kernels captured into a graph are replayed behind our back, so capture is detected and such kernels are serviced by the heartbeat instead. On Windows, the launch queue is flushed after arming because WDDM may batch command submission. --- CUDACore/src/compiler/compilation.jl | 32 ++- CUDACore/src/compiler/execution.jl | 58 ++++- perf/hostcall.jl | 35 +++ perf/runbenchmarks.jl | 1 + test/core/hostcall.jl | 365 +++++++++++++++++++++++++++ 5 files changed, 485 insertions(+), 6 deletions(-) create mode 100644 perf/hostcall.jl diff --git a/CUDACore/src/compiler/compilation.jl b/CUDACore/src/compiler/compilation.jl index 8dfeae0760..6467a0d8ce 100644 --- a/CUDACore/src/compiler/compilation.jl +++ b/CUDACore/src/compiler/compilation.jl @@ -201,10 +201,17 @@ mutable struct CUDACompilerResults entry::Union{Nothing,String} relocations::GPUCompiler.Relocations + # whether the kernel calls host functions, and the statically-known targets of those + # calls (identifier => key type), as recovered from the compiled method instances; the + # identifiers are baked into the image, so the table travels with it + hostcall::Bool + hostcall_targets::Vector{Pair{UInt64,Type}} + # session-local kernel handles, linear-scanned by context; usually holds a single entry kernels::Vector{Tuple{CuContext,CuFunction}} CUDACompilerResults() = new(nothing, nothing, GPUCompiler.Relocations(), + false, Pair{UInt64,Type}[], Tuple{CuContext,CuFunction}[]) end @@ -387,6 +394,21 @@ function compile(@nospecialize(job::CompilerJob)) invoke_frozen(GPUCompiler.compile, :asm, job) end + # recover the hostcall targets: the device-side `hostcall_impl` is specialized on the + # key type of every statically-known call, and `meta.compiled` lists everything codegen + # emitted (including deferred compilation jobs) + hostcall = false + hostcall_targets = Pair{UInt64,Type}[] + for mi in keys(meta.compiled) + mi.def isa Method || continue + mi.def.module === CUDACore && mi.def.name === :hostcall_impl || continue + hostcall = true + K = mi.specTypes.parameters[2] + K isa DataType && K <: Type && K !== Type || continue + K = K.parameters[1] + push!(hostcall_targets, hostcall_target_id_value(K) => K) + end + # check if we'll need the device runtime undefined_fs = filter(collect(functions(meta.ir))) do f isdeclaration(f) && !LLVM.isintrinsic(f) && @@ -544,7 +566,8 @@ function compile(@nospecialize(job::CompilerJob)) rm(ptxas_output) end - return (image, entry=LLVM.name(meta.entry), relocations=meta.relocations) + return (image, entry=LLVM.name(meta.entry), relocations=meta.relocations, + hostcall, hostcall_targets) end # link a compiled image into a session-local `CuFunction` on the active context @@ -553,7 +576,10 @@ function link_kernel(image::Vector{UInt8}, entry::String, # load as an executable kernel object on the current context mod = try CuModule(image) - catch + catch err + # loading synchronizes the device first, which may surface an unrelated exception + # from an earlier kernel; only driver errors are about our image + err isa CuError || rethrow() # the driver rejected our compiled image (e.g. ERROR_NOT_SUPPORTED). dump the cubin # so the failure can be reported with a reproducer, mirroring how we keep the PTX # around when `ptxas` fails above. @@ -597,6 +623,8 @@ function compile_or_lookup(@nospecialize(job::CompilerJob))::CUDACompilerResults res.image = compiled.image res.entry = compiled.entry res.relocations = compiled.relocations + res.hostcall = compiled.hostcall + res.hostcall_targets = compiled.hostcall_targets end return res end diff --git a/CUDACore/src/compiler/execution.jl b/CUDACore/src/compiler/execution.jl index aa43a913bb..15c9520e21 100644 --- a/CUDACore/src/compiler/execution.jl +++ b/CUDACore/src/compiler/execution.jl @@ -565,11 +565,53 @@ end # finalize types call_tt = Base.to_tuple_type(call_t) - quote - cudacall(kernel.fun, $call_tt, $(call_args...); call_kwargs...) + if kernel <: HostKernel + quote + if kernel.hostcall + hostcall_launch(kernel.fun, $call_tt, $(call_args...); call_kwargs...) + else + cudacall(kernel.fun, $call_tt, $(call_args...); call_kwargs...) + end + end + else + quote + cudacall(kernel.fun, $call_tt, $(call_args...); call_kwargs...) + end end end +# launch a kernel that may call host functions: the hostcall server is armed for the +# duration of the kernel, and disarmed by a host function enqueued after it. +@inline function hostcall_launch(fun, tt, args...; stream::CuStream=stream(), kwargs...) + if is_capturing(stream) + # graph replays are not visible to us; such kernels are serviced by the heartbeat + hostcall_mark_graph!() + return cudacall(fun, tt, args...; stream, kwargs...) + end + hostcall_arm!() + try + cudacall(fun, tt, args...; stream, kwargs...) + @static if Sys.iswindows() + # WDDM may batch command submission (observed to be eager with hardware GPU + # scheduling, but not guaranteed without it); a stream query is a cheap way to + # flush the queue so that the kernel is running while the host polls for it + unchecked_cuStreamQuery(stream) + end + catch + hostcall_disarm!() + rethrow() + end + try + hostcall_disarm!(stream) + catch + # The launch is already submitted, but failed callback submission must not leave + # the server permanently armed. Heartbeat polling still services this kernel. + hostcall_disarm!() + rethrow() + end + return +end + ## host-side kernels @@ -581,6 +623,7 @@ struct HostKernel{F,TT} <: AbstractKernel{F,TT} f::F fun::CuFunction state::KernelState + hostcall::Bool # whether launches need the hostcall server to be armed end @doc (@doc AbstractKernel) HostKernel @@ -694,8 +737,12 @@ function cufunction(f::F, tt::TT=Tuple{}; kwargs...) where {F,TT} end end if fun === nothing + if res.hostcall && !hostcall_available(cuda.device) + error("This kernel calls host functions, but hostcall is not available on this device (see the `hostcall` preference, and `CUDA.hostcall_available`).") + end fun = link_kernel(res.image::Vector{UInt8}, res.entry::String, res.relocations) + register_hostcall_targets!(res.hostcall_targets) # don't cache session-local handles while generating output: the results struct # is serialized into the package image along with its CodeInstance, and the # handles would come back dangling. @@ -710,11 +757,14 @@ function cufunction(f::F, tt::TT=Tuple{}; kwargs...) where {F,TT} kernel = get(_kernel_instances, key, nothing) if kernel === nothing # create the kernel state object + # kernels that call host functions get a full-size hostcall area + ports = res.hostcall ? hostcall_default_ports(cuda.device) : HOSTCALL_MIN_PORTS exception_info, fallback = create_exceptions!(fun.mod) - client = hostcall_client(ctx, cuda.device, exception_info, fallback) + client = hostcall_client(ctx, cuda.device, exception_info, fallback; + ports, heartbeat=!res.hostcall) state = KernelState(client, UInt32(0)) - kernel = HostKernel{F,tt}(f, fun, state) + kernel = HostKernel{F,tt}(f, fun, state, res.hostcall) _kernel_instances[key] = kernel end return kernel::HostKernel{F,tt} diff --git a/perf/hostcall.jl b/perf/hostcall.jl new file mode 100644 index 0000000000..e2ef217f92 --- /dev/null +++ b/perf/hostcall.jl @@ -0,0 +1,35 @@ +# hostcall: calling host functions from kernels + +group = addgroup!(SUITE, "hostcall") + +hostcall_identity(x::Int) = x + +# launch overhead of a kernel that is armed for hostcalls (but never calls) +function hostcall_armed_kernel(out, flag) + if flag + out[1] = @hostcall hostcall_identity(1)::Int + end + return +end +out = CUDA.zeros(Int, 1) +group["launch_armed"] = @benchmarkable @cuda hostcall_armed_kernel($out, false) + +# a single blocking call per warp; latency dominated +function hostcall_blocking_kernel(out) + i = (blockIdx().x - 1) * blockDim().x + threadIdx().x + out[i] = @hostcall hostcall_identity(Int(i))::Int + return +end +out1 = CUDA.zeros(Int, 32) +group["blocking_1warp"] = @async_benchmarkable @cuda threads=32 hostcall_blocking_kernel($out1) +out512 = CUDA.zeros(Int, 32 * 512) +group["blocking_512warps"] = @async_benchmarkable @cuda threads=256 blocks=64 hostcall_blocking_kernel($out512) + +# fire-and-forget calls, drained by synchronize() +function hostcall_async_kernel(out) + i = (blockIdx().x - 1) * blockDim().x + threadIdx().x + @hostcall async=true hostcall_identity(Int(i)) + return +end +group["async_1warp"] = @async_benchmarkable @cuda threads=32 hostcall_async_kernel($out1) +group["async_512warps"] = @async_benchmarkable @cuda threads=256 blocks=64 hostcall_async_kernel($out512) diff --git a/perf/runbenchmarks.jl b/perf/runbenchmarks.jl index 3b26b5d8aa..7d04d70dc3 100644 --- a/perf/runbenchmarks.jl +++ b/perf/runbenchmarks.jl @@ -26,6 +26,7 @@ SUITE = BenchmarkGroup() include("cuda.jl") include("kernel.jl") +include("hostcall.jl") include("array.jl") @info "Preparing main benchmarks" diff --git a/test/core/hostcall.jl b/test/core/hostcall.jl index 0a15e43e08..650078adcc 100644 --- a/test/core/hostcall.jl +++ b/test/core/hostcall.jl @@ -1,8 +1,373 @@ +using CUDA: HostcallException + # the kernel state is part of the launch ABI and gets forwarded to dynamic parallelism # child launches, so it must stay compact @test sizeof(CUDACore.KernelState) == 2sizeof(UInt) +hostcall_lookup(i::Int) = 10f0 * i +hostcall_increment(i::Int) = i + 1 +const hostcall_counter = Threads.Atomic{Int}(0) +hostcall_count(i::Int) = (Threads.atomic_add!(hostcall_counter, i); nothing) +const hostcall_async_big_calls = Threads.Atomic{Int}(0) +function hostcall_async_big(i::Int) + Threads.atomic_add!(hostcall_async_big_calls, 1) + return ntuple(j -> i + j, 20) +end +hostcall_big(t::NTuple{12,Float64}, i::Int) = + ntuple(j -> Int(t[mod1(j, 12)]) + i + j, 20) +hostcall_sum_ptr(ptr::CuPtr{Int}, n::Int) = + Int(sum(Array(unsafe_wrap(CuArray, ptr, n)))) +hostcall_boom(i::Int) = i == 3 ? error("boom $i") : 2i +hostcall_tagged(i::Int, tag::Int) = i + tag +hostcall_fail(::Int) = error("boom") +hostcall_double(x::Int) = 2.0 * x + +struct HostcallScale + a::Float32 +end +(s::HostcallScale)(x) = s.a * x + +# Calls from kernels into host functions, serviced by a foreign server thread. +@testset "service" begin + function lookup(out) + i = Int(threadIdx().x) + out[i] = @hostcall(hostcall_lookup(i)::Float32) + 1f0 + return + end + out = CUDA.zeros(Float32, 64) + @cuda threads=64 lookup(out) + synchronize() + @test Array(out) == 10f0 .* (1:64) .+ 1 + + # partial warps, several blocks + out = CUDA.zeros(Float32, 64) + @cuda threads=20 blocks=3 lookup(out) + synchronize() + @test Array(out)[1:20] == 10f0 .* (1:20) .+ 1 + + # the functional form and the macro, with an explicit return type + function lookup2(out) + i = Int(threadIdx().x) + out[i] = @hostcall(hostcall_lookup(i)::Float32) + + hostcall(hostcall_lookup, Float32, i) + return + end + out = CUDA.zeros(Float32, 64) + @cuda threads=64 lookup2(out) + synchronize() + @test Array(out) == 20f0 .* (1:64) + + # the kernel keeps making progress while the main thread is blocked in the driver + out = CUDA.zeros(Float32, 64) + @cuda threads=64 lookup(out) + synchronize(; blocking=true) + @test Array(out) == 10f0 .* (1:64) .+ 1 + + # many warps contending for ports + function increment(out) + i = (blockIdx().x - 1) * blockDim().x + threadIdx().x + out[i] = @hostcall hostcall_increment(Int(i))::Int + return + end + out = CUDA.zeros(Int, 256 * 64) + @cuda threads=256 blocks=64 increment(out) + synchronize() + @test Array(out) == (1:256*64) .+ 1 + + # asynchronous calls are complete when synchronize() returns + hostcall_counter[] = 0 + function count() + i = Int(threadIdx().x) + CUDA.hostcall_async(hostcall_count, i) + @hostcall async=true hostcall_count(i) + return + end + @cuda threads=32 blocks=4 count() + synchronize() + @test hostcall_counter[] == 2 * 4 * sum(1:32) + + hostcall_counter[] = 0 + @cuda threads=32 count() + event = CuEvent() + record(event) + synchronize(event) + @test hostcall_counter[] == 2 * sum(1:32) + + # Asynchronous calls ignore the return value and never wait for result packets. + hostcall_async_big_calls[] = 0 + function async_big() + CUDA.hostcall_async(hostcall_async_big, Int(threadIdx().x)) + return + end + @cuda threads=32 async_big() + synchronize() + @test hostcall_async_big_calls[] == 32 + + # arguments and results larger than a packet + function big(out) + i = Int(threadIdx().x) + t = ntuple(j -> Float64(i * j), Val(12)) + r = @hostcall hostcall_big(t, i)::NTuple{20,Int} + s = 0 + for j in 1:20 + s += r[j] + end + out[i] = s + return + end + out = CUDA.zeros(Int, 40) + @cuda threads=40 big(out) + synchronize() + @test Array(out) == [sum(ntuple(j -> Int(Float64(i * mod1(j, 12))) + i + j, 20)) for i in 1:40] + + # device pointers arrive as CuPtr + function pointer_kernel(out, arr) + out[1] = @hostcall hostcall_sum_ptr(pointer(arr), length(arr))::Int + return + end + arr = CuArray(1:10) + out = CUDA.zeros(Int, 1) + @cuda pointer_kernel(out, arr) + synchronize() + @test Array(out)[1] == 55 + + # A hash collision must fail at registration instead of silently changing dispatch. + K1 = Tuple{typeof(identity),Int,Tuple{Int}} + K2 = Tuple{typeof(abs),Int,Tuple{Int}} + id = CUDACore.hostcall_target_id_value(K1) + @test id & CUDACore.HOSTCALL_STATIC_ID_BIT != 0 + CUDACore.register_hostcall_targets!([id => K1]) + @test_throws ErrorException CUDACore.register_hostcall_targets!([id => K2]) +end + @testset "idle backoff" begin # This used to call `usleep` unconditionally, which is unavailable on Windows. @test_nowarn CUDACore.hostcall_backoff() end + +@testset "exceptions" begin + function double(out) + i = Int(threadIdx().x) + out[i] = @hostcall hostcall_boom(i)::Int + return + end + out = CUDA.zeros(Int, 8) + @cuda threads=8 double(out) + err = try + synchronize() + nothing + catch err + err + end + @test err isa HostcallException + @test err.error isa ErrorException && err.error.msg == "boom 3" + @test occursin("boom 3", sprint(showerror, err)) + @test err.device == device() + # the exception has been consumed + synchronize() +end + +@testset "process layouts" begin + # the server thread does not depend on Julia's thread pools: hostcalls keep being + # serviced while the main thread is blocked in the driver, even without any other + # Julia threads + script = """ + using CUDA, Test + lookup_value(i::Int) = Float32(i) + function lookup(out) + i = Int(threadIdx().x) + out[i] = @hostcall(lookup_value(i)::Float32) + 1f0 + return + end + out = CUDA.zeros(Float32, 64) + @cuda threads=64 lookup(out) + synchronize(; blocking=true) + @test Array(out) == Float32.(2:65) + println("OK") + """ + # Julia 1.10 and 1.11 reject an explicit zero for the interactive pool, + # although `-t 1` produces the same one-default, zero-interactive layout. + thread_layouts = VERSION >= v"1.12" ? ["1,0", "1,1", "2,1"] : ["1", "1,1", "2,1"] + for threads in thread_layouts + proc, out, err = julia_exec(`-t $threads -e $script`) + success(proc) || @error "hostcall subprocess failed" threads stdout=out stderr=err + @test success(proc) + @test occursin("OK", out) + end +end + +if length(devices()) > 1 +@testset "multiple devices" begin + function increment(out) + i = Int(threadIdx().x) + out[i] = @hostcall hostcall_increment(i)::Int + return + end + results = [] + for dev in devices() + device!(dev) do + out = CUDA.zeros(Int, 32) + @cuda threads=32 increment(out) + push!(results, out) + end + end + for (dev, out) in zip(devices(), results) + device!(dev) do + synchronize() + @test Array(out) == (1:32) .+ 1 + end + end + + # concurrent blocking calls from kernels on every device: the server interleaves + # contexts, and results must not cross over between areas + function tagged(out, tag) + i = (blockIdx().x - 1) * blockDim().x + threadIdx().x + out[i] = @hostcall hostcall_tagged(Int(i), tag)::Int + return + end + outs = Dict(dev => device!(dev) do + out = CUDA.zeros(Int, 256 * 16) + tag = 1_000_000 * (deviceid(dev) + 1) + @cuda threads=256 blocks=16 tagged(out, tag) + out + end for dev in devices()) + for dev in devices() + device!(dev) do + synchronize() + @test Array(outs[dev]) == (1:256*16) .+ 1_000_000 * (deviceid(dev) + 1) + end + end + + # exceptions are reported per context: a handler error on one device surfaces when + # synchronizing that device, not another one + devA, devB = collect(Iterators.take(devices(), 2)) + function fail(out) + out[1] = @hostcall hostcall_fail(1)::Int + return + end + device!(devB) do + # a single lane, so that exactly one exception is recorded + @cuda threads=1 fail(CUDA.zeros(Int, 1)) + CUDACore.cuStreamSynchronize(stream()) # wait for the kernel without checking + end + device!(devA) do + synchronize() + end + err = device!(devB) do + try + synchronize() + nothing + catch err + err + end + end + @test err isa HostcallException + @test err.device == devB +end +end + +@testset "static targets" begin + function kernel(out) + i = Int(threadIdx().x) + a = @hostcall hostcall_double(i)::Float64 + b = @hostcall HostcallScale(3f0)(i)::Float32 + offset = 10 + c = @hostcall (x -> x + offset)(i)::Int # closure capturing an isbits value + d = hostcall(hostcall_double, Float64, i) + out[i] = a + b + c + d + return + end + out = CUDA.zeros(Float64, 32) + @cuda threads=32 kernel(out) + synchronize() + @test Array(out) == [2i + 3i + i + 10 + 2i for i in 1:32] + + # the compiler records the targets and marks the kernel + k = @cuda launch=false kernel(out) + @test k.hostcall + + # asynchronous calls and the print family, whose output is emitted at synchronization + function printer() + @hostcall async=true println("thread ", threadIdx().x) + @hostcall print("!")::Nothing + return + end + _, output = @grab_output begin + @cuda threads=2 printer() + synchronize() + end + @test occursin("thread 1", output) + @test occursin("thread 2", output) + @test count("!", output) == 2 +end + +@testset "graph capture" begin + # kernels replayed from a graph are not armed; they are serviced by the heartbeat + function kernel(out) + i = Int(threadIdx().x) + out[i] = @hostcall hostcall_double(i)::Float64 + return + end + out = CUDA.zeros(Float64, 32) + for i in 1:3 + CUDA.@captured begin + @cuda threads=32 kernel(out) + end + synchronize() + @test Array(out) == 2.0 .* (1:32) + out .= 0 + end +end + +@testset "precompiled kernels" begin + # a kernel compiled during package precompilation carries its hostcall targets along + # with the cached image, so calling it in a fresh session works without recompilation + mktempdir() do dir + pkgdir = joinpath(dir, "HostcallPrecompTest") + mkpath(joinpath(pkgdir, "src")) + write(joinpath(pkgdir, "src", "HostcallPrecompTest.jl"), """ + module HostcallPrecompTest + using CUDA + triple(x::Int) = 3.0 * x + function kernel(out) + i = Int(threadIdx().x) + out[i] = @hostcall triple(i)::Float64 + @hostcall async=true println("precompiled hostcall ", i) + return + end + const TT = Tuple{CuDeviceVector{Float64,CUDA.AS.Global}} + function run() + out = CUDA.zeros(Float64, 4) + @cuda threads=4 kernel(out) + synchronize() + return Array(out) + end + if ccall(:jl_generating_output, Cint, ()) != 0 && CUDA.functional() + # compile (but do not launch) the kernel during precompilation + cufunction(kernel, TT) + end + end + """) + script = """ + pushfirst!(LOAD_PATH, $(repr(dir))) + using CUDA, HostcallPrecompTest + using CUDACore: GPUCompiler, methodinstance, CompilerJob, compiler_config + # the image, with its hostcall targets, must come from the package image + job = CompilerJob(methodinstance(typeof(HostcallPrecompTest.kernel), HostcallPrecompTest.TT), + compiler_config(device())) + res = GPUCompiler.cached_results(CUDACore.CUDACompilerResults, job) + println("cached: ", res !== nothing && res.image !== nothing && res.hostcall && + length(res.hostcall_targets) == 2) + println("result: ", HostcallPrecompTest.run()) + """ + # first run precompiles the package, the second one is a fresh session + for i in 1:2 + proc, out, err = julia_exec(`-e $script`) + @test success(proc) + # GPUCompiler's package-image cache is only available on Julia 1.11+. + VERSION >= v"1.11" && @test occursin("cached: true", out) + @test occursin("result: [3.0, 6.0, 9.0, 12.0]", out) + @test occursin("precompiled hostcall 1", out) + end + end +end From 60b9a15fa6d5b6a73e78766c1e9863d292be100c Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Mon, 24 Aug 2026 09:58:29 +0200 Subject: [PATCH 04/10] Report device exceptions through hostcall. Send the exception name, reason and (with -g2) stack frames through a built-in hostcall target instead of printing them from the device, and attach the decoded report to the KernelException thrown at synchronization; the strings are module constants, copied by the host on the hostcall stream. The device never waits for the host on this path, and the exception output lock admits only one lane, so the sender only needs a scalar subset of the warp-collective protocol (which also keeps compile time and PTX size down for every throwing kernel, and sidesteps a CUDA 12.9 ptxas crash on out-of-line aggregate debug info). Reports need no registration, so precompiled kernels report fine in a fresh session; printf-based reporting remains as the fallback when hostcall is unavailable. --- CUDACore/src/compiler/exceptions.jl | 127 +++++++++++++++++++++- CUDACore/src/device/runtime.jl | 118 ++++++++++++++++---- CUDACore/src/hostcall.jl | 4 +- test/core/exceptions.jl | 163 ++++++++++++++++++++++++++-- 4 files changed, 379 insertions(+), 33 deletions(-) diff --git a/CUDACore/src/compiler/exceptions.jl b/CUDACore/src/compiler/exceptions.jl index 276f6e2c3f..c06c566a7e 100644 --- a/CUDACore/src/compiler/exceptions.jl +++ b/CUDACore/src/compiler/exceptions.jl @@ -2,12 +2,123 @@ ## exception type +# what the device reported about an exception (see `ExceptionReport` in the runtime) +struct KernelExceptionInfo + name::String + subtype::String + reason::String + thread::NTuple{3,Int} + block::NTuple{3,Int} + stacktrace::Union{Nothing,Vector{@NamedTuple{idx::Int, func::String, file::String, line::Int}}} +end + struct KernelException <: Exception dev::CuDevice + info::Union{Nothing,KernelExceptionInfo} end +KernelException(dev::CuDevice) = KernelException(dev, nothing) function Base.showerror(io::IO, err::KernelException) - print(io, "KernelException: exception thrown during kernel execution on device $(name(err.dev))") + info = err.info + if info === nothing + print(io, "KernelException: exception thrown during kernel execution on device $(name(err.dev))") + return + end + kind = isempty(info.subtype) ? info.name : info.subtype + isempty(kind) && (kind = "exception") + article = lowercase(first(kind)) in "aeiou" ? "an" : "a" + print(io, "KernelException: ", article, " ", kind, + " was thrown during kernel execution on device ", name(err.dev), + ", thread ", info.thread, " in block ", info.block, ".") + isempty(info.reason) || print(io, "\n", info.reason) + if info.stacktrace === nothing + print(io, "\nStacktrace not available, run Julia on debug level 2 for more details (by passing -g2 to the executable).") + else + print(io, "\nStacktrace:") + for frame in sort(info.stacktrace; by=f->f.idx) + print(io, "\n [", frame.idx, "] ", frame.func, " at ", frame.file, ":", frame.line) + end + end +end + +# reports received through the hostcall area, keyed by context and reporting thread; +# frames may arrive in any order and are assembled on the host +const kernel_exception_reports = Dict{CuContext, Dict{NTuple{6,Int}, KernelExceptionInfo}}() +const kernel_exception_reports_lock = Threads.SpinLock() + +# copy a NUL-terminated string from device memory on the given stream. the string is a +# module global, so we cannot read past the allocation that contains it. +function device_string(ptr::Ptr{UInt8}, stream::CuStream) + ptr == C_NULL && return "" + dptr = reinterpret(CuPtr{UInt8}, ptr) + base = Ref{CuPtr{Cvoid}}() + size = Ref{Csize_t}() + res = unchecked_cuMemGetAddressRange_v2(base, size, dptr) + res == SUCCESS || return "" + avail = Int(size[]) - (Int(dptr) - Int(base[])) + nbytes = clamp(avail, 0, 4096) + nbytes == 0 && return "" + buf = Vector{UInt8}(undef, nbytes) + res = unchecked_cuMemcpyDtoHAsync_v2(buf, dptr, nbytes, stream) + res == SUCCESS || return "" + cuStreamSynchronize(stream) + n = findfirst(iszero, buf) + return String(n === nothing ? buf : buf[1:n-1]) +end + +# the built-in handler for exception reports (registered in hostcall.jl); runs on the +# server thread (or in a draining task) with the context active and the hostcall stream +function service_exception_report(p, hdr) + area = p.area + mask = hdr.mask + for lane in 0:31 + (mask >> lane) & 1 == 0 && continue + report = unsafe_load(convert(Ptr{ExceptionReport}, lane_packet(p, lane))) + key = (Int(report.block.x), Int(report.block.y), Int(report.block.z), + Int(report.thread.x), Int(report.thread.y), Int(report.thread.z)) + stream = hostcall_stream(area) + if report.kind == EXCEPTION_REPORT_FRAME + frame = (; idx=Int(report.idx), func=device_string(report.a, stream), + file=device_string(report.b, stream), line=Int(report.line)) + @lock kernel_exception_reports_lock begin + reports = get!(Dict{NTuple{6,Int}, KernelExceptionInfo}, kernel_exception_reports, area.ctx) + info = get(reports, key, nothing) + if info === nothing + # the frame arrived before the report; create a placeholder + info = KernelExceptionInfo("", "", "", key[4:6], key[1:3], [frame]) + reports[key] = info + elseif info.stacktrace !== nothing + push!(info.stacktrace, frame) + end + end + else + name = device_string(report.a, stream) + subtype = device_string(report.b, stream) + reason = device_string(report.c, stream) + @lock kernel_exception_reports_lock begin + reports = get!(Dict{NTuple{6,Int}, KernelExceptionInfo}, kernel_exception_reports, area.ctx) + frames = if report.kind == EXCEPTION_REPORT_NAME + prev = get(reports, key, nothing) + prev === nothing || prev.stacktrace === nothing ? @NamedTuple{idx::Int, func::String, file::String, line::Int}[] : prev.stacktrace + else + nothing + end + reports[key] = KernelExceptionInfo(name, subtype, reason, key[4:6], key[1:3], frames) + end + end + end + hport_flip!(p) + return +end + +# take the reports of a context, if any +function take_exception_reports!(ctx::CuContext) + @lock kernel_exception_reports_lock begin + reports = get(kernel_exception_reports, ctx, nothing) + reports === nothing && return nothing + delete!(kernel_exception_reports, ctx) + return reports + end end @@ -51,15 +162,20 @@ function check_exceptions(ctx::CuContext=context()) # restore the structure unsafe_store!(exception_info, ExceptionInfo_st()) + # pick up the report the device sent before setting the flag + hostcall_drain() + reports = take_exception_reports!(ctx) + # throw host-side dev = device(ctx) - throw(KernelException(dev)) + info = reports === nothing || isempty(reports) ? nothing : first(values(reports)) + throw(KernelException(dev, info)) end return end # Drop exception state before destroying a context. The flag is raw pinned memory and must -# be released explicitly. +# be released explicitly; reports for a destroyed context can no longer be observed. function forget_exceptions!(pred) forgotten = @lock exception_infos_lock begin entries = Pair{CuContext,HostMemory}[] @@ -73,6 +189,11 @@ function forget_exceptions!(pred) for (_, mem) in forgotten free(mem) end + @lock kernel_exception_reports_lock begin + for ctx in collect(keys(kernel_exception_reports)) + pred(ctx) && delete!(kernel_exception_reports, ctx) + end + end return end forget_exceptions!(ctx::CuContext) = forget_exceptions!(candidate -> candidate == ctx) diff --git a/CUDACore/src/device/runtime.jl b/CUDACore/src/device/runtime.jl index d9ba82229e..e9ba62c2b9 100644 --- a/CUDACore/src/device/runtime.jl +++ b/CUDACore/src/device/runtime.jl @@ -123,6 +123,79 @@ end end +# exception reports are sent to the host through the hostcall area (when available) as +# packets with this layout, one per report or stack frame. the strings are pointers to +# module-constant device strings, copied over by the host. +const EXCEPTION_REPORT_NAME = UInt32(1) # the exception; stack frames follow +const EXCEPTION_REPORT_FRAME = UInt32(2) # one stack frame +const EXCEPTION_REPORT_NOTRACE = UInt32(3) # the exception, without stack frames +struct ExceptionReport + kind::UInt32 + idx::Int32 # frame index + a::Ptr{UInt8} # exception name / frame function + b::Ptr{UInt8} # subtype / frame file + c::Ptr{UInt8} # reason / unused + line::Int32 + thread::@NamedTuple{x::Int32,y::Int32,z::Int32} + block::@NamedTuple{x::Int32,y::Int32,z::Int32} +end + +# Send a report without waiting for the host. The exception output lock admits exactly one +# lane, so this only needs the scalar subset of the warp-collective hostcall protocol. +# Keep this inline: CUDA 12.9 ptxas crashes on the debug information for an out-of-line +# function taking ExceptionReport by value. Inlining also produces less PTX for this path. +@inline function send_exception_report(client::HostcallClient, report::ExceptionReport) + index = hostcall_start_index(client.nports) + lane = laneid() - Int32(1) + mask = UInt32(1) << lane + while true + word = client.lock + 4 * (index >> 5) + bit = UInt32(1) << (index & UInt32(31)) + if (atomic_or!(word, bit) & bit) == 0 + fence_gpu() + in = mailbox_load(client.inbox + 4index) + out = mailbox_load(client.outbox + 4index) + if hostcall_owned(in, out) + unsafe_store!(client.header + sizeof(HostcallHeader) * index, + HostcallHeader(mask, UInt32(0), HC_EXCEPTION)) + packet = client.packet + (index * HOSTCALL_LANES + lane) * HOSTCALL_PACKET_SIZE + unsafe_store!(reinterpret(LLVMPtr{ExceptionReport,AS.Global}, packet), report) + fence_sys() + mailbox_store!(client.outbox + 4index, out ⊻ UInt32(1)) + fence_gpu() + atomic_and!(word, ~bit) + return + end + fence_gpu() + atomic_and!(word, ~bit) + end + index += UInt32(1) + index >= client.nports && (index = UInt32(0)) + end +end + +@noinline function print_exception_report(ex, info::ExceptionInfo, trace::Bool) + # override the type GPUCompiler deduced if a quirk supplied a subtype + info.subtype != C_NULL && (ex = info.subtype) + @cuprintf("ERROR: a %s was thrown during kernel execution on thread (%d, %d, %d) in block (%d, %d, %d).\n", + ex, threadIdx().x, threadIdx().y, threadIdx().z, + blockIdx().x, blockIdx().y, blockIdx().z) + if info.reason != C_NULL + @cuprintf("%s\n", info.reason) + end + if trace + @cuprintf("Stacktrace:\n") + else + @cuprintf("Stacktrace not available, run Julia on debug level 2 for more details (by passing -g2 to the executable).\n") + end + return +end + +@noinline function print_exception_frame(idx, func, file, line) + @cuprintf(" [%d] %s at %s:%d\n", idx, func, file, line) + return +end + # it's not useful to have several threads report exceptions (interleaved output, can crash # CUDA), so use an output lock to only have a single thread write an exception message @inline function lock_output!(info::ExceptionInfo) @@ -145,16 +218,14 @@ function report_exception(ex) # this is the first reporting function being called, so claim the exception info = exception_info() if lock_output!(info) - # override the exception type GPUCompiler deduced if the user provided a subtype - if info.subtype != C_NULL - ex = info.subtype + client = hostcall_client() + if client.nports != 0 + send_exception_report(client, + ExceptionReport(EXCEPTION_REPORT_NOTRACE, 0, ex, info.subtype, info.reason, 0, + threadIdx(), blockIdx())) + else + print_exception_report(ex, info, false) end - @cuprintf("ERROR: a %s was thrown during kernel execution on thread (%d, %d, %d) in block (%d, %d, %d).\n", - ex, threadIdx().x, threadIdx().y, threadIdx().z, blockIdx().x, blockIdx().y, blockIdx().z) - if info.reason != C_NULL - @cuprintf("%s\n", info.reason) - end - @cuprintf("Stacktrace not available, run Julia on debug level 2 for more details (by passing -g2 to the executable).\n") end return end @@ -164,16 +235,14 @@ function report_exception_name(ex) # this is the first reporting function being called, so claim the exception if lock_output!(info) - # override the exception type GPUCompiler deduced if the user provided a subtype - if info.subtype != C_NULL - ex = info.subtype - end - @cuprintf("ERROR: a %s was thrown during kernel execution on thread (%d, %d, %d) in block (%d, %d, %d).\n", - ex, threadIdx().x, threadIdx().y, threadIdx().z, blockIdx().x, blockIdx().y, blockIdx().z) - if info.reason != C_NULL - @cuprintf("%s\n", info.reason) + client = hostcall_client() + if client.nports != 0 + send_exception_report(client, + ExceptionReport(EXCEPTION_REPORT_NAME, 0, ex, info.subtype, info.reason, 0, + threadIdx(), blockIdx())) + else + print_exception_report(ex, info, true) end - @cuprintf("Stacktrace:\n") end return end @@ -182,7 +251,14 @@ function report_exception_frame(idx, func, file, line) info = exception_info() if lock_output!(info) - @cuprintf(" [%d] %s at %s:%d\n", idx, func, file, line) + client = hostcall_client() + if client.nports != 0 + send_exception_report(client, + ExceptionReport(EXCEPTION_REPORT_FRAME, idx, func, file, C_NULL, line, + threadIdx(), blockIdx())) + else + print_exception_frame(idx, func, file, line) + end end return end @@ -192,7 +268,9 @@ function signal_exception() # finalize output if lock_output!(info) - @cuprintf("\n") + if hostcall_client().nports == 0 + @cuprintf("\n") + end info.output_lock = 2 end diff --git a/CUDACore/src/hostcall.jl b/CUDACore/src/hostcall.jl index f3c69b6547..773db96b5c 100644 --- a/CUDACore/src/hostcall.jl +++ b/CUDACore/src/hostcall.jl @@ -536,7 +536,9 @@ function service_target!(p::HostPort, target::HostcallTarget, hdr::HostcallHeade end # built-in targets, implemented by the runtime library and serviced by these handlers -const hostcall_builtins = Dict{UInt64,Function}() +const hostcall_builtins = Dict{UInt64,Function}( + HC_EXCEPTION => service_exception_report, +) function service_port!(a::HostcallArea, i::Int, out::UInt32) hdr = unsafe_load(header_ptr(a, i)) diff --git a/test/core/exceptions.jl b/test/core/exceptions.jl index ea0e567b79..ba7b0f7af6 100644 --- a/test/core/exceptions.jl +++ b/test/core/exceptions.jl @@ -1,8 +1,10 @@ # XXX: these tests occasionally hang under compute-sanitizer if !sanitize -host_error_re = r"ERROR: (KernelException: exception thrown during kernel execution on device|CUDA error: an illegal instruction was encountered|CUDA error: unspecified launch failure)" -device_error_re = r"ERROR: a \w+ was thrown during kernel execution" +# with hostcall available (the default), the device's report travels with the exception; +# without it, the device prints it with printf and the exception is message-less +host_error_re = r"ERROR: (KernelException: .*during kernel execution on device|CUDA error: an illegal instruction was encountered|CUDA error: unspecified launch failure)" +device_error_re = r"a \w+ was thrown during kernel execution" @testset "stack traces at different debug levels" begin @@ -34,19 +36,32 @@ let (proc, out, err) = julia_exec(`-g0 -e $script`) @test !success(proc) @test occursin(host_error_re, err) @test !occursin(device_error_re, out) + @test !occursin(device_error_re, err) # NOTE: stdout sometimes contain a failure to free the CuArray with ILLEGAL_ACCESS end let (proc, out, err) = julia_exec(`-g1 -e $script`) @test !success(proc) @test occursin(host_error_re, err) - @test count(device_error_re, out) == 1 - @test count("BoundsError", out) == 1 - @test count("Out-of-bounds array access", out) == 1 - @test occursin("Stacktrace not available", out) + @test count(device_error_re, err) == 1 + @test count("BoundsError", err) == 1 + @test count("Out-of-bounds array access", err) == 1 + @test occursin("Stacktrace not available", err) + @test !occursin(device_error_re, out) end let (proc, out, err) = julia_exec(`-g2 -e $script`) + @test !success(proc) + @test occursin(host_error_re, err) + @test count(device_error_re, err) == 1 + @test count("BoundsError", err) == 1 + @test count("Out-of-bounds array access", err) == 1 + @test occursin("] kernel at $(joinpath(".", "none"))", err) + @test !occursin(device_error_re, out) +end + +# without hostcall, the device prints the report itself +let (proc, out, err) = julia_exec(`-g2 -e $script`, "JULIA_CUDA_HOSTCALL" => "false") @test !success(proc) @test occursin(host_error_re, err) @test count(device_error_re, out) == 1 @@ -74,11 +89,141 @@ script = """ let (proc, out, err) = julia_exec(`-g2 -e $script`) @test !success(proc) @test occursin(host_error_re, err) - @test occursin(device_error_re, out) - @test occursin("foo at $(joinpath(".", "none"))", out) - @test occursin("bar at $(joinpath(".", "none"))", out) + @test occursin(device_error_re, err) + @test occursin("foo at $(joinpath(".", "none"))", err) + @test occursin("bar at $(joinpath(".", "none"))", err) +end + +end + +@testset "in-process" begin + # the exception carries the device's report + function kernel(arr, val) + arr[threadIdx().x] = val + return + end + gpu = CuArray(zeros(Int)) + @cuda threads=3 kernel(gpu, 1) + err = try + synchronize() + nothing + catch err + err + end + @test err isa CUDACore.KernelException + msg = sprint(showerror, err) + @test occursin(device_error_re, msg) + @test occursin("BoundsError", msg) + @test occursin("Out-of-bounds array access", msg) + # the exception has been consumed, and the device is usable again + synchronize() + gpu = CuArray(zeros(Int, 3)) + @cuda threads=3 kernel(gpu, 1) + synchronize() + @test Array(gpu) == [1, 1, 1] + + # a report from a kernel while the main thread is blocked in the driver + gpu = CuArray(zeros(Int)) + @cuda threads=3 kernel(gpu, 1) + err = try + synchronize(; blocking=true) + nothing + catch err + err + end + @test err isa CUDACore.KernelException + @test occursin("BoundsError", sprint(showerror, err)) + synchronize() end +if length(devices()) > 1 +@testset "multiple devices" begin + # device exceptions are reported when synchronizing the device they happened on + function kernel(arr, val) + arr[threadIdx().x] = val + return + end + devA, devB = collect(Iterators.take(devices(), 2)) + device!(devB) do + @cuda threads=3 kernel(CuArray(zeros(Int)), 1) + CUDACore.cuStreamSynchronize(stream()) # wait for the kernel without checking + end + device!(devA) do + synchronize() # not this device's exception + end + err = device!(devB) do + try + synchronize() + nothing + catch err + err + end + end + @test err isa CUDACore.KernelException + @test err.dev == devB + @test occursin("BoundsError", sprint(showerror, err)) + device!(devB) do + synchronize() + end + + # two devices failing concurrently + device!(devA) do + @cuda threads=3 kernel(CuArray(zeros(Int)), 1) + end + device!(devB) do + @cuda threads=3 kernel(CuArray(zeros(Int)), 2) + end + for dev in (devA, devB) + err = device!(dev) do + try + synchronize() + nothing + catch err + err + end + end + @test err isa CUDACore.KernelException + @test err.dev == dev + end +end +end + +@testset "precompiled kernels" begin + # exception reports do not depend on any per-kernel registration, so a kernel + # compiled during precompilation reports just as well in a fresh session + mktempdir() do dir + pkgdir = joinpath(dir, "ExceptionPrecompTest") + mkpath(joinpath(pkgdir, "src")) + write(joinpath(pkgdir, "src", "ExceptionPrecompTest.jl"), """ + module ExceptionPrecompTest + using CUDA + function kernel(arr, val) + arr[threadIdx().x] = val + return + end + const TT = Tuple{CuDeviceVector{Int,CUDA.AS.Global}, Int} + function run() + gpu = CuArray(zeros(Int)) + @cuda threads=3 kernel(gpu, 1) + synchronize() + end + if ccall(:jl_generating_output, Cint, ()) != 0 && CUDA.functional() + cufunction(kernel, TT) + end + end + """) + script = """ + pushfirst!(LOAD_PATH, $(repr(dir))) + using CUDA, ExceptionPrecompTest + ExceptionPrecompTest.run() + """ + for i in 1:2 + proc, out, err = julia_exec(`-g1 -e $script`) + @test !success(proc) + @test occursin(host_error_re, err) + @test occursin("BoundsError", err) + end + end end end From 7942a6400713eb0d5b67ab025ddc5dc9ff820852 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Mon, 24 Aug 2026 09:58:29 +0200 Subject: [PATCH 05/10] Document hostcalls. Add a manual page describing the API layers, handler rules, synchronization semantics, multi-device behavior, performance characteristics and preferences; reference the device API from the kernel programming docs, update the debugging page for the new exception output, and add a NEWS entry. --- NEWS.md | 12 +++ docs/make.jl | 1 + docs/src/api/kernel.md | 22 +++++ docs/src/development/debugging.md | 22 +++-- docs/src/development/hostcall.md | 129 ++++++++++++++++++++++++++++++ 5 files changed, 179 insertions(+), 7 deletions(-) create mode 100644 docs/src/development/hostcall.md diff --git a/NEWS.md b/NEWS.md index 52de53e09c..93db5035e7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -27,6 +27,18 @@ that package's image instead of being recompiled in every fresh session; on Julia 1.10 the cache stays session-local ([#3185](https://github.com/JuliaGPU/CUDA.jl/pull/3185)). +Kernels can now call host functions. `@hostcall f(args...)::R` invokes any +suitable Julia function on the host and returns its result to the calling +thread; `@hostcall async=true f(args...)` is a fire-and-forget variant whose +call has completed by the next synchronization. A raw port API is available +for library code. The calls are serviced by a dedicated host thread, so they +keep making progress even when Julia's own threads are blocked in CUDA API +calls. Device-side exceptions now use the same mechanism: the +`KernelException` thrown when synchronizing carries the exception type, +message and (with `-g2`) device stack trace, instead of the device printing +those details to standard output +([#3243](https://github.com/JuliaGPU/CUDA.jl/pull/3243)). + *Technically breaking changes*: - The CUDA compiler artifacts (`ptxas` and friends) are now selected diff --git a/docs/make.jl b/docs/make.jl index 11fd90488e..8eb1a15655 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -87,6 +87,7 @@ function main() "development/kernel.md", "development/troubleshooting.md", "development/debugging.md", + "development/hostcall.md", ], "Hacking" => Any[ "hacking/exposing_new_intrinsics.md", diff --git a/docs/src/api/kernel.md b/docs/src/api/kernel.md index cc911baf32..f937d9a63d 100644 --- a/docs/src/api/kernel.md +++ b/docs/src/api/kernel.md @@ -124,6 +124,28 @@ shfl_xor_sync ``` +## Host calls + +```@docs +@hostcall +hostcall +hostcall_async +HostcallException +hostcall_available +``` + +Raw ports, for library code: + +```@docs +HostcallClient +HostcallPort +hostcall_open +hostcall_send! +hostcall_recv! +hostcall_close! +hostcall_lane_packet +``` + ## Assertions ```@docs diff --git a/docs/src/development/debugging.md b/docs/src/development/debugging.md index d48d7d7ed3..bad088aaa3 100644 --- a/docs/src/development/debugging.md +++ b/docs/src/development/debugging.md @@ -1,4 +1,4 @@ -# Debugging +# [Debugging](@id DebuggingKernels) Even if your kernel executes, it may be computing the wrong values, or even error at run time. To debug these issues, both CUDA.jl and the CUDA toolkit provide several utilities. @@ -35,18 +35,21 @@ kernel (generic function with 1 method) julia> @cuda threads=2 kernel(CuArray([1])) ``` -If we execute this code, we'll get a very short error message: +If we execute this code and synchronize, we'll get a short error message: ``` -ERROR: a exception was thrown during kernel execution. -Run Julia on debug level 2 for device stack traces. +julia> synchronize() +ERROR: KernelException: a BoundsError was thrown during kernel execution on device NVIDIA GeForce RTX 5080, thread (2, 1, 1) in block (1, 1, 1). +Out-of-bounds array access +Stacktrace not available, run Julia on debug level 2 for more details (by passing -g2 to the executable). ``` As the message suggests, we can have CUDA.jl emit more rich stack trace information by setting Julia's debug level to 2 or higher by passing `-g2` to the `julia` invocation: ``` -ERROR: a exception was thrown during kernel execution. +ERROR: KernelException: a BoundsError was thrown during kernel execution on device NVIDIA GeForce RTX 5080, thread (2, 1, 1) in block (1, 1, 1). +Out-of-bounds array access Stacktrace: [1] throw_boundserror at abstractarray.jl:541 [2] checkbounds at abstractarray.jl:506 @@ -55,9 +58,14 @@ Stacktrace: [5] kernel at REPL[4]:2 ``` +The device reports these details to the host through the hostcall mechanism (see +[Calling host functions from kernels](@ref)), which is why they are part of the exception +rather than printed by the device. When hostcall is unavailable (e.g. disabled by the +`hostcall` preference), the device prints the report to standard output instead. + Note that these messages are embedded in the module (CUDA does not support stack unwinding), and thus bloat its size. To avoid any overhead, you can disable these messages by setting -the debug level to 0 (passing `-g0` to `julia`). This disabled any device-side message, but +the debug level to 0 (passing `-g0` to `julia`). This disables any device-side message, but retains the host-side detection: ``` @@ -65,7 +73,7 @@ julia> @cuda threads=2 kernel(CuArray([1])) # no device-side error message! julia> synchronize() -ERROR: KernelException: exception thrown during kernel execution +ERROR: KernelException: exception thrown during kernel execution on device NVIDIA GeForce RTX 5080 ``` diff --git a/docs/src/development/hostcall.md b/docs/src/development/hostcall.md new file mode 100644 index 0000000000..230be2732e --- /dev/null +++ b/docs/src/development/hostcall.md @@ -0,0 +1,129 @@ +# Calling host functions from kernels + +CUDA.jl kernels can call statically identifiable Julia functions on the host through the +*hostcall* mechanism. This is meant for "unlikely" paths in device code (reporting, +logging, loading data on demand, error handling), not as a bulk data path: a call costs a +few microseconds and goes through a small packet per thread. + +```julia +load(i) = DATA[i] # named host function + +function kernel(out) + i = threadIdx().x + out[i] = @hostcall load(i)::Float32 # blocking call, returns a value + @hostcall async=true println("thread ", i) # fire-and-forget + return +end + +@cuda threads=64 kernel(out) +synchronize() # asynchronous calls have completed, output has been printed +``` + +## API + +Two layers are available, both built on the same protocol. + +- [`@hostcall f(args...)::R`](@ref @hostcall) calls any host function whose value can be + recovered from its type: named functions, and isbits functors or closures (captured values + are shipped along with the arguments). The return type annotation is required, `@ccall`-style; + `@hostcall async=true f(args...)` returns immediately and implies `R === Nothing`. Arguments + may be annotated (`a::T`) to convert them before shipping. Functional forms: + `hostcall(f, R, args...)` and `hostcall_async(f, args...)`. +- Raw ports (`hostcall_open`, `hostcall_send!`, `hostcall_recv!`, `hostcall_close!`) for + library code that wants to stream data through the 64-byte per-lane packets itself. + +## Semantics and rules + +- **Warp-collective**: all lanes that reach a call site together share one port, and every + lane submits its own arguments and receives its own result. Divergent lanes simply make + separate calls. Calls are serviced in no particular order. +- **Values are shipped in their Julia layout** (values larger than a packet are split over + several packets). Arguments may contain compiler-relocated host constants such as string + literals, but arbitrary Julia references are unsupported. Pointers to device memory + (`pointer(arr)`) arrive on the host as `CuPtr`; device arrays themselves are not converted, + so pass their pointer and size explicitly. Results must be isbits (or `Nothing`); the + handler's return value is converted to `R`. Non-isbits return types are rejected when the + kernel is compiled. +- **Handlers run on a dedicated host thread** — a foreign thread that does not depend on + Julia's thread pools, so hostcalls make progress even with `-t1` and while the launching + thread is blocked in the driver. The handler runs with the kernel's context active and a + dedicated non-blocking stream as its task-local stream. Handlers may use the CUDA API on that + stream (e.g. copy device memory to the host), but: + - they must not synchronize the device, or wait for work on the stream the calling kernel + runs on: the kernel is waiting for the handler; + - they must not compile or load kernels (loading a module synchronizes the device), so + only call kernels that have been compiled before, and prefer `cuMemAllocAsync`-style + stream-ordered allocations (CUDA.jl's array allocations are); + - they must not wait on Julia tasks or conditions, or perform libuv-backed I/O + (`println`, files, `run`) while the launching thread may be blocked in a CUDA call + (julia#55525). The `print` family (`print`, `println`, `printstyled`, `show`, `display`) + called directly as a hostcall target is special-cased: its output is queued and written + at the next `synchronize()`, or earlier by a printer task when thread 1 is free. +- **Errors**: an exception thrown by a handler, an unknown target, or a result that cannot be + converted stops all lanes of that call on the device (like a device-side exception) and is + rethrown as a [`HostcallException`](@ref CUDACore.HostcallException) at the next stream, event, + or device synchronization, which also completes pending asynchronous calls and flushes queued + output. A handler exception therefore poisons every lane of the warp-level call. +- **Asynchronous calls** never send a result back to the device. +- **Nesting**: a handler cannot itself wait for another hostcall (e.g. by launching a kernel + that hostcalls and synchronizing it), since the server is busy running the handler. +- **Graphs**: kernels replayed from a captured graph are serviced by a 1 ms heartbeat + instead of the armed polling loop, so their calls see millisecond latency. +- **Launching**: launch kernels that use the high-level API through `@cuda` or a callable + `HostKernel`, which arms the server while the kernel runs. Direct driver-level launches + bypass that integration and may deadlock on a blocking call. + +## Exception reporting + +CUDA.jl itself uses hostcall to report device-side exceptions: the runtime library sends the +exception name, reason and (with `-g2`) stack frames through the hostcall area, without +waiting for the host, and `synchronize()` attaches the decoded report to the `KernelException` +it throws (see [Debugging](@ref DebuggingKernels)). This needs no registration, so it also works for kernels +compiled during package precompilation. Every kernel that can throw therefore refers to a small +hostcall area (64 ports), which is created on first use in each context; when hostcall is +unavailable, the device falls back to printing the report with `printf`. + +## Performance + +A blocking call costs roughly 4–5 µs on a PCIe system (most of it PCIe latency: each mailbox +probe is ~0.5 µs), an asynchronous one ~3 µs; one host thread services on the order of a +million warp-level calls per second. A kernel that uses hostcalls pays a few microseconds of +extra launch overhead (the server is *armed* for its duration and disarmed by a host function +enqueued after it); kernels that do not use hostcalls are unaffected. While armed, the server +thread polls (one CPU core), backing off to short sleeps when nothing happens; when idle it +sleeps, waking up every millisecond to service stragglers. + +## Multiple devices + +Hostcall areas are per context, created lazily and sized for their device (the default +number of ports is the number of resident warps of that device, so heterogeneous GPUs get +differently sized areas), and a single server thread services the areas of all devices, +switching to the calling kernel's context for every call. Consequences: + +- The server is a resource shared by all devices: the latency of a call grows with the + total number of warps waiting for service across all devices. +- Exceptions are reported per context. `synchronize()`, `synchronize(stream)` and + `device_synchronize()` throw the [`HostcallException`](@ref CUDACore.HostcallException)s + and `KernelException`s of the context they synchronize, and the exception names the + device; with the usual pattern of one task per device, each task sees the errors of its + own kernels. Errors in the server thread itself are reported by whichever synchronization + comes first. +- Synchronizing any device does complete pending asynchronous calls of all devices, + running their handlers with their own context active. + +## Configuration + +Preferences (set with `Preferences.set_preferences!(CUDACore, ...)` and restart; the +preferences belong to the `CUDACore` package, not `CUDA`): +- `hostcall` (default `true`): disable the mechanism entirely; kernels using it fail to + link, and device exceptions fall back to `printf`-based reporting. +- `hostcall_ports`: the number of ports (warp-level call slots) per context; the default is the + number of resident warps of the device (~8 MiB of pinned memory on a large GPU). Contexts + start with a small area until a kernel that calls host functions is linked. + +## Display watchdogs + +On devices with a display watchdog, a kernel blocked in a hostcall counts as running. A +slow handler can therefore push the kernel over the watchdog limit, like any other +long-running kernel. Hostcalls remain enabled by default on these devices, but handlers +should avoid long or unbounded waits. From 5c40f4240d47b4af418c98936de0f704c8975bce Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Mon, 24 Aug 2026 13:26:10 +0200 Subject: [PATCH 06/10] Make hostcall mandatory. Hostcall is core infrastructure for exception reporting and future runtime services, so remove the preference, environment variable, and printf fallback that allowed it to be disabled. Keep the no-port client only as a precompilation placeholder; its reporting guard produces a report-less KernelException if it is ever used. --- CUDACore/src/compiler/exceptions.jl | 5 +-- CUDACore/src/compiler/execution.jl | 5 +-- CUDACore/src/device/intrinsics/hostcall.jl | 3 +- CUDACore/src/device/runtime.jl | 42 +++++----------------- CUDACore/src/hostcall.jl | 30 ++++------------ docs/src/api/kernel.md | 1 - docs/src/development/debugging.md | 3 +- docs/src/development/hostcall.md | 20 +++++------ test/core/exceptions.jl | 13 +------ 9 files changed, 31 insertions(+), 91 deletions(-) diff --git a/CUDACore/src/compiler/exceptions.jl b/CUDACore/src/compiler/exceptions.jl index c06c566a7e..5818a9dd54 100644 --- a/CUDACore/src/compiler/exceptions.jl +++ b/CUDACore/src/compiler/exceptions.jl @@ -127,8 +127,9 @@ end const exception_infos = Dict{CuContext, HostMemory}() const exception_infos_lock = ReentrantLock() -# A no-hostcall descriptor follows the exception flag in the same mapped allocation. This -# gives every kernel one runtime-state pointer, even when hostcalls are unavailable. +# A no-port descriptor follows the exception flag in the same mapped allocation. It is +# used for kernels compiled during precompilation (which are never launched), and gives +# every kernel one runtime-state pointer. const exception_client_offset = cld(sizeof(ExceptionInfo_st), sizeof(UInt)) * sizeof(UInt) const exception_state_size = exception_client_offset + sizeof(HostcallClient) diff --git a/CUDACore/src/compiler/execution.jl b/CUDACore/src/compiler/execution.jl index 15c9520e21..5a45473f8d 100644 --- a/CUDACore/src/compiler/execution.jl +++ b/CUDACore/src/compiler/execution.jl @@ -737,9 +737,6 @@ function cufunction(f::F, tt::TT=Tuple{}; kwargs...) where {F,TT} end end if fun === nothing - if res.hostcall && !hostcall_available(cuda.device) - error("This kernel calls host functions, but hostcall is not available on this device (see the `hostcall` preference, and `CUDA.hostcall_available`).") - end fun = link_kernel(res.image::Vector{UInt8}, res.entry::String, res.relocations) register_hostcall_targets!(res.hostcall_targets) @@ -760,7 +757,7 @@ function cufunction(f::F, tt::TT=Tuple{}; kwargs...) where {F,TT} # kernels that call host functions get a full-size hostcall area ports = res.hostcall ? hostcall_default_ports(cuda.device) : HOSTCALL_MIN_PORTS exception_info, fallback = create_exceptions!(fun.mod) - client = hostcall_client(ctx, cuda.device, exception_info, fallback; + client = hostcall_client(ctx, exception_info, fallback; ports, heartbeat=!res.hostcall) state = KernelState(client, UInt32(0)) diff --git a/CUDACore/src/device/intrinsics/hostcall.jl b/CUDACore/src/device/intrinsics/hostcall.jl index ebc985f36f..34aa3c7075 100644 --- a/CUDACore/src/device/intrinsics/hostcall.jl +++ b/CUDACore/src/device/intrinsics/hostcall.jl @@ -52,7 +52,8 @@ const HOSTCALL_STATUS_ERROR = UInt32(1) # the handler threw, or the target is Device-side descriptor of a hostcall area: the number of ports and pointers to the mailboxes, headers, packets (all in pinned host memory) and the lock bitfield (in device -memory). A client with `nports == 0` indicates that hostcalls are not available. +memory). A client with `nports == 0` is a placeholder used for kernels compiled during +precompilation, which are never launched. The descriptor also points to the context's exception state. The kernel state holds only a pointer to this descriptor, keeping it small (it is passed to child launches during dynamic parallelism). diff --git a/CUDACore/src/device/runtime.jl b/CUDACore/src/device/runtime.jl index e9ba62c2b9..7493ee3fd5 100644 --- a/CUDACore/src/device/runtime.jl +++ b/CUDACore/src/device/runtime.jl @@ -123,8 +123,8 @@ end end -# exception reports are sent to the host through the hostcall area (when available) as -# packets with this layout, one per report or stack frame. the strings are pointers to +# exception reports are sent to the host through the hostcall area as packets with this +# layout, one per report or stack frame. the strings are pointers to # module-constant device strings, copied over by the host. const EXCEPTION_REPORT_NAME = UInt32(1) # the exception; stack frames follow const EXCEPTION_REPORT_FRAME = UInt32(2) # one stack frame @@ -174,28 +174,6 @@ end end end -@noinline function print_exception_report(ex, info::ExceptionInfo, trace::Bool) - # override the type GPUCompiler deduced if a quirk supplied a subtype - info.subtype != C_NULL && (ex = info.subtype) - @cuprintf("ERROR: a %s was thrown during kernel execution on thread (%d, %d, %d) in block (%d, %d, %d).\n", - ex, threadIdx().x, threadIdx().y, threadIdx().z, - blockIdx().x, blockIdx().y, blockIdx().z) - if info.reason != C_NULL - @cuprintf("%s\n", info.reason) - end - if trace - @cuprintf("Stacktrace:\n") - else - @cuprintf("Stacktrace not available, run Julia on debug level 2 for more details (by passing -g2 to the executable).\n") - end - return -end - -@noinline function print_exception_frame(idx, func, file, line) - @cuprintf(" [%d] %s at %s:%d\n", idx, func, file, line) - return -end - # it's not useful to have several threads report exceptions (interleaved output, can crash # CUDA), so use an output lock to only have a single thread write an exception message @inline function lock_output!(info::ExceptionInfo) @@ -214,6 +192,11 @@ end end end +# NOTE: kernels always run with a functional hostcall client; a no-port client only exists +# for kernels compiled during precompilation, which are never launched. still guard +# against it: `send_exception_report` with zero ports would divide by zero and spin. +# the exception itself is still signalled through the flag, just without a report. + function report_exception(ex) # this is the first reporting function being called, so claim the exception info = exception_info() @@ -223,8 +206,6 @@ function report_exception(ex) send_exception_report(client, ExceptionReport(EXCEPTION_REPORT_NOTRACE, 0, ex, info.subtype, info.reason, 0, threadIdx(), blockIdx())) - else - print_exception_report(ex, info, false) end end return @@ -240,8 +221,6 @@ function report_exception_name(ex) send_exception_report(client, ExceptionReport(EXCEPTION_REPORT_NAME, 0, ex, info.subtype, info.reason, 0, threadIdx(), blockIdx())) - else - print_exception_report(ex, info, true) end end return @@ -256,8 +235,6 @@ function report_exception_frame(idx, func, file, line) send_exception_report(client, ExceptionReport(EXCEPTION_REPORT_FRAME, idx, func, file, C_NULL, line, threadIdx(), blockIdx())) - else - print_exception_frame(idx, func, file, line) end end return @@ -266,11 +243,8 @@ end function signal_exception() info = exception_info() - # finalize output + # mark the report as complete if lock_output!(info) - if hostcall_client().nports == 0 - @cuprintf("\n") - end info.output_lock = 2 end diff --git a/CUDACore/src/hostcall.jl b/CUDACore/src/hostcall.jl index 773db96b5c..ff3ec3b4f6 100644 --- a/CUDACore/src/hostcall.jl +++ b/CUDACore/src/hostcall.jl @@ -7,33 +7,16 @@ # independent of Julia's scheduler, so hostcalls keep being serviced while the launching # thread is blocked in a CUDA call. -@public HostcallException, HostcallArea, hostcall_area, hostcall_drain, hostcall_available +@public HostcallException, HostcallArea, hostcall_area, hostcall_drain using Preferences: @load_preference ## preferences -# whether hostcalls are enabled at all -const hostcall_enabled = @load_preference("hostcall", true)::Bool - # the number of ports per area; defaults to the number of resident warps of the device const hostcall_ports_pref = @load_preference("hostcall_ports", nothing) -""" - hostcall_available([dev::CuDevice]) -> Bool - -Whether hostcalls (and thus hostcall-based functionality such as device exception -reporting) are available on `dev`. Controlled by the `hostcall` preference. Note that on -devices with a display watchdog (Windows WDDM, or a display attached), kernels blocked in -a hostcall remain subject to that watchdog, like any other running kernel. -""" -function hostcall_available(::CuDevice=device()) - hostcall_enabled || return false - get(ENV, "JULIA_CUDA_HOSTCALL", "true") == "false" && return false - return true -end - ## area @@ -173,17 +156,16 @@ function hostcall_area(ctx::CuContext, exception_info::ExceptionInfo; end """ - hostcall_client(ctx, dev, exception_info, fallback; ports, heartbeat) + hostcall_client(ctx, exception_info, fallback; ports, heartbeat) -Pointer to a suitable runtime descriptor for kernels launched in `ctx`. When hostcalls are -unavailable, `fallback` is returned so device exception handling remains usable. +Pointer to a suitable runtime descriptor for kernels launched in `ctx`. """ -function hostcall_client(ctx::CuContext, dev::CuDevice, exception_info::ExceptionInfo, +function hostcall_client(ctx::CuContext, exception_info::ExceptionInfo, fallback::HostcallClientPtr; ports::Integer=HOSTCALL_MIN_PORTS, heartbeat::Bool=true) - hostcall_available(dev) || return fallback # kernels compiled during precompilation are not launched; creating the area would - # start the server thread in the precompilation process + # start the server thread in the precompilation process. such kernels get the no-port + # fallback descriptor, which is never dereferenced on the device. ccall(:jl_generating_output, Cint, ()) != 0 && return fallback return hostcall_area(ctx, exception_info; ports, heartbeat).client end diff --git a/docs/src/api/kernel.md b/docs/src/api/kernel.md index f937d9a63d..c964eb90f0 100644 --- a/docs/src/api/kernel.md +++ b/docs/src/api/kernel.md @@ -131,7 +131,6 @@ shfl_xor_sync hostcall hostcall_async HostcallException -hostcall_available ``` Raw ports, for library code: diff --git a/docs/src/development/debugging.md b/docs/src/development/debugging.md index bad088aaa3..a5690f67b3 100644 --- a/docs/src/development/debugging.md +++ b/docs/src/development/debugging.md @@ -60,8 +60,7 @@ Stacktrace: The device reports these details to the host through the hostcall mechanism (see [Calling host functions from kernels](@ref)), which is why they are part of the exception -rather than printed by the device. When hostcall is unavailable (e.g. disabled by the -`hostcall` preference), the device prints the report to standard output instead. +rather than printed by the device. Note that these messages are embedded in the module (CUDA does not support stack unwinding), and thus bloat its size. To avoid any overhead, you can disable these messages by setting diff --git a/docs/src/development/hostcall.md b/docs/src/development/hostcall.md index 230be2732e..15ea40216b 100644 --- a/docs/src/development/hostcall.md +++ b/docs/src/development/hostcall.md @@ -80,8 +80,9 @@ exception name, reason and (with `-g2`) stack frames through the hostcall area, waiting for the host, and `synchronize()` attaches the decoded report to the `KernelException` it throws (see [Debugging](@ref DebuggingKernels)). This needs no registration, so it also works for kernels compiled during package precompilation. Every kernel that can throw therefore refers to a small -hostcall area (64 ports), which is created on first use in each context; when hostcall is -unavailable, the device falls back to printing the report with `printf`. +hostcall area (64 ports), which is created on first use in each context. Hostcall is core +infrastructure and cannot be disabled: exception reporting depends on it, as will other +functionality built on top of it. ## Performance @@ -113,17 +114,14 @@ switching to the calling kernel's context for every call. Consequences: ## Configuration -Preferences (set with `Preferences.set_preferences!(CUDACore, ...)` and restart; the -preferences belong to the `CUDACore` package, not `CUDA`): -- `hostcall` (default `true`): disable the mechanism entirely; kernels using it fail to - link, and device exceptions fall back to `printf`-based reporting. -- `hostcall_ports`: the number of ports (warp-level call slots) per context; the default is the - number of resident warps of the device (~8 MiB of pinned memory on a large GPU). Contexts - start with a small area until a kernel that calls host functions is linked. +The `hostcall_ports` preference (set with `Preferences.set_preferences!(CUDACore, ...)` and +restart; the preference belongs to the `CUDACore` package, not `CUDA`) controls the number +of ports (warp-level call slots) per context; the default is the number of resident warps +of the device (~8 MiB of pinned memory on a large GPU). Contexts start with a small area +until a kernel that calls host functions is linked. ## Display watchdogs On devices with a display watchdog, a kernel blocked in a hostcall counts as running. A slow handler can therefore push the kernel over the watchdog limit, like any other -long-running kernel. Hostcalls remain enabled by default on these devices, but handlers -should avoid long or unbounded waits. +long-running kernel, so handlers should avoid long or unbounded waits. diff --git a/test/core/exceptions.jl b/test/core/exceptions.jl index ba7b0f7af6..c2bf79d942 100644 --- a/test/core/exceptions.jl +++ b/test/core/exceptions.jl @@ -1,8 +1,7 @@ # XXX: these tests occasionally hang under compute-sanitizer if !sanitize -# with hostcall available (the default), the device's report travels with the exception; -# without it, the device prints it with printf and the exception is message-less +# the device's report travels to the host through hostcall, as part of the exception host_error_re = r"ERROR: (KernelException: .*during kernel execution on device|CUDA error: an illegal instruction was encountered|CUDA error: unspecified launch failure)" device_error_re = r"a \w+ was thrown during kernel execution" @@ -60,16 +59,6 @@ let (proc, out, err) = julia_exec(`-g2 -e $script`) @test !occursin(device_error_re, out) end -# without hostcall, the device prints the report itself -let (proc, out, err) = julia_exec(`-g2 -e $script`, "JULIA_CUDA_HOSTCALL" => "false") - @test !success(proc) - @test occursin(host_error_re, err) - @test count(device_error_re, out) == 1 - @test count("BoundsError", out) == 1 - @test count("Out-of-bounds array access", out) == 1 - @test occursin("] kernel at $(joinpath(".", "none"))", out) -end - end @testset "#329" begin From bf33bba69295c514dc794e5ee57d95970309cb7f Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Mon, 24 Aug 2026 13:39:11 +0200 Subject: [PATCH 07/10] Report device out-of-memory conditions through hostcall. Factor exception transport into the public single-lane asynchronous primitive hostcall_send_scalar!, and use it to report failed allocation sizes. The host merges this detail into the exception report in either arrival order, so synchronization reports the size through KernelException instead of device printf. --- CUDACore/src/compiler/exceptions.jl | 35 +++++++++- CUDACore/src/device/intrinsics/hostcall.jl | 45 +++++++++++++ CUDACore/src/device/runtime.jl | 78 +++++++++------------- CUDACore/src/hostcall.jl | 1 + NEWS.md | 9 +-- docs/src/api/kernel.md | 1 + docs/src/development/hostcall.md | 4 +- test/core/exceptions.jl | 38 +++++++++++ 8 files changed, 160 insertions(+), 51 deletions(-) diff --git a/CUDACore/src/compiler/exceptions.jl b/CUDACore/src/compiler/exceptions.jl index 5818a9dd54..0b1701b102 100644 --- a/CUDACore/src/compiler/exceptions.jl +++ b/CUDACore/src/compiler/exceptions.jl @@ -97,8 +97,13 @@ function service_exception_report(p, hdr) reason = device_string(report.c, stream) @lock kernel_exception_reports_lock begin reports = get!(Dict{NTuple{6,Int}, KernelExceptionInfo}, kernel_exception_reports, area.ctx) + prev = get(reports, key, nothing) + # an OOM report may have recorded a reason before the exception's own + # report arrived (see `service_oom_report`) + if isempty(reason) && prev !== nothing + reason = prev.reason + end frames = if report.kind == EXCEPTION_REPORT_NAME - prev = get(reports, key, nothing) prev === nothing || prev.stacktrace === nothing ? @NamedTuple{idx::Int, func::String, file::String, line::Int}[] : prev.stacktrace else nothing @@ -111,6 +116,34 @@ function service_exception_report(p, hdr) return end +# the built-in handler for out-of-memory reports: attach the size of the failed allocation +# to the reporting thread's exception report (the device throws an OutOfMemoryError right +# after sending this, but its reports may be serviced in any order) +function service_oom_report(p, hdr) + mask = hdr.mask + for lane in 0:31 + (mask >> lane) & 1 == 0 && continue + report = unsafe_load(convert(Ptr{OOMReport}, lane_packet(p, lane))) + key = (Int(report.block.x), Int(report.block.y), Int(report.block.z), + Int(report.thread.x), Int(report.thread.y), Int(report.thread.z)) + reason = "Out of dynamic GPU memory (trying to allocate $(report.sz) bytes)" + @lock kernel_exception_reports_lock begin + reports = get!(Dict{NTuple{6,Int}, KernelExceptionInfo}, kernel_exception_reports, p.area.ctx) + info = get(reports, key, nothing) + if info === nothing + # the exception's own report has not arrived yet; create a placeholder + reports[key] = KernelExceptionInfo("", "", reason, key[4:6], key[1:3], + @NamedTuple{idx::Int, func::String, file::String, line::Int}[]) + elseif isempty(info.reason) + reports[key] = KernelExceptionInfo(info.name, info.subtype, reason, + info.thread, info.block, info.stacktrace) + end + end + end + hport_flip!(p) + return +end + # take the reports of a context, if any function take_exception_reports!(ctx::CuContext) @lock kernel_exception_reports_lock begin diff --git a/CUDACore/src/device/intrinsics/hostcall.jl b/CUDACore/src/device/intrinsics/hostcall.jl index 34aa3c7075..c78c42b41b 100644 --- a/CUDACore/src/device/intrinsics/hostcall.jl +++ b/CUDACore/src/device/intrinsics/hostcall.jl @@ -13,6 +13,7 @@ export @hostcall, hostcall, hostcall_async @public HostcallClient, HostcallPort, HostcallHeader, hostcall_open, hostcall_send!, hostcall_recv!, hostcall_close!, + hostcall_send_scalar!, hostcall_lane_packet, HOSTCALL_PACKET_SIZE, hostcall_packet_layout @@ -41,6 +42,7 @@ const HOSTCALL_FLAG_ASYNC = UInt32(1) const HOSTCALL_BUILTIN_IDS = UInt64(256) const HOSTCALL_STATIC_ID_BIT = UInt64(0x8000_0000_0000_0000) const HC_EXCEPTION = UInt64(1) +const HC_OOM = UInt64(2) # inbox words carry a status in the bits above the ownership bit; the host sets these # when it replies to a port. bit 0 is the ownership bit. @@ -357,6 +359,49 @@ receive) the call is completed asynchronously by the host. return end +""" + hostcall_send_scalar!(client::HostcallClient, target::UInt64, value) + +Send a single `value` (at most one packet in size) to `target` from the calling lane, +without waiting for the host to service the call. Unlike the warp-collective port API, +this may be called from arbitrarily divergent code, on any hardware: it involves no warp +intrinsics. The runtime uses it for exception and out-of-memory reports. +""" +# Keep this inline: CUDA 12.9 ptxas crashes on the debug information for an out-of-line +# function taking an aggregate value by value. Inlining also produces less PTX. +@inline function hostcall_send_scalar!(c::HostcallClient, target::UInt64, + value::T) where {T} + GPUCompiler.@static_assert(sizeof(T) <= HOSTCALL_PACKET_SIZE, + "hostcall_send_scalar! values must fit in one packet") + index = hostcall_start_index(c.nports) + lane = laneid() - Int32(1) + mask = UInt32(1) << lane + while true + word = c.lock + 4 * (index >> 5) + bit = UInt32(1) << (index & UInt32(31)) + if (atomic_or!(word, bit) & bit) == 0 + fence_gpu() + in = mailbox_load(c.inbox + 4index) + out = mailbox_load(c.outbox + 4index) + if hostcall_owned(in, out) + unsafe_store!(c.header + sizeof(HostcallHeader) * index, + HostcallHeader(mask, UInt32(0), target)) + packet = c.packet + (index * HOSTCALL_LANES + lane) * HOSTCALL_PACKET_SIZE + unsafe_store!(reinterpret(LLVMPtr{T,AS.Global}, packet), value) + fence_sys() + mailbox_store!(c.outbox + 4index, out ⊻ UInt32(1)) + fence_gpu() + atomic_and!(word, ~bit) + return + end + fence_gpu() + atomic_and!(word, ~bit) + end + index += UInt32(1) + index >= c.nports && (index = UInt32(0)) + end +end + ## value marshalling diff --git a/CUDACore/src/device/runtime.jl b/CUDACore/src/device/runtime.jl index 7493ee3fd5..c49ea17e8e 100644 --- a/CUDACore/src/device/runtime.jl +++ b/CUDACore/src/device/runtime.jl @@ -140,39 +140,8 @@ struct ExceptionReport block::@NamedTuple{x::Int32,y::Int32,z::Int32} end -# Send a report without waiting for the host. The exception output lock admits exactly one -# lane, so this only needs the scalar subset of the warp-collective hostcall protocol. -# Keep this inline: CUDA 12.9 ptxas crashes on the debug information for an out-of-line -# function taking ExceptionReport by value. Inlining also produces less PTX for this path. -@inline function send_exception_report(client::HostcallClient, report::ExceptionReport) - index = hostcall_start_index(client.nports) - lane = laneid() - Int32(1) - mask = UInt32(1) << lane - while true - word = client.lock + 4 * (index >> 5) - bit = UInt32(1) << (index & UInt32(31)) - if (atomic_or!(word, bit) & bit) == 0 - fence_gpu() - in = mailbox_load(client.inbox + 4index) - out = mailbox_load(client.outbox + 4index) - if hostcall_owned(in, out) - unsafe_store!(client.header + sizeof(HostcallHeader) * index, - HostcallHeader(mask, UInt32(0), HC_EXCEPTION)) - packet = client.packet + (index * HOSTCALL_LANES + lane) * HOSTCALL_PACKET_SIZE - unsafe_store!(reinterpret(LLVMPtr{ExceptionReport,AS.Global}, packet), report) - fence_sys() - mailbox_store!(client.outbox + 4index, out ⊻ UInt32(1)) - fence_gpu() - atomic_and!(word, ~bit) - return - end - fence_gpu() - atomic_and!(word, ~bit) - end - index += UInt32(1) - index >= client.nports && (index = UInt32(0)) - end -end +# reports are sent through `hostcall_send_scalar!`: the exception output lock admits +# exactly one lane, so the warp-collective port protocol is not needed. # it's not useful to have several threads report exceptions (interleaved output, can crash # CUDA), so use an output lock to only have a single thread write an exception message @@ -194,7 +163,7 @@ end # NOTE: kernels always run with a functional hostcall client; a no-port client only exists # for kernels compiled during precompilation, which are never launched. still guard -# against it: `send_exception_report` with zero ports would divide by zero and spin. +# against it: `hostcall_send_scalar!` with zero ports would divide by zero and spin. # the exception itself is still signalled through the flag, just without a report. function report_exception(ex) @@ -203,7 +172,7 @@ function report_exception(ex) if lock_output!(info) client = hostcall_client() if client.nports != 0 - send_exception_report(client, + hostcall_send_scalar!(client, HC_EXCEPTION, ExceptionReport(EXCEPTION_REPORT_NOTRACE, 0, ex, info.subtype, info.reason, 0, threadIdx(), blockIdx())) end @@ -218,7 +187,7 @@ function report_exception_name(ex) if lock_output!(info) client = hostcall_client() if client.nports != 0 - send_exception_report(client, + hostcall_send_scalar!(client, HC_EXCEPTION, ExceptionReport(EXCEPTION_REPORT_NAME, 0, ex, info.subtype, info.reason, 0, threadIdx(), blockIdx())) end @@ -232,7 +201,7 @@ function report_exception_frame(idx, func, file, line) if lock_output!(info) client = hostcall_client() if client.nports != 0 - send_exception_report(client, + hostcall_send_scalar!(client, HC_EXCEPTION, ExceptionReport(EXCEPTION_REPORT_FRAME, idx, func, file, C_NULL, line, threadIdx(), blockIdx())) end @@ -240,6 +209,33 @@ function report_exception_frame(idx, func, file, line) return end +# out-of-memory reports carry the size of the failed allocation. `gc_pool_alloc` throws an +# OutOfMemoryError right after calling `report_oom`, whose report only carries the +# exception name; the host merges the two by thread and attaches the size as the reason. +struct OOMReport + sz::UInt64 + thread::@NamedTuple{x::Int32,y::Int32,z::Int32} + block::@NamedTuple{x::Int32,y::Int32,z::Int32} +end + +function report_oom(sz) + # this runs before the ensuing exception, so claim that exception's report: the same + # thread re-enters the output lock when it throws, and other threads (which may be + # failing the same allocation) do not send reports of their own + info = exception_info() + if lock_output!(info) + # the throw in `gc_pool_alloc` lowers to a generic "exception" name, so label the + # report like the quirks do + info.subtype = @strptr "OutOfMemoryError" + client = hostcall_client() + if client.nports != 0 + hostcall_send_scalar!(client, HC_OOM, + OOMReport(sz % UInt64, threadIdx(), blockIdx())) + end + end + return +end + function signal_exception() info = exception_info() @@ -273,11 +269,3 @@ end end @inline exception_info() = reinterpret(ExceptionInfo, hostcall_client().exception_info) - - -## other - -function report_oom(sz) - @cuprintf("ERROR: Out of dynamic GPU memory (trying to allocate %d bytes)\n", sz) - return -end diff --git a/CUDACore/src/hostcall.jl b/CUDACore/src/hostcall.jl index ff3ec3b4f6..91967cce06 100644 --- a/CUDACore/src/hostcall.jl +++ b/CUDACore/src/hostcall.jl @@ -520,6 +520,7 @@ end # built-in targets, implemented by the runtime library and serviced by these handlers const hostcall_builtins = Dict{UInt64,Function}( HC_EXCEPTION => service_exception_report, + HC_OOM => service_oom_report, ) function service_port!(a::HostcallArea, i::Int, out::UInt32) diff --git a/NEWS.md b/NEWS.md index 93db5035e7..45ed7599b9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -33,10 +33,11 @@ thread; `@hostcall async=true f(args...)` is a fire-and-forget variant whose call has completed by the next synchronization. A raw port API is available for library code. The calls are serviced by a dedicated host thread, so they keep making progress even when Julia's own threads are blocked in CUDA API -calls. Device-side exceptions now use the same mechanism: the -`KernelException` thrown when synchronizing carries the exception type, -message and (with `-g2`) device stack trace, instead of the device printing -those details to standard output +calls. Device-side exception and out-of-memory reporting now use the same +mechanism: the `KernelException` thrown when synchronizing carries the +exception type, message, (with `-g2`) device stack trace, and the size of a +failed device-side allocation, instead of the device printing those details +to standard output ([#3243](https://github.com/JuliaGPU/CUDA.jl/pull/3243)). *Technically breaking changes*: diff --git a/docs/src/api/kernel.md b/docs/src/api/kernel.md index c964eb90f0..d9d96a598f 100644 --- a/docs/src/api/kernel.md +++ b/docs/src/api/kernel.md @@ -142,6 +142,7 @@ hostcall_open hostcall_send! hostcall_recv! hostcall_close! +hostcall_send_scalar! hostcall_lane_packet ``` diff --git a/docs/src/development/hostcall.md b/docs/src/development/hostcall.md index 15ea40216b..2db98fb43b 100644 --- a/docs/src/development/hostcall.md +++ b/docs/src/development/hostcall.md @@ -79,7 +79,9 @@ CUDA.jl itself uses hostcall to report device-side exceptions: the runtime libra exception name, reason and (with `-g2`) stack frames through the hostcall area, without waiting for the host, and `synchronize()` attaches the decoded report to the `KernelException` it throws (see [Debugging](@ref DebuggingKernels)). This needs no registration, so it also works for kernels -compiled during package precompilation. Every kernel that can throw therefore refers to a small +compiled during package precompilation. Out-of-memory failures of device-side allocations +are reported the same way: the size of the failed allocation is attached to the +`OutOfMemoryError` that follows it. Every kernel that can throw therefore refers to a small hostcall area (64 ports), which is created on first use in each context. Hostcall is core infrastructure and cannot be disabled: exception reporting depends on it, as will other functionality built on top of it. diff --git a/test/core/exceptions.jl b/test/core/exceptions.jl index c2bf79d942..c7f19f1080 100644 --- a/test/core/exceptions.jl +++ b/test/core/exceptions.jl @@ -61,6 +61,44 @@ end end +@testset "out-of-memory reports" begin + +# device-side allocation failures attach the size of the failed allocation to the +# ensuing OutOfMemoryError's report +script = """ + using CUDA + + CUDA.limit!(CUDA.LIMIT_MALLOC_HEAP_SIZE, 32*1024) + + mutable struct Box + x::Int64 + end + @noinline make_box(x) = Box(x) + + function kernel(a) + i = 1 + while i <= 100_000 + b = make_box(Int64(i)) + a[1] = b.x + i += 1 + end + return + end + + a = CuArray{Int64}(undef, 1) + @cuda kernel(a) + synchronize() +""" + +let (proc, out, err) = julia_exec(`-g1 -e $script`) + @test !success(proc) + @test occursin(host_error_re, err) + @test occursin("OutOfMemoryError", err) + @test occursin(r"Out of dynamic GPU memory \(trying to allocate \d+ bytes\)", err) +end + +end + @testset "#329" begin script = """ From 7a842c6cc7228365d147b8e9a58d34bf71b737c8 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Mon, 24 Aug 2026 14:06:00 +0200 Subject: [PATCH 08/10] Add a blocking scalar hostcall round trip. Factor the single-lane port claim, submit, and unlock operations out of hostcall_send_scalar!, and add hostcall_call_scalar! for request and reply. The primitive uses no warp collectives, so an elected lane may call from divergent code; pre-Volta callers must not independently elect several lanes from one converged warp. --- CUDACore/src/device/intrinsics/hostcall.jl | 115 ++++++++++++++++----- docs/src/api/kernel.md | 1 + docs/src/development/hostcall.md | 5 + test/core/device/hostcall.jl | 107 +++++++++++++++++++ 4 files changed, 201 insertions(+), 27 deletions(-) diff --git a/CUDACore/src/device/intrinsics/hostcall.jl b/CUDACore/src/device/intrinsics/hostcall.jl index c78c42b41b..7f183b093f 100644 --- a/CUDACore/src/device/intrinsics/hostcall.jl +++ b/CUDACore/src/device/intrinsics/hostcall.jl @@ -13,7 +13,7 @@ export @hostcall, hostcall, hostcall_async @public HostcallClient, HostcallPort, HostcallHeader, hostcall_open, hostcall_send!, hostcall_recv!, hostcall_close!, - hostcall_send_scalar!, + hostcall_send_scalar!, hostcall_call_scalar!, hostcall_lane_packet, HOSTCALL_PACKET_SIZE, hostcall_packet_layout @@ -359,23 +359,18 @@ receive) the call is completed asynchronously by the host. return end -""" - hostcall_send_scalar!(client::HostcallClient, target::UInt64, value) +# The scalar tier: operations for one elected lane that involve no warp intrinsics, so the +# lane may call from divergent code on any hardware. On pre-Volta hardware, multiple lanes +# of one converged warp must not claim ports independently: lockstep reconvergence can keep +# successful claimants from submitting while their peers spin. The runtime's exception +# lock enforces the single-lane condition; other callers must do likewise. +# +# Keep everything inline: CUDA 12.9 ptxas crashes on the debug information for an +# out-of-line function taking an aggregate value by value. Inlining also produces less PTX. -Send a single `value` (at most one packet in size) to `target` from the calling lane, -without waiting for the host to service the call. Unlike the warp-collective port API, -this may be called from arbitrarily divergent code, on any hardware: it involves no warp -intrinsics. The runtime uses it for exception and out-of-memory reports. -""" -# Keep this inline: CUDA 12.9 ptxas crashes on the debug information for an out-of-line -# function taking an aggregate value by value. Inlining also produces less PTX. -@inline function hostcall_send_scalar!(c::HostcallClient, target::UInt64, - value::T) where {T} - GPUCompiler.@static_assert(sizeof(T) <= HOSTCALL_PACKET_SIZE, - "hostcall_send_scalar! values must fit in one packet") +# claim a free, device-owned port for the calling lane; returns the index and outbox word +@inline function hostcall_claim_scalar!(c::HostcallClient) index = hostcall_start_index(c.nports) - lane = laneid() - Int32(1) - mask = UInt32(1) << lane while true word = c.lock + 4 * (index >> 5) bit = UInt32(1) << (index & UInt32(31)) @@ -383,17 +378,7 @@ intrinsics. The runtime uses it for exception and out-of-memory reports. fence_gpu() in = mailbox_load(c.inbox + 4index) out = mailbox_load(c.outbox + 4index) - if hostcall_owned(in, out) - unsafe_store!(c.header + sizeof(HostcallHeader) * index, - HostcallHeader(mask, UInt32(0), target)) - packet = c.packet + (index * HOSTCALL_LANES + lane) * HOSTCALL_PACKET_SIZE - unsafe_store!(reinterpret(LLVMPtr{T,AS.Global}, packet), value) - fence_sys() - mailbox_store!(c.outbox + 4index, out ⊻ UInt32(1)) - fence_gpu() - atomic_and!(word, ~bit) - return - end + hostcall_owned(in, out) && return index, out fence_gpu() atomic_and!(word, ~bit) end @@ -402,6 +387,82 @@ intrinsics. The runtime uses it for exception and out-of-memory reports. end end +@inline function hostcall_unlock_scalar!(c::HostcallClient, index::UInt32) + fence_gpu() + word = c.lock + 4 * (index >> 5) + bit = UInt32(1) << (index & UInt32(31)) + atomic_and!(word, ~bit) + return +end + +# write the header and the lane's packet, and hand the buffer to the host; returns the +# flipped outbox word +@inline function hostcall_submit_scalar!(c::HostcallClient, index::UInt32, out::UInt32, + target::UInt64, value::T) where {T} + lane = laneid() - Int32(1) + mask = UInt32(1) << lane + unsafe_store!(c.header + sizeof(HostcallHeader) * index, + HostcallHeader(mask, UInt32(0), target)) + packet = c.packet + (index * HOSTCALL_LANES + lane) * HOSTCALL_PACKET_SIZE + unsafe_store!(reinterpret(LLVMPtr{T,AS.Global}, packet), value) + fence_sys() + inverted = out ⊻ UInt32(1) + mailbox_store!(c.outbox + 4index, inverted) + return inverted +end + +""" + hostcall_send_scalar!(client::HostcallClient, target::UInt64, value) + +Send a single `value` (at most one packet in size) to `target` from one elected lane, +without waiting for the host to service the call. Unlike the warp-collective port API, +this may be called from divergent code and involves no warp intrinsics. On pre-Volta +hardware, do not call it independently from several lanes of one converged warp. See also +[`hostcall_call_scalar!`](@ref) for the blocking variant. +""" +@inline function hostcall_send_scalar!(c::HostcallClient, target::UInt64, + value::T) where {T} + GPUCompiler.@static_assert(sizeof(T) <= HOSTCALL_PACKET_SIZE, + "hostcall_send_scalar! values must fit in one packet") + index, out = hostcall_claim_scalar!(c) + hostcall_submit_scalar!(c, index, out, target, value) + hostcall_unlock_scalar!(c, index) + return +end + +""" + hostcall_call_scalar!(client::HostcallClient, target::UInt64, request, RT) -> (value, status) + +Send `request` to `target` from one elected lane and wait for the host's reply, returning +the value of type `RT` read back from the packet and the status word set by the host (`0` +on success). Like [`hostcall_send_scalar!`](@ref), this may be called from divergent code, +but several lanes of one converged pre-Volta warp must not call it independently. Both +`request` and `RT` must fit in one packet. +""" +@inline function hostcall_call_scalar!(c::HostcallClient, target::UInt64, request::T, + ::Type{RT}) where {T,RT} + GPUCompiler.@static_assert(sizeof(T) <= HOSTCALL_PACKET_SIZE, + "hostcall_call_scalar! requests must fit in one packet") + GPUCompiler.@static_assert(sizeof(RT) <= HOSTCALL_PACKET_SIZE, + "hostcall_call_scalar! replies must fit in one packet") + index, out = hostcall_claim_scalar!(c) + out = hostcall_submit_scalar!(c, index, out, target, request) + # wait for the reply (scalar counterpart of `hostcall_wait_for_ownership`, which is + # warp-collective and thus off limits here) + ns = HOSTCALL_BACKOFF_MIN + in = mailbox_load(c.inbox + 4index) + while !hostcall_owned(in, out) + ns = hostcall_backoff(ns) + in = mailbox_load(c.inbox + 4index) + end + fence_sys() + lane = laneid() - Int32(1) + packet = c.packet + (index * HOSTCALL_LANES + lane) * HOSTCALL_PACKET_SIZE + val = unsafe_load(reinterpret(LLVMPtr{RT,AS.Global}, packet)) + hostcall_unlock_scalar!(c, index) + return val, in >> 1 +end + ## value marshalling diff --git a/docs/src/api/kernel.md b/docs/src/api/kernel.md index d9d96a598f..5a502a5d2d 100644 --- a/docs/src/api/kernel.md +++ b/docs/src/api/kernel.md @@ -143,6 +143,7 @@ hostcall_send! hostcall_recv! hostcall_close! hostcall_send_scalar! +hostcall_call_scalar! hostcall_lane_packet ``` diff --git a/docs/src/development/hostcall.md b/docs/src/development/hostcall.md index 2db98fb43b..2a03be3188 100644 --- a/docs/src/development/hostcall.md +++ b/docs/src/development/hostcall.md @@ -31,6 +31,11 @@ Two layers are available, both built on the same protocol. `hostcall(f, R, args...)` and `hostcall_async(f, args...)`. - Raw ports (`hostcall_open`, `hostcall_send!`, `hostcall_recv!`, `hostcall_close!`) for library code that wants to stream data through the 64-byte per-lane packets itself. + These are warp-collective. A scalar tier (`hostcall_send_scalar!`, + `hostcall_call_scalar!`) sends a single packet from one elected lane — fire-and-forget + or as a blocking round trip — without any warp intrinsics. The elected lane may be in + divergent code; on pre-Volta hardware, several lanes of one converged warp must not use + this tier independently. The runtime uses it for exception and out-of-memory reports. ## Semantics and rules diff --git a/test/core/device/hostcall.jl b/test/core/device/hostcall.jl index 60112d260e..871a2c081a 100644 --- a/test/core/device/hostcall.jl +++ b/test/core/device/hostcall.jl @@ -1,5 +1,6 @@ using CUDA: HostcallClient, HostcallPort, HostcallHeader, hostcall_open, hostcall_send!, hostcall_recv!, hostcall_close!, + hostcall_send_scalar!, hostcall_call_scalar!, hostcall_lane_packet, HOSTCALL_PACKET_SIZE, hostcall_packet_layout @testset "@hostcall syntax" begin @@ -231,6 +232,112 @@ end end end +@testset "scalar tier" begin + area = TestPoller.Area(64) + try + OP_ADD1 = UInt64(1) + OP_RECORD = UInt64(3) + OP_FAIL = UInt64(4) + records = Threads.Atomic{Int}(0) + function handler(target, lane, pkt) + p = convert(Ptr{UInt64}, pkt) + if target == OP_ADD1 + unsafe_store!(p, unsafe_load(p) + 1) + elseif target == OP_RECORD + Threads.atomic_add!(records, 1) + elseif target == OP_FAIL + return 1 + end + return 0 + end + + # blocking round trips and fire-and-forget sends from divergent lanes + function scalar_kernel(client, out) + t = (blockIdx().x - 1) * blockDim().x + threadIdx().x + if isodd(t) + v, _ = hostcall_call_scalar!(client, OP_ADD1, UInt64(t), UInt64) + out[t] = v + else + hostcall_send_scalar!(client, OP_RECORD, UInt64(t)) + out[t] = UInt64(t) + end + return + end + if capability(device()) >= v"7.0" + for (threads, blocks) in [(32, 1), (20, 2), (256, 4)] + records[] = 0 + out = CUDA.zeros(UInt64, threads * blocks) + @cuda threads=threads blocks=blocks scalar_kernel(area.client, out) + TestPoller.serve!(handler, area) + synchronize() + n = threads * blocks + @test Array(out) == [isodd(t) ? UInt64(t + 1) : UInt64(t) for t in 1:n] + @test records[] == count(iseven, 1:n) + end + else + # Pre-Volta warps execute divergent paths in lockstep. Elect one lane so a + # successful claimant can submit without waiting for peers that still need a + # port; callers must use the same pattern. + function scalar_elected_kernel(client, out) + if laneid() == 1 + t = (blockIdx().x - 1) * blockDim().x + threadIdx().x + v, _ = hostcall_call_scalar!(client, OP_ADD1, UInt64(t), UInt64) + hostcall_send_scalar!(client, OP_RECORD, v) + out[t] = v + end + return + end + for (threads, blocks) in [(32, 1), (20, 2), (256, 4)] + records[] = 0 + out = CUDA.zeros(UInt64, threads * blocks) + @cuda threads=threads blocks=blocks scalar_elected_kernel(area.client, out) + TestPoller.serve!(handler, area) + synchronize() + expected = zeros(UInt64, threads * blocks) + for block in 0:blocks-1, thread in 1:32:threads + t = block * threads + thread + expected[t] = t + 1 + end + @test Array(out) == expected + @test records[] == blocks * cld(threads, 32) + end + end + + # an error status from the host is returned alongside the value + function failing_scalar(client, out) + _, status = hostcall_call_scalar!(client, OP_FAIL, UInt64(0), UInt64) + out[threadIdx().x] = status + return + end + threads = capability(device()) >= v"7.0" ? 4 : 1 + out = CUDA.zeros(UInt32, threads) + @cuda threads=threads failing_scalar(area.client, out) + TestPoller.serve!(handler, area) + synchronize() + @test all(==(1), Array(out)) + + # all ports are released afterwards + @test all(==(0), Array(area.locks)) + finally + TestPoller.free(area) + end + + # the scalar tier services divergent code, so it must not use warp intrinsics + function scalar_probe(client, out) + v, _ = hostcall_call_scalar!(client, UInt64(1), UInt64(1), UInt64) + hostcall_send_scalar!(client, UInt64(2), v) + out[1] = v + return + end + tt = Tuple{HostcallClient, CuDeviceVector{UInt64,1}} + for arch in [sm"70", sm"60"] + ptx = sprint(io -> CUDA.code_ptx(io, scalar_probe, tt; arch)) + @test !occursin("activemask", ptx) + @test !occursin("bar.warp.sync", ptx) + @test !occursin("shfl", ptx) + end +end + @testset "memory model" begin # the mailbox accesses should be system-scope, the fences and sleeps present function probe(client, out) From 49ee415caed1bbe1beaa4785c8817d8502bb0baa Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Mon, 24 Aug 2026 15:48:47 +0200 Subject: [PATCH 09/10] Identify and invoke hostcall targets like Julia's invoke. Use the relocated pointer to each target's key type as its wire identifier. Julia codegen and the relocation resolver root and canonicalize these values, so the image and host registry agree without hashes or collision handling, including across cached images. Cache each target's dispatch-resolved MethodInstance by world and call Julia's exported jl_invoke entry point. This preserves invokelatest semantics across handler redefinition while leaving CodeInstance publication, compilation, and invocation to Julia's runtime. Publish registry snapshots atomically so the service thread reads them without locking. --- CUDACore/src/compiler/compilation.jl | 32 +++-- CUDACore/src/device/intrinsics/hostcall.jl | 22 +-- CUDACore/src/hostcall.jl | 156 +++++++++++++++++---- docs/src/development/hostcall.md | 7 + test/core/hostcall.jl | 53 ++++++- 5 files changed, 218 insertions(+), 52 deletions(-) diff --git a/CUDACore/src/compiler/compilation.jl b/CUDACore/src/compiler/compilation.jl index 6467a0d8ce..e0ff034e37 100644 --- a/CUDACore/src/compiler/compilation.jl +++ b/CUDACore/src/compiler/compilation.jl @@ -201,17 +201,19 @@ mutable struct CUDACompilerResults entry::Union{Nothing,String} relocations::GPUCompiler.Relocations - # whether the kernel calls host functions, and the statically-known targets of those - # calls (identifier => key type), as recovered from the compiled method instances; the - # identifiers are baked into the image, so the table travels with it + # whether the kernel calls host functions, and the key types of the statically-known + # targets of those calls, as recovered from the compiled method instances. the wire + # identifiers are not stored: they are the addresses of the egal-rooted target markers, + # recomputed at registration in each session (the image's relocated marker slots + # resolve to the same addresses) hostcall::Bool - hostcall_targets::Vector{Pair{UInt64,Type}} + hostcall_targets::Vector{Type} # session-local kernel handles, linear-scanned by context; usually holds a single entry kernels::Vector{Tuple{CuContext,CuFunction}} CUDACompilerResults() = new(nothing, nothing, GPUCompiler.Relocations(), - false, Pair{UInt64,Type}[], + false, Type[], Tuple{CuContext,CuFunction}[]) end @@ -386,6 +388,17 @@ device_compatible_layout(@nospecialize(T)) = Base.datatype_alignment(Int128) == 16 || !layout_reaches(S -> device_layout(S) === :mismatch, T) +# Recover the value of a type-valued dispatch key. Julia 1.14 uses `Core.TypeEgal{T}` +# for closed type arguments, while earlier releases represent them as `Type{T}`. +@inline function hostcall_key_type(@nospecialize(T)) + Base.isType(T) || return nothing + @static if isdefined(Base, :type_parameter) + return Base.type_parameter(T) + else + return T.parameters[1] + end +end + # compile to executable machine code function compile(@nospecialize(job::CompilerJob)) # lower to PTX @@ -398,15 +411,14 @@ function compile(@nospecialize(job::CompilerJob)) # key type of every statically-known call, and `meta.compiled` lists everything codegen # emitted (including deferred compilation jobs) hostcall = false - hostcall_targets = Pair{UInt64,Type}[] + hostcall_targets = Type[] for mi in keys(meta.compiled) mi.def isa Method || continue mi.def.module === CUDACore && mi.def.name === :hostcall_impl || continue hostcall = true - K = mi.specTypes.parameters[2] - K isa DataType && K <: Type && K !== Type || continue - K = K.parameters[1] - push!(hostcall_targets, hostcall_target_id_value(K) => K) + K = hostcall_key_type(mi.specTypes.parameters[2]) + K === nothing && continue + push!(hostcall_targets, K) end # check if we'll need the device runtime diff --git a/CUDACore/src/device/intrinsics/hostcall.jl b/CUDACore/src/device/intrinsics/hostcall.jl index 7f183b093f..c083f805dd 100644 --- a/CUDACore/src/device/intrinsics/hostcall.jl +++ b/CUDACore/src/device/intrinsics/hostcall.jl @@ -38,9 +38,9 @@ const HOSTCALL_PORT_BYTES = HOSTCALL_PACKET_SIZE * HOSTCALL_LANES # Header flags used by the generic host service. Raw protocols may use the remaining bits. const HOSTCALL_FLAG_ASYNC = UInt32(1) -# Built-in targets occupy the low identifiers; static targets have the high bit set. +# Built-in targets occupy the low identifiers; statically-known targets are identified by +# the address of their rooted key type `Tuple{F,RT,AT}`, which is never that small. const HOSTCALL_BUILTIN_IDS = UInt64(256) -const HOSTCALL_STATIC_ID_BIT = UInt64(0x8000_0000_0000_0000) const HC_EXCEPTION = UInt64(1) const HC_OOM = UInt64(2) @@ -546,16 +546,16 @@ end hostconvert(x) = x hostconvert(p::LLVMPtr{T}) where {T} = reinterpret(CuPtr{T}, p) -# The hash is computed while compiling and travels with the image. The high bit keeps -# static targets disjoint from built-ins; the registry detects collisions. -function hostcall_target_id_value(@nospecialize(K::Type)) - return (hash(K) % UInt64) | HOSTCALL_STATIC_ID_BIT -end -@generated hostcall_target_id(::Type{K}) where {K} = :($(hostcall_target_id_value(K))) +# `jl_value_ptr` is a codegen intrinsic (no runtime call): since `K` is a compile-time +# literal, this lowers to a load from its relocation slot, like `emit_invoke`'s literal +# MethodInstance pointer. +@inline hostcall_target_id(::Type{K}) where {K} = + reinterpret(UInt64, ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), K)) -# the marker function the compiler scans for: `K` is the registry key -# (`Tuple{typeof(f), RT, AT}`). it must not be inlined so that its specializations show up -# in the compiled method instances, and must not throw. +# `K` is the registry key (`Tuple{typeof(f), RT, AT}`). the compiler scans the compiled +# method instances for specializations of this function to find the kernel's targets, so +# it must not be inlined (which also keeps the protocol out of the kernel's hot code), and +# must not throw. @noinline function hostcall_impl(::Type{K}, ::Type{RT}, args::AT, ::Val{async}) where {K, RT, AT<:Tuple, async} client = hostcall_client() diff --git a/CUDACore/src/hostcall.jl b/CUDACore/src/hostcall.jl index 91967cce06..5e4d19e24c 100644 --- a/CUDACore/src/hostcall.jl +++ b/CUDACore/src/hostcall.jl @@ -173,47 +173,121 @@ end ## target registry -# a statically-known host function and how its arguments and result are encoded -struct HostcallTarget - key::Type # registry key `Tuple{F,RT,AT}` - f::Any - RT::Type # return type (`Nothing` for calls without a reply) - AT::Type # payload tuple type; includes `f` when it is not stored - stored::Bool # callable is stored in `f`, rather than included in the payload -end - -const hostcall_targets = Dict{UInt64,HostcallTarget}() +# a statically-known host function and how its arguments and result are encoded. `method` +# caches the MethodInstance that `jl_invoke` calls through, tagged with the world in which +# dispatch resolved it (see "invoking targets" below). +mutable struct HostcallTarget + const key::Type # registry key `Tuple{F,RT,AT}` + const f::Any + const RT::Type # return type (`Nothing` for calls without a reply) + const AT::Type # payload tuple type; includes `f` when it is not stored + const stored::Bool # callable is stored in `f`, rather than included in the payload + Base.@atomic method::Union{Nothing,Tuple{UInt,Core.MethodInstance}} +end + +# targets keyed by the address of their rooted key type — the word the kernel's relocated +# literal slot holds. an immutable snapshot is swapped on registration, so the server +# thread reads it without locking (like `hostcall_areas`). +mutable struct HostcallTargets + Base.@atomic table::Dict{UInt64,HostcallTarget} +end +const hostcall_targets = HostcallTargets(Dict{UInt64,HostcallTarget}()) const hostcall_targets_lock = Threads.SpinLock() +# The wire identifier of a key: the relocation resolver permanently roots and canonicalizes +# the key type, then returns its address. This matches the word in the image whether codegen +# baked the literal directly or `link_kernel` re-resolved a cached image's relocation slot. +hostcall_target_word(@nospecialize(K::Type)) = + UInt64(GPUCompiler.resolve_relocation_target(GPUCompiler.JuliaValueRef(K))) + """ - register_hostcall_targets!(targets) + register_hostcall_targets!(keys) -Register the statically-known hostcall targets of a kernel (pairs of identifier and key -type `Tuple{F,RT,AT}`, as recorded by the compiler) so that the server can dispatch calls. +Register the statically-known hostcall targets of a kernel (key types `Tuple{F,RT,AT}`, as +recorded by the compiler) so that the server can dispatch calls. Registration is +idempotent: a key's identifier is the address of the key type itself, so distinct keys +cannot collide. """ -function register_hostcall_targets!(targets) - isempty(targets) && return - for (id, K) in targets +function register_hostcall_targets!(keys) + isempty(keys) && return + for K in keys + id = hostcall_target_word(K) + haskey(Base.@atomic(hostcall_targets.table), id) && continue F, RT, AT = K.parameters stored = Base.issingletontype(F) f = stored ? F.instance : nothing - target = HostcallTarget(K, f, RT, AT, stored) + target = HostcallTarget(K, f, RT, AT, stored, nothing) - # Compile the handler on the registering thread, before taking the registry lock. - precompile(stored ? Tuple{F, AT.parameters...} : AT) + # Compile the handler and resolve its MethodInstance on the registering thread, + # before publishing the target: the server thread should not have to compile. + seed_hostcall_method!(target) @lock hostcall_targets_lock begin - previous = get(hostcall_targets, id, nothing) - if previous === nothing - hostcall_targets[id] = target - elseif previous.key !== K - error("hostcall target identifier collision between $(previous.key) and $K") + table = Base.@atomic hostcall_targets.table + if !haskey(table, id) + table = copy(table) + table[id] = target + Base.@atomic hostcall_targets.table = table end end end return end -hostcall_target(id::UInt64) = @lock hostcall_targets_lock get(hostcall_targets, id, nothing) +hostcall_target(id::UInt64) = get(Base.@atomic(hostcall_targets.table), id, nothing) + + +## invoking targets + +# The service path calls Julia's exported `jl_invoke`: resolve the handler's MethodInstance +# once per serviced call, then invoke every lane through Julia's own compiled fast path. +# The cache is tagged with the world counter because replacing a method does not invalidate +# the old MethodInstance: currency in the *latest* world — hostcalls behave like +# `invokelatest` — is a dispatch-level property that must be re-resolved when the world +# moves. + +# the handler call signature, after the argument conversion `call_arguments` applies +hostconvert_host_type(@nospecialize(P)) = P <: LLVMPtr ? CuPtr{P.parameters[1]} : P +function hostcall_handler_sig(target::HostcallTarget) + AT = target.AT + params = Any[hostconvert_host_type(P) for P in AT.parameters] + target.stored && pushfirst!(params, target.key.parameters[1]) + return Tuple{params...} +end + +# Resolve and cache the handler's MethodInstance for the current world. This returns +# `nothing` when dispatch fails, in which case the service path uses `invokelatest` to +# report the ordinary MethodError. Resolution runs in the latest world explicitly: the +# server thread's adopted task remains in the world in which it was adopted. +function seed_hostcall_method!(target::HostcallTarget) + world = Base.get_world_counter() + method = Base.invoke_in_world(world, resolve_hostcall_method, target, world) + Base.@atomic target.method = method + return method +end +function resolve_hostcall_method(target::HostcallTarget, world::UInt) + try + sig = hostcall_handler_sig(target) + precompile(sig) + ft = sig.parameters[1] + tt = Tuple{sig.parameters[2:end]...} + mi = methodinstance(ft, tt, world) + return (world, mi) + catch + return nothing + end +end + +# Call Julia's `jl_invoke`, which finds or compiles a CodeInstance for `mi` and invokes its +# boxed entry point. The handler is precompiled at registration, so the common path is the +# same atomic cache walk and indirect call Julia uses for an `invoke` expression. `args` +# excludes `f`, and the caller runs in the seed world (see `service_target!`). +@inline function invoke_hostcall_method(mi::Core.MethodInstance, @nospecialize(f), + args::Vector{Any}) + GC.@preserve args begin + ccall(:jl_invoke, Any, (Any, Ptr{Any}, UInt32, Any), + f, pointer(args), length(args), mi) + end +end # conversion of values received from the device (pointers already arrive as `CuPtr`, see # the device-side `hostconvert`) @@ -482,9 +556,32 @@ end # invoke a registered target for every live lane of a port function service_target!(p::HostPort, target::HostcallTarget, hdr::HostcallHeader) + # Resolve the handler's method once per call. A stale cache (the world moved since the + # last resolution, e.g. the handler was redefined) is re-seeded before any lane calls. + printing = target.stored && is_print_target(target.f) + method = nothing + if !printing + method = Base.@atomic target.method + if method === nothing || method[1] != Base.get_world_counter() + method = seed_hostcall_method!(target) + end + end + if method === nothing + service_lanes!(p, target, hdr, nothing) + else + # Switch to the seed world once for the whole call, so `jl_invoke` and calls inside + # the handler use the same latest-world snapshot. + Base.invoke_in_world(method[1], service_lanes!, p, target, hdr, method[2]) + end + return +end + +function service_lanes!(p::HostPort, target::HostcallTarget, hdr::HostcallHeader, + mi::Union{Nothing,Core.MethodInstance}) mask = hdr.mask RT = target.RT reply = (hdr.flags & HOSTCALL_FLAG_ASYNC) == 0 && RT !== Nothing + printing = target.stored && is_print_target(target.f) payloads = read_lanes(p, target.AT, mask) results = Vector{Any}(undef, 32) status = HOSTCALL_STATUS_OK @@ -494,9 +591,11 @@ function service_target!(p::HostPort, target::HostcallTarget, hdr::HostcallHeade f = target.stored ? target.f : getfield(payload, 1) args = call_arguments(payload, target.stored ? 0 : 1) try - if is_print_target(f) + if printing queue_hostcall_output(f, args...) rv = nothing + elseif mi !== nothing + rv = invoke_hostcall_method(mi, f, args) else rv = Base.invokelatest(f, args...) end @@ -724,6 +823,11 @@ function hostcall_server_main(srv::HostcallServer) end end +# NOTE: the adopted task's world age is fixed when the thread enters Julia and never +# advances (the server never returns to top level), so anything the server dispatches +# resolves in that world: later method definitions — handler redefinitions, but also +# Revise edits to the server code itself — are only visible through `invokelatest` / +# `invoke_in_world`, which is how handlers are resolved and invoked. function hostcall_server_entry(::Ptr{Cvoid}) srv = hostcall_server[]::HostcallServer hostcall_server_main(srv) diff --git a/docs/src/development/hostcall.md b/docs/src/development/hostcall.md index 2a03be3188..28de8a85ae 100644 --- a/docs/src/development/hostcall.md +++ b/docs/src/development/hostcall.md @@ -64,6 +64,13 @@ Two layers are available, both built on the same protocol. (julia#55525). The `print` family (`print`, `println`, `printstyled`, `show`, `display`) called directly as a hostcall target is special-cased: its output is queued and written at the next `synchronize()`, or earlier by a printer task when thread 1 is free. +- **Handlers run in the latest world**, like `invokelatest`: redefining a handler takes + effect immediately — without recompiling the kernel, and including for kernels that are + already running. Internally, calls are dispatched like Julia's `invoke`: the compiled + kernel identifies each target by a literal pointer to its rooted key type (cached images + carry a relocation that is re-resolved on load), and the service thread calls `jl_invoke` + with a cached `MethodInstance` that is re-resolved when the world moves, so the steady + state performs no generic dispatch. - **Errors**: an exception thrown by a handler, an unknown target, or a result that cannot be converted stops all lanes of that call on the device (like a device-side exception) and is rethrown as a [`HostcallException`](@ref CUDACore.HostcallException) at the next stream, event, diff --git a/test/core/hostcall.jl b/test/core/hostcall.jl index 650078adcc..0058221549 100644 --- a/test/core/hostcall.jl +++ b/test/core/hostcall.jl @@ -4,6 +4,16 @@ using CUDA: HostcallException # child launches, so it must stay compact @test sizeof(CUDACore.KernelState) == 2sizeof(UInt) +# Closed type arguments use TypeEgal dispatch keys on Julia 1.14 and later. +let K = Tuple{typeof(identity),Int,Tuple{Int}} + P = @static if isdefined(Core, :TypeEgal) + Core.TypeEgal{K} + else + Type{K} + end + @test CUDACore.hostcall_key_type(P) === K +end + hostcall_lookup(i::Int) = 10f0 * i hostcall_increment(i::Int) = i + 1 const hostcall_counter = Threads.Atomic{Int}(0) @@ -131,13 +141,46 @@ end synchronize() @test Array(out)[1] == 55 - # A hash collision must fail at registration instead of silently changing dispatch. + # Targets are identified by the address of their rooted key type, like Julia's invoke + # references its callee: distinct keys cannot collide, identifiers never fall in the + # builtin id space, and registration is idempotent. K1 = Tuple{typeof(identity),Int,Tuple{Int}} K2 = Tuple{typeof(abs),Int,Tuple{Int}} - id = CUDACore.hostcall_target_id_value(K1) - @test id & CUDACore.HOSTCALL_STATIC_ID_BIT != 0 - CUDACore.register_hostcall_targets!([id => K1]) - @test_throws ErrorException CUDACore.register_hostcall_targets!([id => K2]) + id1 = CUDACore.hostcall_target_word(K1) + id2 = CUDACore.hostcall_target_word(K2) + @test id1 != id2 + @test id1 >= CUDACore.HOSTCALL_BUILTIN_IDS + @test id1 == UInt64(CUDACore.GPUCompiler.resolve_relocation_target( + CUDACore.GPUCompiler.JuliaValueRef(K1))) + @test id1 == CUDACore.hostcall_target_word(K1) + CUDACore.register_hostcall_targets!([K1]) + CUDACore.register_hostcall_targets!([K1]) + target = CUDACore.hostcall_target(id1) + @test target !== nothing + @test target.key === K1 +end + +# handler and kernel for the redefinition testset below: they must be globals, both for the +# redefinition to be a plain method replacement (redefining a local boxes the binding, which +# the kernel would then capture) and so the kernel need not be recompiled +hostcall_redef(x::Int) = x + 1 +function hostcall_redef_kernel(out) + out[1] = @hostcall hostcall_redef(1)::Int + return +end + +@testset "handler redefinition" begin + # hostcalls behave like `invokelatest`: redefining the handler between launches takes + # effect without recompiling the kernel, re-seeding the service fast path + out = CUDA.zeros(Int, 1) + @cuda hostcall_redef_kernel(out) + synchronize() + @test Array(out)[1] == 2 + + @eval hostcall_redef(x::Int) = x + 41 + @cuda hostcall_redef_kernel(out) + synchronize() + @test Array(out)[1] == 42 end @testset "idle backoff" begin From 5319ac3c4117a34be37fc2ebb31d1fe77c6a3969 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Mon, 24 Aug 2026 16:46:48 +0200 Subject: [PATCH 10/10] Ensure hostcall ports cover resident warps A blocking warp must not wait for another resident warp to release a port, because GPUs do not guarantee forward progress. Treat hostcall_ports as a lower bound and raise the cap to LLVM libc RPC's 16K maximum. --- CUDACore/src/hostcall.jl | 16 +++++++++++----- docs/src/development/hostcall.md | 7 ++++--- test/core/hostcall.jl | 7 +++++++ 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/CUDACore/src/hostcall.jl b/CUDACore/src/hostcall.jl index 5e4d19e24c..458cdbf2e4 100644 --- a/CUDACore/src/hostcall.jl +++ b/CUDACore/src/hostcall.jl @@ -53,7 +53,7 @@ end const HOSTCALL_SWEEP_CHUNK = 64 const HOSTCALL_MIN_PORTS = 64 -const HOSTCALL_MAX_PORTS = 4096 +const HOSTCALL_MAX_PORTS = 16384 # also LLVM libc RPC's protocol maximum function HostcallArea(ctx::CuContext, nports::Integer, exception_info::ExceptionInfo; heartbeat::Bool=true) @@ -115,12 +115,18 @@ end # the default number of ports: enough for every resident warp, so that a warp never has to # wait for another warp to release a port (GPUs have no forward-progress guarantee) function hostcall_default_ports(dev::CuDevice) - if hostcall_ports_pref !== nothing - return Int(hostcall_ports_pref) - end sms = attribute(dev, DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT) warps = attribute(dev, DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR) ÷ 32 - return clamp(sms * warps, HOSTCALL_MIN_PORTS, HOSTCALL_MAX_PORTS) + ports = max(sms * warps, HOSTCALL_MIN_PORTS) + if hostcall_ports_pref !== nothing + preferred = Int(hostcall_ports_pref) + 1 <= preferred <= HOSTCALL_MAX_PORTS || + throw(ArgumentError("hostcall_ports must be between 1 and $HOSTCALL_MAX_PORTS")) + ports = max(ports, preferred) + end + ports <= HOSTCALL_MAX_PORTS || + error("device requires $ports hostcall ports, more than the supported maximum of $HOSTCALL_MAX_PORTS") + return ports end # all areas, as an immutable snapshot that the server thread can read without locking diff --git a/docs/src/development/hostcall.md b/docs/src/development/hostcall.md index 28de8a85ae..5c27724ddc 100644 --- a/docs/src/development/hostcall.md +++ b/docs/src/development/hostcall.md @@ -129,9 +129,10 @@ switching to the calling kernel's context for every call. Consequences: ## Configuration The `hostcall_ports` preference (set with `Preferences.set_preferences!(CUDACore, ...)` and -restart; the preference belongs to the `CUDACore` package, not `CUDA`) controls the number -of ports (warp-level call slots) per context; the default is the number of resident warps -of the device (~8 MiB of pinned memory on a large GPU). Contexts start with a small area +restart; the preference belongs to the `CUDACore` package, not `CUDA`) sets a lower bound +on the number of ports (warp-level call slots) per context. CUDA.jl always allocates at +least one per resident warp to avoid depending on GPU forward progress; the resulting +area uses about 8 MiB of pinned memory on a large GPU. Contexts start with a small area until a kernel that calls host functions is linked. ## Display watchdogs diff --git a/test/core/hostcall.jl b/test/core/hostcall.jl index 0058221549..91a4156384 100644 --- a/test/core/hostcall.jl +++ b/test/core/hostcall.jl @@ -14,6 +14,13 @@ let K = Tuple{typeof(identity),Int,Tuple{Int}} @test CUDACore.hostcall_key_type(P) === K end +# Blocking calls must not rely on forward progress from warps waiting for a port. +let dev = device() + sms = attribute(dev, CUDACore.DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT) + warps = attribute(dev, CUDACore.DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR) ÷ 32 + @test CUDACore.hostcall_default_ports(dev) >= sms * warps +end + hostcall_lookup(i::Int) = 10f0 * i hostcall_increment(i::Int) = i + 1 const hostcall_counter = Threads.Atomic{Int}(0)