use device libs from AMDGPU_LLVM_Backend_jll (2)

Project.toml

@@ -20,17 +20,18 @@ GPUCompiler = "61eb1bfa-7361-4325-ad38-22787b887f55"
 GPUToolbox = "096a3bc2-3ced-46d0-87f4-dd12716f4bfc"
 KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c"
 LLVM = "929cbde3-209d-540e-8aea-75f648917ca0"
+LLVMDowngrader_jll = "f52de702-fb25-5922-94ba-81dd59b07444"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
 Preferences = "21216c6a-2e73-6563-6e65-726566657250"
 PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
-ROCmDeviceLibs_jll = "873c0968-716b-5aa7-bb8d-d1e2e2aeff2d"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Random123 = "74087812-796a-5b5d-8853-05524746bad3"
 RandomNumbers = "e6cf234a-135c-5ec9-84dd-332b85af5143"
+Scratch = "6c6a2e73-6563-6170-7368-637461726353"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
@@ -64,12 +65,13 @@ GPUCompiler = "2.2.3"
 GPUToolbox = "3"
 KernelAbstractions = "0.9.2"
 LLVM = "9"
+LLVMDowngrader_jll = "0.9.1"
 PrecompileTools = "1"
 Preferences = "1"
 PrettyTables = "3"
-ROCmDeviceLibs_jll = "=5.6.1, =6.2.1, =7.0.2"
 Random123 = "1.6"
 RandomNumbers = "1.5"
+Scratch = "1"
 SparseMatricesCSR = "0.6.9"
 SpecialFunctions = "2"
 StaticArraysCore = "1"

src/discovery/discovery.jl

@@ -6,8 +6,9 @@ export librocrand, librocfft, libMIOpen_path
 export libhiptensor
 
 using AMDGPU_LLVM_Backend_jll
-using ROCmDeviceLibs_jll
+using LLVMDowngrader_jll
 using Preferences
+using Scratch
 using Libdl
 
 include("utils.jl")
@@ -25,12 +26,78 @@ function get_ld_lld(rocm_path::String)::Tuple{String, Bool}
     return (AMDGPU_LLVM_Backend_jll.lld_path, true)
 end
 
+# bitcode versions `llvm-downgrade` can target.
+# The 15 target emits opaque pointers, but GPUCompiler uses typed pointers on LLVM 15 and 16
+# (Julia 1.10 and 1.11), so both use the 14 target instead.
+const DOWNGRADE_TARGETS = (v"14", #=v"15",=# v"18")
+
+# downgrade the device libs to the latest LLVM version Julia supports
+function downgrade_device_libs(src_dir::String)::String
+    target = maximum(Iterators.filter(<=(Base.libllvm_version), DOWNGRADE_TARGETS))
+    # ensure this is rebuilt if any of the relevant jlls or the target changes
+    scratch_name = replace(string(
+        "device_libs-", pkgversion(AMDGPU_LLVM_Backend_jll),
+        "-", first(basename(AMDGPU_LLVM_Backend_jll.artifact_dir), 8),
+        "-downgrader-", pkgversion(LLVMDowngrader_jll),
+        "-", first(basename(LLVMDowngrader_jll.artifact_dir), 8),
+        "-llvm-", target.major), "+" => "_") # artifact versions include +, which Scratch does not like
+    dir = @get_scratch!(scratch_name)
+    marker = joinpath(dir, "downgrade_complete")
+    isfile(marker) && return dir
+
+    # Use a temp dir, so concurrent processes don't interfere
+    mktempdir(dirname(dir)) do tmp
+        for file in readdir(src_dir)
+            endswith(file, ".bc") || continue
+            # Just skip libraries the downgrader can't handle. `link_device_libs!` will throw an error if it is actually needed
+            cmd = `$(LLVMDowngrader_jll.llvm_downgrade()) --bitcode-version=$(target.major).$(target.minor) -o $(joinpath(tmp, file)) $(joinpath(src_dir, file))`
+            err_io = IOBuffer()
+            try
+                run(pipeline(cmd; stderr=err_io))
+            catch
+                @warn """Failed to downgrade device library `$file` to LLVM $(target.major), skipping it.
+                $(rstrip(String(take!(err_io))))
+                """
+                rm(joinpath(tmp, file); force=true)
+            end
+        end
+        for file in readdir(tmp)
+            mv(joinpath(tmp, file), joinpath(dir, file); force=true)
+        end
+    end
+    touch(marker)
+    return dir
+end
+
 function get_device_libs(from_artifact::Bool; rocm_path::String)
-    if from_artifact && ROCmDeviceLibs_jll.is_available()
-        ROCmDeviceLibs_jll.bitcode_path
-    else
-        find_device_libs(rocm_path)
+    artifact_err = nothing
+    if from_artifact &&
+        AMDGPU_LLVM_Backend_jll.is_available() &&
+        isdefined(AMDGPU_LLVM_Backend_jll, :bitcode_path) &&
+        LLVMDowngrader_jll.is_available()
+
+        try
+            return downgrade_device_libs(AMDGPU_LLVM_Backend_jll.bitcode_path)
+        catch err
+            artifact_err = (err, catch_backtrace())
+        end
+    end
+
+    device_libs = find_device_libs(rocm_path)
+    if !isnothing(artifact_err)
+        if isempty(device_libs)
+            @warn """Failed to downgrade artifact device libraries and no system-wide \
+            device libraries found in `$rocm_path`.
+            Ensure `JULIA_DEPOT_PATH` is writeable, or point `ROCM_PATH` to a local ROCm
+            installation.
+            """ exception=artifact_err
+        else
+            @warn """Failed to downgrade artifact device libraries, \
+            falling back to system-wide device libraries in `$device_libs`.
+            """ exception=artifact_err
+        end
     end
+    return device_libs
 end
 
 function _hip_runtime_version()