use device libs from AMDGPU_LLVM_Backend_jll

Project.toml

@@ -20,17 +20,18 @@ GPUCompiler = "61eb1bfa-7361-4325-ad38-22787b887f55"
 GPUToolbox = "096a3bc2-3ced-46d0-87f4-dd12716f4bfc"
 KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c"
 LLVM = "929cbde3-209d-540e-8aea-75f648917ca0"
+LLVMDowngrader_jll = "f52de702-fb25-5922-94ba-81dd59b07444"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
 Preferences = "21216c6a-2e73-6563-6e65-726566657250"
 PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
-ROCmDeviceLibs_jll = "873c0968-716b-5aa7-bb8d-d1e2e2aeff2d"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Random123 = "74087812-796a-5b5d-8853-05524746bad3"
 RandomNumbers = "e6cf234a-135c-5ec9-84dd-332b85af5143"
+Scratch = "6c6a2e73-6563-6170-7368-637461726353"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
@@ -64,12 +65,13 @@ GPUCompiler = "2"
 GPUToolbox = "0.1.0, 0.2, 0.3, 1, 2, 3"
 KernelAbstractions = "0.9.2"
 LLVM = "9"
+LLVMDowngrader_jll = "0.9"
 PrecompileTools = "1"
 Preferences = "1"
 PrettyTables = "3"
-ROCmDeviceLibs_jll = "=5.6.1, =6.2.1, =7.0.2"
 Random123 = "1.6"
 RandomNumbers = "1.5"
+Scratch = "1"
 SparseMatricesCSR = "0.6.9"
 SpecialFunctions = "2"
 StaticArraysCore = "1"

src/discovery/discovery.jl

@@ -5,8 +5,9 @@ export librocblas, librocsparse, librocsolver
 export librocrand, librocfft, libMIOpen_path
 
 using AMDGPU_LLVM_Backend_jll
-using ROCmDeviceLibs_jll
+using LLVMDowngrader_jll
 using Preferences
+using Scratch
 using Libdl
 
 include("utils.jl")
@@ -24,12 +25,52 @@ function get_ld_lld(rocm_path::String)::Tuple{String, Bool}
     return (AMDGPU_LLVM_Backend_jll.lld_path, true)
 end
 
+# bitcode versions `llvm-downgrade` can target.
+const DOWNGRADE_TARGETS = (v"5", v"7", v"14", #=v"15",=# v"18")
+
+# downgrade the device libs to the latest LLVM version Julia supports
+function downgrade_device_libs(src_dir::String)::String
+    target = maximum(Iterators.filter(<=(Base.libllvm_version), DOWNGRADE_TARGETS))
+    # ensure this is rebuilt if any of the relevant jlls or the target changes
+    scratch_name = replace(string(
+        "device_libs-", pkgversion(AMDGPU_LLVM_Backend_jll),
+        "-", first(basename(AMDGPU_LLVM_Backend_jll.artifact_dir), 8),
+        "-downgrader-", pkgversion(LLVMDowngrader_jll),
+        "-", first(basename(LLVMDowngrader_jll.artifact_dir), 8),
+        "-llvm-", target.major), "+" => "_") # artifact versions include +, which Scratch does not like
+    dir = @get_scratch!(scratch_name)
+    marker = joinpath(dir, "downgrade_complete")
+    isfile(marker) && return dir
+
+    # Use a temp dir, so concurrent processes don't interfere
+    mktempdir(dirname(dir)) do tmp
+        for file in readdir(src_dir)
+            endswith(file, ".bc") || continue
+            run(`$(LLVMDowngrader_jll.llvm_downgrade()) --bitcode-version=$(target.major).$(target.minor) -o $(joinpath(tmp, file)) $(joinpath(src_dir, file))`)
+        end
+        for file in readdir(tmp)
+            mv(joinpath(tmp, file), joinpath(dir, file); force=true)
+        end
+    end
+    touch(marker)
+    return dir
+end
+
 function get_device_libs(from_artifact::Bool; rocm_path::String)
-    if from_artifact && ROCmDeviceLibs_jll.is_available()
-        ROCmDeviceLibs_jll.bitcode_path
-    else
-        find_device_libs(rocm_path)
+    if from_artifact &&
+        AMDGPU_LLVM_Backend_jll.is_available() &&
+        isdefined(AMDGPU_LLVM_Backend_jll, :bitcode_path) &&
+        LLVMDowngrader_jll.is_available()
+
+        try
+            return downgrade_device_libs(AMDGPU_LLVM_Backend_jll.bitcode_path)
+        catch err
+            @warn """Failed to downgrade artifact device libraries, \
+            falling back to system-wide device libraries.
+            """ exception=(err, catch_backtrace())
+        end
     end
+    return find_device_libs(rocm_path)
 end
 
 function _hip_runtime_version()