refactor discovery, ship therock tarballs as artifacts (2)

.buildkite/pipeline.yml

@@ -1,3 +1,10 @@
+# AMDGPU.jl chooses between the ROCm_Runtime artifacts and a local ROCm through
+# ROCm_Runtime's "local" preference. Every GPU step writes that preference and
+# then asserts the provider it actually resolved to, so that a change of default
+# cannot silently swap the ROCm underneath a job. The artifacts are lazy, so
+# Pkg.instantiate/update download nothing; the pin only has to be in place
+# before AMDGPU is first loaded, which also covers the steps where the
+# julia-test plugin instantiates before the command below runs.
 steps:
   - label: "Documentation"
     plugins:
@@ -14,11 +21,12 @@ steps:
       julia --project -e '
         println("--- :julia: Instantiating project")
         using Pkg
+        Pkg.develop([PackageSpec(path="ROCm_Runtime"), PackageSpec(path="ROCm_Runtime_Discovery")])
         Pkg.instantiate()
         Pkg.build()
 
         Pkg.activate("docs")
-        Pkg.develop(PackageSpec(path="."))
+        Pkg.develop([PackageSpec(path="."), PackageSpec(path="ROCm_Runtime"), PackageSpec(path="ROCm_Runtime_Discovery")])
         Pkg.instantiate()
 
         push!(LOAD_PATH, @__DIR__)
@@ -31,11 +39,38 @@ steps:
     if: build.message !~ /\[skip docs\]/
     timeout_in_minutes: 20
 
-  - label: "Julia {{matrix.version}}"
+  # Julia 1.10 doesn't support [sources] in Project.toml, so the workspace
+  # packages have to be dev'ed manually before instantiating -- which rules out
+  # the julia-test plugin, whose pre-command hook instantiates the project
+  # before the step's command gets a chance to run.
+  - label: "Julia 1.10 (ROCm artifacts)"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.10"
+      - JuliaCI/julia-coverage#v1:
+          codecov: true
+    agents:
+      queue: "rocm"
+      rocmgpu: "*"
+    if: build.message !~ /\[skip tests\]/
+    command: |
+      printf '[ROCm_Runtime]\nlocal = false\n' > LocalPreferences.toml
+      julia --project -e '
+        using Pkg
+        Pkg.develop([PackageSpec(path="ROCm_Runtime"), PackageSpec(path="ROCm_Runtime_Discovery")])
+        Pkg.update()'
+      julia --project -e 'using AMDGPU; AMDGPU.local_rocm && error("this step selected the ROCm artifacts, but AMDGPU resolved to a local ROCm")'
+      julia --project -e 'using Pkg; Pkg.test(; coverage=true, test_args=["--verbose"])'
+    timeout_in_minutes: 90
+    env:
+      JULIA_NUM_THREADS: 4
+      JULIA_AMDGPU_CORE_MUST_LOAD: "1"
+      JULIA_AMDGPU_HIP_MUST_LOAD: "1"
+
+  - label: "Julia {{matrix.version}} (ROCm artifacts)"
     matrix:
       setup:
         version:
-          - "1.10"
           - "1.11"
           - "1.12"
           - "1.13"
@@ -61,45 +96,95 @@ steps:
       queue: "rocm"
       rocmgpu: "*"
     if: build.message !~ /\[skip tests\]/
-    command: "julia --project -e 'using Pkg; Pkg.update()'"
+    command: |
+      printf '[ROCm_Runtime]\nlocal = false\n' > LocalPreferences.toml
+      julia --project -e 'using Pkg; Pkg.update()'
+      julia --project -e 'using AMDGPU; AMDGPU.local_rocm && error("this step selected the ROCm artifacts, but AMDGPU resolved to a local ROCm")'
     timeout_in_minutes: 90
     env:
       JULIA_NUM_THREADS: 4
       JULIA_AMDGPU_CORE_MUST_LOAD: "1"
       JULIA_AMDGPU_HIP_MUST_LOAD: "1"
-      JULIA_AMDGPU_DISABLE_ARTIFACTS: "1"
 
-  - label: "Julia {{matrix.version}} Enzyme"
-    matrix:
-      setup:
-        version:
-          - "1.10"
-          - "1.11"
+  # Coverage for the local-ROCm provider, also exercised by the CSCS
+  # MI300 pipeline. Does not use the julia-test plugin, so the provider is
+  # pinned before AMDGPU is first loaded and no artifact is fetched.
+  - label: "Julia 1.12 (local ROCm)"
     plugins:
       - JuliaCI/julia#v1:
-          version: "{{matrix.version}}"
+          version: "1.12"
+      - JuliaCI/julia-coverage#v1:
+          codecov: true
+    agents:
+      queue: "rocm"
+      rocmgpu: "*"
+    if: build.message !~ /\[skip tests\]/
+    command: |
+      printf '[ROCm_Runtime]\nlocal = true\n' > LocalPreferences.toml
+      julia --project -e 'using Pkg; Pkg.update()'
+      julia --project -e 'using AMDGPU; AMDGPU.local_rocm || error("this step selected a local ROCm, but AMDGPU resolved to the ROCm artifacts")'
+      julia --project -e 'using Pkg; Pkg.test(; coverage=true, test_args=["--verbose"])'
+    timeout_in_minutes: 90
+    env:
+      JULIA_NUM_THREADS: 4
+      JULIA_AMDGPU_CORE_MUST_LOAD: "1"
+      JULIA_AMDGPU_HIP_MUST_LOAD: "1"
+
+  # see the "Julia 1.10 (ROCm artifacts)" step for why this doesn't use the plugin
+  - label: "Julia 1.10 Enzyme (ROCm artifacts)"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.10"
+    agents:
+      queue: "rocm"
+      rocmgpu: "*"
+    if: build.message !~ /\[skip tests\]/
+    command: |
+      printf '[ROCm_Runtime]\nlocal = false\n' > LocalPreferences.toml
+      julia --project -e '
+        using Pkg
+        Pkg.develop([PackageSpec(path="ROCm_Runtime"), PackageSpec(path="ROCm_Runtime_Discovery")])
+        Pkg.update()'
+      julia --project -e 'using AMDGPU; AMDGPU.local_rocm && error("this step selected the ROCm artifacts, but AMDGPU resolved to a local ROCm")'
+      julia --project -e 'using Pkg; Pkg.test(; test_args=["enzyme"])'
+    timeout_in_minutes: 45
+    soft_fail: true
+    env:
+      JULIA_NUM_THREADS: 4
+      JULIA_AMDGPU_CORE_MUST_LOAD: "1"
+      JULIA_AMDGPU_HIP_MUST_LOAD: "1"
+
+  - label: "Julia 1.11 Enzyme (ROCm artifacts)"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.11"
       - JuliaCI/julia-test#v1:
           test_args: "enzyme"
     agents:
       queue: "rocm"
       rocmgpu: "*"
     if: build.message !~ /\[skip tests\]/
-    command: "julia --project -e 'using Pkg; Pkg.update()'"
+    command: |
+      printf '[ROCm_Runtime]\nlocal = false\n' > LocalPreferences.toml
+      julia --project -e 'using Pkg; Pkg.update()'
+      julia --project -e 'using AMDGPU; AMDGPU.local_rocm && error("this step selected the ROCm artifacts, but AMDGPU resolved to a local ROCm")'
     timeout_in_minutes: 45
     soft_fail: true
     env:
       JULIA_NUM_THREADS: 4
       JULIA_AMDGPU_CORE_MUST_LOAD: "1"
       JULIA_AMDGPU_HIP_MUST_LOAD: "1"
-      JULIA_AMDGPU_DISABLE_ARTIFACTS: "1"
 
+  # see the "Julia 1.10 (ROCm artifacts)" step for why this doesn't use the plugin
   - label: "GPU-less environment"
     plugins:
       - JuliaCI/julia#v1:
           version: "1.10"
-      - JuliaCI/julia-test#v1:
-          run_tests: false
     command: |
+      julia --project -e '
+        using Pkg
+        Pkg.develop([PackageSpec(path="ROCm_Runtime"), PackageSpec(path="ROCm_Runtime_Discovery")])
+        Pkg.instantiate()'
       julia --project -e '
         using AMDGPU
         @assert !AMDGPU.functional()'
@@ -113,12 +198,14 @@ steps:
       - JuliaCI/julia#v1:
           version: "1.12"
     command: |
+      printf '[ROCm_Runtime]\nlocal = false\n' > perf/LocalPreferences.toml
       julia --project=perf -e '
         println("--- :julia: Instantiating benchmark project")
         using Pkg
-        Pkg.develop(PackageSpec(path="."))
-        Pkg.instantiate()
-
+        Pkg.develop([PackageSpec(path="."), PackageSpec(path="ROCm_Runtime"), PackageSpec(path="ROCm_Runtime_Discovery")])
+        Pkg.instantiate()'
+      julia --project=perf -e 'using AMDGPU; AMDGPU.local_rocm && error("this step selected the ROCm artifacts, but AMDGPU resolved to a local ROCm")'
+      julia --project=perf -e '
         println("+++ :julia: Running benchmarks")
         include("perf/runbenchmarks.jl")'
     artifact_paths:
@@ -131,7 +218,6 @@ steps:
     env:
       JULIA_AMDGPU_CORE_MUST_LOAD: "1"
       JULIA_AMDGPU_HIP_MUST_LOAD: "1"
-      JULIA_AMDGPU_DISABLE_ARTIFACTS: "1"
 
 env:
   JULIA_AMDGPU_LOGGING_ENABLED: true

.github/workflows/rocm-artifact-hashes.yml

@@ -0,0 +1,53 @@
+name: ROCm artifact hashes
+
+on:
+  workflow_dispatch:
+
+jobs:
+  hash-rocm-artifacts:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: "1.12"
+      - name: Download tarballs and compute hashes
+        shell: julia {0}
+        run: |
+          import Pkg
+          Pkg.add("ArtifactUtils")
+          using ArtifactUtils, Base.BinaryPlatforms
+          const rocm_version = "7.14.0"
+          const rocm_tag = "7.14"
+
+          for rocm_arch in [
+            "gfx908",
+            "gfx90a",
+            "gfx94X-dcgpu",
+            "gfx950-dcgpu",
+            "gfx101X-dgpu",
+            "gfx103X-all",
+            "gfx110X-all",
+            "gfx1150",
+            "gfx1151",
+            "gfx1152",
+            "gfx1153",
+            "gfx120X-all",
+          ]
+            @info "Linux, $rocm_arch"
+            platform = Platform("x86_64", "linux", Dict(
+                "libc" => "glibc", "rocm" => rocm_tag,
+                "rocm_arch" => replace(rocm_arch, "-" => "_")))
+            add_artifact!("Artifacts.toml", "ROCm_Runtime", "https://repo.amd.com/rocm/tarball-multi-arch/therock-dist-linux-$rocm_arch-$rocm_version.tar.gz"; lazy = true, platform)
+
+            startswith(rocm_arch, "gfx9") && continue
+            @info "Windows, $rocm_arch"
+            platform = Platform("x86_64", "windows", Dict(
+                "rocm" => rocm_tag,
+                "rocm_arch" => replace(rocm_arch, "-" => "_")))
+            add_artifact!("Artifacts.toml", "ROCm_Runtime", "https://repo.amd.com/rocm/tarball-multi-arch/therock-dist-windows-$rocm_arch-$rocm_version.tar.gz"; lazy = true, platform)
+          end
+      - name: Upload hash report
+        uses: actions/upload-artifact@v4
+        with:
+          name: Artifacts.toml
+          path: Artifacts.toml

Project.toml

@@ -4,7 +4,7 @@ version = "2.8.0"
 authors = ["Julian P Samaroo <jpsamaroo@jpsamaroo.me>", "Valentin Churavy <v.churavy@gmail.com>", "Anton Smirnov <tonysmn97@gmail.com>"]
 
 [workspace]
-projects = ["test", "docs", "perf"]
+projects = ["test", "docs", "perf", "ROCm_Runtime", "ROCm_Runtime_Discovery"]
 
 [deps]
 AMDGPU_LLVM_Backend_jll = "cc5c0156-bd05-5a77-8a68-bb0aafb29019"
@@ -28,6 +28,8 @@ Preferences = "21216c6a-2e73-6563-6e65-726566657250"
 PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 ROCmDeviceLibs_jll = "873c0968-716b-5aa7-bb8d-d1e2e2aeff2d"
+ROCm_Runtime = "3129f4d2-de71-4ff3-9833-76037e3ea355"
+ROCm_Runtime_Discovery = "0bf8d700-1fac-4a41-93ae-0195dedb3a15"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Random123 = "74087812-796a-5b5d-8853-05524746bad3"
 RandomNumbers = "e6cf234a-135c-5ec9-84dd-332b85af5143"
@@ -36,6 +38,10 @@ StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 UnsafeAtomics = "013be700-e6cd-48c3-b4a1-df204f14c38f"
 
+[sources]
+ROCm_Runtime = {path = "ROCm_Runtime"}
+ROCm_Runtime_Discovery = {path = "ROCm_Runtime_Discovery"}
+
 [weakdeps]
 ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
 EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869"
@@ -68,6 +74,8 @@ PrecompileTools = "1"
 Preferences = "1"
 PrettyTables = "3"
 ROCmDeviceLibs_jll = "=5.6.1, =6.2.1, =7.0.2"
+ROCm_Runtime = "7.14"
+ROCm_Runtime_Discovery = "0.1"
 Random123 = "1.6"
 RandomNumbers = "1.5"
 SparseMatricesCSR = "0.6.9"