From 0ed6818a8db9d0d5a683d0d6cf05426df3a9805e Mon Sep 17 00:00:00 2001 From: Carroll Vance Date: Sun, 4 Oct 2026 19:42:35 +0000 Subject: [PATCH 1/3] Bazel image: build on rules_julia_depot's image rules, rules_oci 2.3.3, push :latest rules_julia_depot now provides the Julia-specific image layers as rules (julia/image.bzl), so //deploy stops carrying its own copies: - julia_image_env replaces the hand-built JULIA_DEPOT_PATH. It declares the image layout once, and the compiled layer and the precompile test read the same layout, so the caches are built for and checked against the environment the image runs with. The workspace root is still an extra depot, which keeps the workspace's own packages relocatable. - julia_dist_layer replaces the cp -aL genrule. It keeps the distribution's relative symlinks, so its libraries ship once instead of twice. - julia_depot_layer and julia_compiled_layer replace the genrules around image_depot.sh and the precompile loop. - julia_precompile_test (//deploy:image_precompile_test) checks the layers without a container, so it needs no podman and can gate a push in CI. image_check stays for what only a container shows: tini, curl and the entrypoints. The CUDA driver stub is now a build-time layer. It goes to the compiled layer and the precompile test, with LD_LIBRARY_PATH={root}/build/cuda-stub, and never to oci_image. That uses the {root} placeholder added to rules_julia_depot in f8d7c22; before it, the rules had no way to name a file in the tree they unpack. Image behaviour changes: - JULIA_PKG_OFFLINE=true and JULIA_CPU_TARGET are set at run time. - PATH is /opt/julia/bin:$PATH, expanded against the base, instead of a copy of the base's PATH. - Layers are ordered least-changed first, so a code change only moves the last ones. - Our own tars normalise permissions the same way the module's do. rules_oci 2.3.3 comes from the release archive because the Bazel Central Registry stops at 2.3.0. The fix that matters is in 2.3.1 (#881): oci_load reported success even when the tar it streams into the loader failed. 2.3.x needs a newer aspect_bazel_lib, so our Bazel 9 workaround is gone, and bsdtar now comes from tar.bzl. rules_shell is 0.8.0. rules_julia_depot dropped //oci:podman.sh, so the loader is now deploy/podman.sh. .github/workflows/image.yml is triggered by hand. It runs manifest_current and image_precompile_test, then pushes :latest to the repository in the DOCKERHUB_REPOSITORY variable. Releases are not named yet, so each run's summary records the digest and the commit. .bazelignore gains the export tests' .CondaPkg env, whose BUILD files broke //.... Co-Authored-By: Claude Opus 5.5 (1M context) --- .bazelignore | 2 + .github/workflows/image.yml | 72 ++++++++ BUILD.bazel | 3 + CONTRIBUTING.md | 8 +- MODULE.bazel | 38 ++-- MODULE.bazel.lock | 241 ++----------------------- deploy/BUILD.bazel | 339 +++++++++++++++++++----------------- deploy/README.md | 38 +++- deploy/podman.sh | 5 + 9 files changed, 334 insertions(+), 412 deletions(-) create mode 100644 .github/workflows/image.yml create mode 100755 deploy/podman.sh diff --git a/.bazelignore b/.bazelignore index f5510a30..58b789aa 100644 --- a/.bazelignore +++ b/.bazelignore @@ -4,3 +4,5 @@ docs/build lib private examples +# The Python env CondaPkg creates for the export tests; it ships BUILD files of its own. +packages/ReactantServerExport/test/.CondaPkg diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml new file mode 100644 index 00000000..7dbabecb --- /dev/null +++ b/.github/workflows/image.yml @@ -0,0 +1,72 @@ +# Build the locked CUDA node image with Bazel (//deploy:image) and push it to Docker Hub as :latest. +# Triggered by hand only: Actions > Image > Run workflow, or `gh workflow run image.yml --ref `. +# +# Releases are not named yet, so every run overwrites :latest. The image digest is printed in the +# run summary; pull by that digest to pin a deployment. +# +# Needs, in the repository settings: +# variable DOCKERHUB_REPOSITORY the target, e.g. index.docker.io// +# secret DOCKERHUB_USERNAME +# secret DOCKERHUB_TOKEN a Docker Hub access token with write access to that repository +name: Image + +on: + workflow_dispatch: + +# Two runs pushing :latest at once would race; the later one waits instead. +concurrency: + group: image-latest + cancel-in-progress: false + +permissions: + contents: read + +jobs: + image: + runs-on: ubuntu-latest + timeout-minutes: 240 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + # The CUDA base, the depot (Reactant's CUDA artifact included) and the layers built from them + # need tens of gigabytes. The runner's root disk is too small for that once its preinstalled + # toolchains are counted, so those are removed and Bazel keeps everything on /mnt. + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + sudo docker image prune --all --force + sudo mkdir -p /mnt/bazel + sudo chown "$(id -u):$(id -g)" /mnt/bazel + printf '%s\n' \ + 'startup --output_user_root=/mnt/bazel/root' \ + 'common --repository_cache=/mnt/bazel/repository-cache' \ + > user.bazelrc + df -h / /mnt + + # Bazelisk is preinstalled as `bazel` and runs the version in .bazelversion. + - name: Check the Julia lock + run: bazel test //deploy:manifest_current + + # Builds every layer, then loads every entry project from them with nothing precompiled and + # no cache rejected. A failure here stops the run before anything is pushed. + - name: Build the image and check it starts warm + run: bazel test //deploy:image_precompile_test + + - name: Log in to Docker Hub + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Push :latest + env: + REPOSITORY: ${{ vars.DOCKERHUB_REPOSITORY }} + run: | + : "${REPOSITORY:?set the DOCKERHUB_REPOSITORY repository variable}" + bazel run //deploy:image_push -- --repository "$REPOSITORY" + digest="$(jq -r '.manifests[0].digest' bazel-bin/deploy/image/index.json)" + { + echo "Pushed \`$REPOSITORY:latest\`" + echo + echo "Digest: \`$digest\` (commit \`$GITHUB_SHA\`)" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/BUILD.bazel b/BUILD.bazel index f6b37617..1460a5a6 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -3,6 +3,9 @@ package(default_visibility = ["//deploy:__pkg__"]) +# The root project, which //deploy:depot_layer instantiates from the committed deploy/Manifest.toml. +exports_files(["Project.toml"]) + # ReactantServerExport is offline tooling and not a workspace member (see Project.toml); the image # does not carry it. _NOT_IN_IMAGE = ["packages/ReactantServerExport/**"] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ee927b7b..74ec7b13 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -139,14 +139,18 @@ bazel test //deploy:manifest_current The Ubuntu packages the image adds to the CUDA base (`curl`, `tini`) are locked the same way in `deploy/debs.lock.json`; moving the base image digest in `MODULE.bazel` calls for -`bazel run //deploy:relock_debs`. After any change to the image, load it and check that it starts -without precompiling: +`bazel run //deploy:relock_debs`. After any change to the image, check that it starts without +precompiling, on its layers and then in a loaded container: ``` +bazel test //deploy:image_precompile_test bazel run //deploy:image_load bazel run //deploy:image_check ``` +The image is published to Docker Hub as `:latest` by the manually triggered +`.github/workflows/image.yml`; see [`deploy/README.md`](deploy/README.md#publishing). + ## Kaimon gate tools (dev only) `packages/ReactantServer/ext/ReactantServerKaimonGateExt.jl` registers four `rserver_*` GateTools with a running diff --git a/MODULE.bazel b/MODULE.bazel index 18b67312..1321e8c8 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -10,21 +10,32 @@ module( # build, so the same commit always yields the same package and artifact set. # Bazel 9 moved sh_binary/sh_test out of the core into rules_shell. -bazel_dep(name = "rules_shell", version = "0.7.1") +bazel_dep(name = "rules_shell", version = "0.8.0") # rules_oci assembles OCI images with no daemon; podman is needed only to load and run the result. -bazel_dep(name = "rules_oci", version = "2.2.6") +# +# 2.3.3, the latest release, which has not reached the Bazel Central Registry (2.3.0 is the newest +# there), so it is fetched from the release archive. The fix that matters here is oci_load's: before +# 2.3.1 (#881) it reported success when the tar it streams into the loader failed, so a broken +# load of this multi-gigabyte image looked like a good one. 2.3.1 also makes oci_pull repositories +# resolve their own dependencies under bzlmod (#891). Since 2.3.0 it also depends on a +# current aspect_bazel_lib, so the Bazel 9 workaround of naming aspect_bazel_lib here is gone. +# Drop the override once the registry has 2.3.3 or later. +bazel_dep(name = "rules_oci", version = "2.3.3") +archive_override( + module_name = "rules_oci", + integrity = "sha256-JD6sh+xB1o0dnm0hiEpQ7CjgMfcCsLn83H+Xw/PVOJo=", + strip_prefix = "rules_oci-2.3.3", + urls = ["https://github.com/bazel-contrib/rules_oci/releases/download/v2.3.3/rules_oci-v2.3.3.tar.gz"], +) -# rules_oci 2.2.6 pins aspect_bazel_lib 2.7.2, which registers its tar and zstd toolchains -# against @local_config_platform. Bazel 9 removed that repo, so toolchain resolution fails for -# every target, including ones unrelated to images ("invalid registered toolchain -# '@zstd_toolchains//:all'"). MVS takes the higher version, so naming it here is the whole fix. -bazel_dep(name = "aspect_bazel_lib", version = "2.22.5") +# bsdtar, from the toolchain rules_oci uses (tar.bzl since rules_oci 2.3), reads .deb files (ar +# archives holding zstd or xz tarballs) and the pulled base image's layers, so //deploy needs +# neither dpkg nor ar on the host. +bazel_dep(name = "tar.bzl", version = "0.10.9") -# bsdtar, from the same toolchain rules_oci uses, reads .deb files (ar archives holding zstd or xz -# tarballs) and the pulled base image's layers, so //deploy needs neither dpkg nor ar on the host. -bazel_lib_toolchains = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "toolchains") -use_repo(bazel_lib_toolchains, "bsd_tar_toolchains") +tar = use_extension("@tar.bzl//tar:extensions.bzl", "toolchains") +use_repo(tar, "bsd_tar_toolchains") # The Ubuntu packages the image adds to the CUDA base (curl, tini and their dependencies), each # pinned by snapshot URL and sha256 in deploy/debs.lock.json. Re-resolve with @@ -36,11 +47,12 @@ debs.lock( ) use_repo(debs, "reactantserver_debs") -# The Julia toolchain, the clean-depot image layer and the podman loader. +# The Julia toolchain, and the image rules (julia/image.bzl): the distribution, depot and +# precompile-cache layers, the image environment, and the precompile test. bazel_dep(name = "rules_julia_depot", version = "0.1.0") git_override( module_name = "rules_julia_depot", - commit = "49bc678d037fa967920edccc380a6721a20050ee", + commit = "f8d7c226136e7f02146d80088c02cb59a97096ad", remote = "https://github.com/csvance/rules_julia_depot.git", ) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 8d0e0e73..1ab8a82b 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -22,9 +22,8 @@ "https://bcr.bazel.build/modules/apple_support/1.24.2/source.json": "2c22c9827093250406c5568da6c54e6fdf0ef06238def3d99c71b12feb057a8d", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/MODULE.bazel": "253d739ba126f62a5767d832765b12b59e9f8d2bc88cc1572f4a73e46eb298ca", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.22.5/MODULE.bazel": "004ba890363d05372a97248c37205ae64b6fa31047629cd2c0895a9d0c7779e8", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.22.5/source.json": "ac2c3213df8f985785f1d0aeb7f0f73d5324e6e67d593d9b9470fb74a25d4a9b", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.21.1/MODULE.bazel": "07e3ce3eaaa50dbd0be7fa0094e36890478937adc780ec53e77fd9fe543af8b1", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.21.1/source.json": "cb7d22ce044efa47c6e251107a35b8a919f5cd35254190d825adff1b7ae21e6e", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "812d2dd42f65dca362152101fbec418029cc8fd34cbad1a2fde905383d705838", "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", @@ -40,6 +39,7 @@ "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", + "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", "https://bcr.bazel.build/modules/bazel_features/1.42.1/MODULE.bazel": "275a59b5406ff18c01739860aa70ad7ccb3cfb474579411decca11c93b951080", "https://bcr.bazel.build/modules/bazel_features/1.42.1/source.json": "fcd4396b2df85f64f2b3bb436ad870793ecf39180f1d796f913cc9276d355309", @@ -64,7 +64,8 @@ "https://bcr.bazel.build/modules/buildozer/8.5.1/MODULE.bazel": "a35d9561b3fc5b18797c330793e99e3b834a473d5fbd3d7d7634aafc9bdb6f8f", "https://bcr.bazel.build/modules/buildozer/8.5.1/source.json": "e3386e6ff4529f2442800dee47ad28d3e6487f36a1f75ae39ae56c70f0cd2fbd", "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/MODULE.bazel": "cdf8cbe5ee750db04b78878c9633cc76e80dcf4416cbe982ac3a9222f80713c8", - "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/source.json": "fa7b512dfcb5eafd90ce3959cf42a2a6fe96144ebbb4b3b3928054895f2afac2", + "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.3/MODULE.bazel": "f1b7bb2dd53e8f2ef984b39485ec8a44e9076dda5c4b8efd2fb4c6a6e856a31d", + "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.3/source.json": "ebe931bfe362e4b41e59ee00a528db6074157ff2ced92eb9e970acab2e1089c9", "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", @@ -81,7 +82,8 @@ "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/source.json": "f448c6e8963fdfa7eb831457df83ad63d3d6355018f6574fb017e8169deb43a9", "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", - "https://bcr.bazel.build/modules/package_metadata/0.0.2/source.json": "e53a759a72488d2c0576f57491ef2da0cf4aab05ac0997314012495935531b73", + "https://bcr.bazel.build/modules/package_metadata/0.0.6/MODULE.bazel": "341dab6f417197494517d54c8e557c0baee1de7aec83543a4fbefe57900acb7e", + "https://bcr.bazel.build/modules/package_metadata/0.0.6/source.json": "9581d8b22db43550ac75ecc314ee4fa0a33400bfdc77d1317d8af6b18dca7756", "https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5", "https://bcr.bazel.build/modules/platforms/0.0.11/MODULE.bazel": "0daefc49732e227caa8bfa834d65dc52e8cc18a2faf80df25e8caea151a9413f", "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", @@ -130,6 +132,7 @@ "https://bcr.bazel.build/modules/rules_cc/0.2.13/MODULE.bazel": "eecdd666eda6be16a8d9dc15e44b5c75133405e820f620a234acc4b1fdc5aa37", "https://bcr.bazel.build/modules/rules_cc/0.2.17/MODULE.bazel": "1849602c86cb60da8613d2de887f9566a6d354a6df6d7009f9d04a14402f9a84", "https://bcr.bazel.build/modules/rules_cc/0.2.17/source.json": "3832f45d145354049137c0090df04629d9c2b5493dc5c2bf46f1834040133a07", + "https://bcr.bazel.build/modules/rules_cc/0.2.4/MODULE.bazel": "1ff1223dfd24f3ecf8f028446d4a27608aa43c3f41e346d22838a4223980b8cc", "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8", @@ -164,8 +167,6 @@ "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d", "https://bcr.bazel.build/modules/rules_license/1.0.0/MODULE.bazel": "a7fda60eefdf3d8c827262ba499957e4df06f659330bbe6cdbdb975b768bb65c", "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb", - "https://bcr.bazel.build/modules/rules_oci/2.2.6/MODULE.bazel": "2ba6ddd679269e00aeffe9ca04faa2d0ca4129650982c9246d0d459fe2da47d9", - "https://bcr.bazel.build/modules/rules_oci/2.2.6/source.json": "94e7decb8f95d9465b0bbea71c65064cd16083be1350c7468f131818641dc4a5", "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", "https://bcr.bazel.build/modules/rules_pkg/1.0.1/MODULE.bazel": "5b1df97dbc29623bccdf2b0dcd0f5cb08e2f2c9050aab1092fd39a41e82686ff", "https://bcr.bazel.build/modules/rules_pkg/1.0.1/source.json": "bd82e5d7b9ce2d31e380dd9f50c111d678c3bdaca190cb76b0e1c71b05e1ba8a", @@ -191,8 +192,8 @@ "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", "https://bcr.bazel.build/modules/rules_shell/0.6.1/MODULE.bazel": "72e76b0eea4e81611ef5452aa82b3da34caca0c8b7b5c0c9584338aa93bae26b", - "https://bcr.bazel.build/modules/rules_shell/0.7.1/MODULE.bazel": "257dd8d667371de804918dfceb86f6ddd2e1b5e025f5d9322878d4a73dc8aa58", - "https://bcr.bazel.build/modules/rules_shell/0.7.1/source.json": "9e6c3ea5766b584f60a44dd36520c4be16a12bb173aad238e018f73083e8feb6", + "https://bcr.bazel.build/modules/rules_shell/0.8.0/MODULE.bazel": "f6a89f1d6a669a26f28fe814503857055d76306b79cfc11d12399af08d0b80ae", + "https://bcr.bazel.build/modules/rules_shell/0.8.0/source.json": "eb53cc815bc503c6683c5fe12d943f98883f81fc22f51403ec8a95610cba4195", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", @@ -200,7 +201,6 @@ "https://bcr.bazel.build/modules/rules_swift/3.1.2/source.json": "e85761f3098a6faf40b8187695e3de6d97944e98abd0d8ce579cb2daf6319a66", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", - "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", "https://bcr.bazel.build/modules/stardoc/0.5.6/MODULE.bazel": "c43dabc564990eeab55e25ed61c07a1aadafe9ece96a4efabb3f8bf9063b71ef", "https://bcr.bazel.build/modules/stardoc/0.6.2/MODULE.bazel": "7060193196395f5dd668eda046ccbeacebfd98efc77fed418dbe2b82ffaa39fd", "https://bcr.bazel.build/modules/stardoc/0.7.0/MODULE.bazel": "05e3d6d30c099b6770e97da986c53bd31844d7f13d41412480ea265ac9e8079c", @@ -210,9 +210,11 @@ "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "5e463fbfba7b1701d957555ed45097d7f984211330106ccd1352c6e0af0dcf91", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/MODULE.bazel": "75aab2373a4bbe2a1260b9bf2a1ebbdbf872d3bd36f80bff058dccd82e89422f", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/source.json": "5fba48bbe0ba48761f9e9f75f92876cafb5d07c0ce059cc7a8027416de94a05b", + "https://bcr.bazel.build/modules/tar.bzl/0.10.9/MODULE.bazel": "4636f5f6a7c34d3d0653634ed95a34560426d548c5301c6c5924c885ef737b46", + "https://bcr.bazel.build/modules/tar.bzl/0.10.9/source.json": "30a5cf0a8281adce53ab1c5a4291e88b2c161e2df1feb6cba4ee50556f1748d7", "https://bcr.bazel.build/modules/tar.bzl/0.2.1/MODULE.bazel": "52d1c00a80a8cc67acbd01649e83d8dd6a9dc426a6c0b754a04fe8c219c76468", "https://bcr.bazel.build/modules/tar.bzl/0.5.1/MODULE.bazel": "7c2eb3dcfc53b0f3d6f9acdfd911ca803eaf92aadf54f8ca6e4c1f3aee288351", - "https://bcr.bazel.build/modules/tar.bzl/0.5.1/source.json": "deed3094f7cc779ed1d37a68403847b0e38d9dd9d931e03cb90825f3368b515f", + "https://bcr.bazel.build/modules/tar.bzl/0.7.0/MODULE.bazel": "cc1acd85da33c80e430b65219a620d54d114628df24a618c3a5fa0b65e988da9", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/yq.bzl/0.1.1/MODULE.bazel": "9039681f9bcb8958ee2c87ffc74bdafba9f4369096a2b5634b88abc0eaefa072", "https://bcr.bazel.build/modules/yq.bzl/0.1.1/source.json": "2d2bad780a9f2b9195a4a370314d2c17ae95eaa745cefc2e12fbc49759b15aa3", @@ -302,172 +304,6 @@ } } }, - "@@rules_oci+//oci:extensions.bzl%oci": { - "general": { - "bzlTransitiveDigest": "TO0sjSNkDsRNtKiwgPJW4ogKM/yK77iip7GpQtVYt44=", - "usagesDigest": "DQiiJQoJaA2MzN61knQBry/pa4YyOviydpfVI5wg2jM=", - "recordedInputs": [ - "REPO_MAPPING:aspect_bazel_lib+,bazel_tools bazel_tools", - "REPO_MAPPING:bazel_features+,bazel_tools bazel_tools", - "REPO_MAPPING:rules_oci+,aspect_bazel_lib aspect_bazel_lib+", - "REPO_MAPPING:rules_oci+,bazel_features bazel_features+", - "REPO_MAPPING:rules_oci+,bazel_skylib bazel_skylib+" - ], - "generatedRepoSpecs": { - "reactantserver_cuda_base_linux_amd64": { - "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_pull", - "attributes": { - "www_authenticate_challenges": {}, - "scheme": "https", - "registry": "index.docker.io", - "repository": "nvidia/cuda", - "identifier": "sha256:08d6facb6a72f918beb3ffe5b57cff72402a203f44428831262e95c7466d4188", - "platform": "linux/amd64", - "target_name": "reactantserver_cuda_base_linux_amd64", - "bazel_tags": [] - } - }, - "reactantserver_cuda_base": { - "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_alias", - "attributes": { - "target_name": "reactantserver_cuda_base", - "www_authenticate_challenges": {}, - "scheme": "https", - "registry": "index.docker.io", - "repository": "nvidia/cuda", - "identifier": "sha256:08d6facb6a72f918beb3ffe5b57cff72402a203f44428831262e95c7466d4188", - "platforms": { - "@@platforms//cpu:x86_64": "@reactantserver_cuda_base_linux_amd64" - }, - "bzlmod_repository": "reactantserver_cuda_base", - "reproducible": true - } - }, - "oci_crane_darwin_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "darwin_amd64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_darwin_arm64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "darwin_arm64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_arm64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_arm64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_armv6": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_armv6", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_i386": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_i386", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_s390x": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_s390x", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_amd64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_windows_armv6": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "windows_armv6", - "crane_version": "v0.18.0" - } - }, - "oci_crane_windows_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "windows_amd64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_toolchains": { - "repoRuleId": "@@rules_oci+//oci/private:toolchains_repo.bzl%toolchains_repo", - "attributes": { - "toolchain_type": "@rules_oci//oci:crane_toolchain_type", - "toolchain": "@oci_crane_{platform}//:crane_toolchain" - } - }, - "oci_regctl_darwin_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "darwin_amd64" - } - }, - "oci_regctl_darwin_arm64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "darwin_arm64" - } - }, - "oci_regctl_linux_arm64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "linux_arm64" - } - }, - "oci_regctl_linux_s390x": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "linux_s390x" - } - }, - "oci_regctl_linux_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "linux_amd64" - } - }, - "oci_regctl_windows_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "windows_amd64" - } - }, - "oci_regctl_toolchains": { - "repoRuleId": "@@rules_oci+//oci/private:toolchains_repo.bzl%toolchains_repo", - "attributes": { - "toolchain_type": "@rules_oci//oci:regctl_toolchain_type", - "toolchain": "@oci_regctl_{platform}//:regctl_toolchain" - } - } - }, - "moduleExtensionMetadata": { - "explicitRootModuleDirectDeps": [ - "reactantserver_cuda_base", - "reactantserver_cuda_base_linux_amd64" - ], - "explicitRootModuleDirectDevDeps": [], - "useAllRepos": "NO", - "reproducible": false - } - } - }, "@@rules_python+//python/extensions:config.bzl%config": { "general": { "bzlTransitiveDigest": "dzD8Q2YmrP3fz8saWLHPmlwPLO91ImtTmP/c9JKTStM=", @@ -665,57 +501,6 @@ } } } - }, - "@@tar.bzl+//tar:extensions.bzl%toolchains": { - "general": { - "bzlTransitiveDigest": "/2afh6fPjq/rcyE/jztQDK3ierehmFFngfvmqyRv72M=", - "usagesDigest": "maF8qsAIqeH1ey8pxP0gNZbvJt34kLZvTFeQ0ntrJVA=", - "recordedInputs": [], - "generatedRepoSpecs": { - "bsd_tar_toolchains": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:toolchain.bzl%tar_toolchains_repo", - "attributes": { - "user_repository_name": "bsd_tar_toolchains" - } - }, - "bsd_tar_toolchains_darwin_amd64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "bsd_tar_toolchains_darwin_arm64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "bsd_tar_toolchains_linux_amd64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "bsd_tar_toolchains_linux_arm64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "bsd_tar_toolchains_windows_amd64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "windows_amd64" - } - }, - "bsd_tar_toolchains_windows_arm64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "windows_arm64" - } - } - } - } } }, "facts": {}, diff --git a/deploy/BUILD.bazel b/deploy/BUILD.bazel index 888efbd5..f95fe62c 100644 --- a/deploy/BUILD.bazel +++ b/deploy/BUILD.bazel @@ -1,4 +1,12 @@ -load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load") +load( + "@rules_julia_depot//julia:image.bzl", + "julia_compiled_layer", + "julia_depot_layer", + "julia_dist_layer", + "julia_image_env", + "julia_precompile_test", +) +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") load("@rules_shell//shell:sh_binary.bzl", "sh_binary") load("@rules_shell//shell:sh_test.bzl", "sh_test") @@ -9,18 +17,25 @@ package(default_visibility = ["//visibility:public"]) # The ReactantServer node image, built from the committed deploy/Manifest.toml. # # bazel build //deploy:image -# bazel run //deploy:image_load # loads it into podman as reactantserver:bazel -# bazel run //deploy:image_check # the loaded image starts without precompiling -# bazel test //deploy:manifest_current # the manifest still matches the workspace -# bazel run //deploy:relock # re-resolve deploy/Manifest.toml after a Project change -# bazel run //deploy:relock_debs # re-resolve deploy/debs.lock.json (curl, tini) +# bazel test //deploy:image_precompile_test # the image starts without precompiling (no podman) +# bazel run //deploy:image_load # loads it into podman as reactantserver:bazel +# bazel run //deploy:image_check # the loaded image: caches, tini, curl, entrypoints +# bazel run //deploy:image_push -- --repository index.docker.io// # pushes :latest +# bazel test //deploy:manifest_current # the manifest still matches the workspace +# bazel run //deploy:relock # re-resolve deploy/Manifest.toml after a Project change +# bazel run //deploy:relock_debs # re-resolve deploy/debs.lock.json (curl, tini) +# +# The Julia-specific layers (the distribution, the depot, the precompile caches), the image's +# environment and the precompile test come from @rules_julia_depot//julia:image.bzl. What stays here +# is what is ours: the Ubuntu packages, the CUDA driver stub the build loads Reactant against, the +# application layer, and the image itself. # # WHY A SECOND MANIFEST. The workspace root's Manifest.toml is gitignored on purpose: development # and CI resolve fresh so compat bumps are picked up. An image must not: this manifest is the # production lock, moved deliberately with //deploy:relock and reviewed like any other change. # It is a manifest for the ROOT Project.toml, and it lives here only so the gitignore can stay as -# it is. Every action stages it beside the root project and the member Project.tomls in a -# temporary tree, because Pkg resolves a manifest next to its (real, symlink-resolved) project. +# it is. The depot layer stages it beside the root project and the member Project.tomls, because +# Pkg resolves a manifest next to its (real, symlink-resolved) project. # CUDA userspace the Reactant_jll artifact is selected for. Must agree with the base image # (@reactantserver_cuda_base, MODULE.bazel): Reactant_jll's platform augmentation reads these at @@ -31,21 +46,14 @@ _REACTANT_GPU_ENV = { "REACTANT_GPU_VERSION": "13.1", } -# Where the image keeps Julia's state, and the order Julia searches it. The workspace root is a -# depot too, and not by accident: Julia records a precompiled package's source files relative to -# the depot that holds them ("@depot/..."), which is what lets a cache built in a temporary tree -# load from /opt. Packages outside every depot are recorded by absolute path and would be stale in -# the image, so the workspace root is listed as a depot to make the workspace's own packages -# relocatable like the registry ones. Julia's bundled depots follow, for the stdlib caches. -# //deploy:compiled_layer builds against exactly this list, with the build tree in place of /opt. -_DEPOT = "/opt/julia-depot" +# Where the image keeps the application. Julia and the depot take image.bzl's defaults, +# /opt/julia and /opt/julia-depot. _APP = "/opt/reactantserver" -_JULIA = "/opt/julia" -_DEPOT_PATH = ":".join([_DEPOT, _APP, _JULIA + "/local/share/julia", _JULIA + "/share/julia"]) -# The projects the image starts Julia in, each precompiled by //deploy:compiled_layer and checked -# by //deploy:image_check. A package's cache depends on the active project's preferences (the -# worker's LocalPreferences.toml configures Reactant), so each entry point needs its own. +# The projects the image starts Julia in, relative to _APP, each precompiled by +# //deploy:compiled_layer and checked by //deploy:image_precompile_test. A package's cache depends +# on the active project's preferences (the worker's LocalPreferences.toml configures Reactant), so +# each entry point needs its own. # packages/ReactantServerNode the supervisor (docker/entrypoint.node.sh) # packages/ReactantServer a worker (spawned by the supervisor, or entrypoint.worker.sh) # packages/ReactantServerGateway the embedded gateway (spawned by the supervisor) @@ -57,11 +65,18 @@ _ENTRY_PROJECTS = [ ".", ] -# The CPU targets the caches are compiled for: the official Julia build's x86_64 set, so the caches -# load on any x86_64 host rather than only on CPUs like the build machine's. Julia rejects a cache -# built for a CPU it is not running on and recompiles, which is the first-start cost this layer -# exists to remove. -_CPU_TARGET = "generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1)" +_ENTRY_PROJECT_PATHS = [_APP if p == "." else _APP + "/" + p for p in _ENTRY_PROJECTS] + +# Where the CUDA driver stub sits in the build-time tree. Never in the image; see :cuda_stub_layer. +_CUDA_STUB_DIR = "/build/cuda-stub" + +# What Julia needs to load the stack on a build host: the driver stub in place of libcuda.so.1, the +# host's GPUs hidden so a build never touches them, and the artifact selection. `{root}` is the +# tree the layers are unpacked into (see image.bzl). +_BUILD_LOAD_ENV = _REACTANT_GPU_ENV | { + "CUDA_VISIBLE_DEVICES": "", + "LD_LIBRARY_PATH": "{root}" + _CUDA_STUB_DIR, +} _STAGE = """ workspace_path() { @@ -87,9 +102,10 @@ stage_workspace() { } """ -# The layers write deterministic tars: sorted, epoch mtimes, root-owned by number. Without these a -# layer's digest changes on every build and nothing downstream can be cached or compared. -_TAR = "tar --create --file \"$@\" --sort=name --mtime=@0 --owner=0 --group=0 --numeric-owner" +# The layers written here are deterministic tars in the same form as image.bzl's: sorted, epoch +# mtimes, root-owned by number, permissions normalised. Without these a layer's digest changes on +# every build and nothing downstream can be cached or compared. +_TAR = "LC_ALL=C tar --create --file \"$@\" --format=gnu --sort=name --mtime=@0 --owner=0 --group=0 --numeric-owner --mode='u=rwX,go=rX'" # The Ubuntu packages the image adds to the base (curl, tini and the libraries curl needs), from # the pinned downloads in deploy/debs.lock.json, unpacked the way dpkg would lay them out. @@ -134,16 +150,18 @@ done ) # The CUDA driver stub, libcuda.so, taken from the pinned CUDA base itself so it always matches the -# base's CUDA release. Build time only: the CUDA build of libReactantExtra.so lists libcuda.so.1 as -# NEEDED, so loading Reactant (which precompiling does) fails on a host without an NVIDIA driver, -# and would load the host's real driver on one that has it. The stub answers every call with "no -# device" and never reaches the image; a container gets the real driver from its host. +# base's CUDA release, as a layer at _CUDA_STUB_DIR. BUILD TIME ONLY: it is among the layers of +# :compiled_layer and :image_precompile_test and never among the image's. The CUDA build of +# libReactantExtra.so lists libcuda.so.1 as NEEDED, so loading Reactant (which precompiling does) +# fails on a host without an NVIDIA driver, and would load the host's real driver on one that has +# it. The stub answers every call with "no device"; a container gets the real driver from its host. # -# The layers are searched in manifest order and the last copy wins, as an overlay would resolve it. +# The base's layers are searched in manifest order and the last copy wins, as an overlay would +# resolve it. genrule( - name = "cuda_stub", + name = "cuda_stub_layer", srcs = ["@reactantserver_cuda_base_linux_amd64"], - outs = ["cuda_stub/libcuda.so.1"], + outs = ["reactantserver-cuda-stub.tar"], cmd = """ set -euo pipefail layout=$(location @reactantserver_cuda_base_linux_amd64) @@ -151,128 +169,79 @@ blobs="$$layout/blobs/sha256" member=usr/local/cuda-{gpu_version}/targets/x86_64-linux/lib/stubs/libcuda.so manifest="$$(grep -o 'sha256:[0-9a-f]*' "$$layout/index.json" | head -1 | cut -d: -f2)" found="$$(mktemp -d)" -trap 'rm -rf "$$found"' EXIT +stage="$$(mktemp -d)" +trap 'rm -rf "$$found" "$$stage"' EXIT for layer in $$(tr ',' '\\n' < "$$blobs/$$manifest" | sed -n '/"layers"/,$$p' | grep -o 'sha256:[0-9a-f]*' | cut -d: -f2); do $(BSDTAR_BIN) -xf "$$blobs/$$layer" -C "$$found" "$$member" 2>/dev/null || true done [ -f "$$found/$$member" ] || {{ echo "FAILED: $$member is not in the base image" >&2; exit 1; }} -cp "$$found/$$member" "$@" -""".format(gpu_version = _REACTANT_GPU_ENV["REACTANT_GPU_VERSION"]), +mkdir -p "$$stage{stub_dir}" +cp "$$found/$$member" "$$stage{stub_dir}/libcuda.so.1" +""".format( + gpu_version = _REACTANT_GPU_ENV["REACTANT_GPU_VERSION"], + stub_dir = _CUDA_STUB_DIR, + ) + _TAR + """ --directory "$$stage" {top} +""".format(top = _CUDA_STUB_DIR.split("/")[1]), toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], tools = ["@bsd_tar_toolchains//:resolved_toolchain"], ) -# The depot, instantiated from the manifest into an empty tree and tarred at /opt/julia-depot. +# The image's Julia layout and environment, declared once: the oci_image takes the file this +# writes, and the compiled layer and the precompile test take the same facts, so the caches are +# built for, and checked against, exactly the environment the image runs with. # -# FULL, not artifacts-only: the image loads its packages from source (with the caches of -# //deploy:compiled_layer), not from a sysimage, so the packages themselves are the code. +# The workspace root is a depot too (extra_depots), and not by accident: Julia records a +# precompiled package's source files relative to the depot that holds them ("@depot/..."), which is +# what lets a cache built in a temporary tree load from /opt. Packages outside every depot are +# recorded by absolute path and would be stale in the image, so the workspace root is listed to make +# the workspace's own packages relocatable like the registry ones. The search order comes out as # -# The source depot handed to image_depot.sh is EMPTY, so the General registry is fetched into the -# clean depot rather than copied from whatever this host has; the manifest pins every package by -# tree hash, so the registry's state cannot change what is installed. Set JULIA_PKG_SERVER in -# user.bazelrc (--action_env) to go through a mirror. -genrule( - name = "depot_layer", - srcs = [ - "Manifest.toml", - "//:workspace_projects", - "@reactantserver_julia//:dist", - "@reactantserver_julia//:bin/julia", - ], - outs = ["reactantserver-depot.tar"], - cmd = _STAGE + """ -set -euo pipefail -export JULIA_BIN="$$(cd "$$(dirname $(location @reactantserver_julia//:bin/julia))" && pwd)/julia" -stage="$$(mktemp -d)" -src_depot="$$(mktemp -d)" -trap 'rm -rf "$$stage" "$$src_depot"' EXIT -stage_workspace "$$stage" -export JULIA_DEPOT_PATH="$$src_depot" -export REACTANT_GPU={gpu} REACTANT_GPU_VERSION={gpu_version} -JULIA_DEPOT_CONTENTS=full \\ - $(location @rules_julia_depot//julia:image_depot.sh) "$$stage" "$@" -""".format( - gpu = _REACTANT_GPU_ENV["REACTANT_GPU"], - gpu_version = _REACTANT_GPU_ENV["REACTANT_GPU_VERSION"], - ), - tags = ["requires-network"], # packages, artifacts and the registry from the package server - tools = ["@rules_julia_depot//julia:image_depot.sh"], -) - -# The precompile caches, so a container starts serving without first compiling ~150 packages. +# /opt/julia-depot:/opt/reactantserver:/opt/julia/local/share/julia:/opt/julia/share/julia # -# Built from the depot and application layers themselves, unpacked into one tree with the image's -# layout, so the caches are for exactly the files the image ships. Julia validates a cache against -# its sources by content, and records them relative to their depot (see _DEPOT_PATH), so caches -# compiled here under /opt load unchanged from /opt. The CUDA stub stands in for the driver -# while Reactant loads; GPUs are hidden so the build never touches the host's. +# The caches are compiled for the official Julia build's x86_64 CPU targets (image.bzl's default), +# so they load on any x86_64 host rather than only on CPUs like the build machine's. # -# Precompiling is also the first time the whole stack is loaded, so a package that fails to load -# fails the build here rather than a container at start. -genrule( - name = "compiled_layer", - srcs = [ - ":app_layer", - ":cuda_stub", - ":depot_layer", - "@reactantserver_julia//:bin/julia", - "@reactantserver_julia//:dist", - ], - outs = ["reactantserver-compiled.tar"], - cmd = """ -set -euo pipefail -julia="$$(readlink -f $(location @reactantserver_julia//:bin/julia))" -julia_prefix="$$(cd "$$(dirname "$$julia")/.." && pwd)" -root="$$(mktemp -d)" -stub="$$(mktemp -d)" -# Pkg leaves installed packages read-only, which rm -rf alone cannot delete. -trap 'chmod -R u+w "$$root"; rm -rf "$$root" "$$stub"' EXIT -tar --extract --file $(location :depot_layer) --directory "$$root" --no-same-owner -tar --extract --file $(location :app_layer) --directory "$$root" --no-same-owner -cp $(location :cuda_stub) "$$stub/libcuda.so.1" -# _DEPOT_PATH, with the build tree in place of /opt and the toolchain's Julia in place of /opt/julia. -export JULIA_DEPOT_PATH="$$root{depot}:$$root{app}:$$julia_prefix/local/share/julia:$$julia_prefix/share/julia" -export JULIA_CPU_TARGET='{cpu_target}' -export JULIA_PKG_OFFLINE=true -export CUDA_VISIBLE_DEVICES= -export LD_LIBRARY_PATH="$$stub" -export REACTANT_GPU={gpu} REACTANT_GPU_VERSION={gpu_version} -for p in {projects}; do - echo "==> precompiling $$p" - "$$julia" --startup-file=no --project="$$root{app}/$$p" -e 'using Pkg; Pkg.precompile(; strict = true)' -done -""".format( - app = _APP, - cpu_target = _CPU_TARGET, - depot = _DEPOT, - gpu = _REACTANT_GPU_ENV["REACTANT_GPU"], - gpu_version = _REACTANT_GPU_ENV["REACTANT_GPU_VERSION"], - projects = " ".join(_ENTRY_PROJECTS), - ) + _TAR + """ --directory "$$root" {compiled} -""".format(compiled = _DEPOT.lstrip("/") + "/compiled"), +# PATH becomes /opt/julia/bin:$PATH, which rules_oci expands against the CUDA base's own PATH. +# JULIA_PKG_OFFLINE is set too: nothing in the image resolves or installs, and a stale cache (a +# mounted, edited package) still precompiles into the depot offline. +julia_image_env( + name = "image_env", + env = _REACTANT_GPU_ENV | { + "REACTANT_WORKSPACE_ROOT": _APP, + "JULIA_SSL_CA_ROOTS_PATH": "/etc/ssl/certs/ca-certificates.crt", + # utility as well as compute: the entrypoint's GPU-reclaim gate runs nvidia-smi. + "NVIDIA_DRIVER_CAPABILITIES": "compute,utility", + "NVIDIA_VISIBLE_DEVICES": "all", + }, + extra_depots = [_APP], + project = _APP, ) -# Julia as a layer at /opt/julia rather than as the base, because the base has to be CUDA's. -genrule( +# Julia at /opt/julia rather than as the base, because the base has to be CUDA's. The +# distribution's own relative symlinks are kept, so its libraries are not shipped twice. +julia_dist_layer( name = "julia_layer", - srcs = [ - "@reactantserver_julia//:dist", - "@reactantserver_julia//:bin/julia", - ], - outs = ["reactantserver-julia.tar"], - cmd = """ -set -euo pipefail -root="$$(cd "$$(dirname $(location @reactantserver_julia//:bin/julia))/.." && pwd)" -stage="$$(mktemp -d)" -trap 'rm -rf "$$stage"' EXIT -mkdir -p "$$stage/opt" -# -L, NOT plain -a: an external repo is a tree of symlinks into the execroot, and copying the -# links would ship /opt/julia/bin/julia pointing at a path that exists only on the build host. -cp -aL "$$root" "$$stage/opt/julia" -chmod -R u+w "$$stage/opt/julia" -rm -f "$$stage/opt/julia/BUILD.bazel" "$$stage/opt/julia/WORKSPACE" "$$stage/opt/julia/REPO.bazel" -""" + _TAR + """ --directory "$$stage" opt -""", + julia = "@reactantserver_julia//:dist", +) + +# The depot, instantiated from the manifest into an empty tree at /opt/julia-depot. +# +# FULL, not artifacts-only: the image loads its packages from source (with the caches of +# //deploy:compiled_layer), not from a sysimage, so the packages themselves are the code. +# +# No `depot`, so the General registry is fetched into the clean depot rather than copied from +# whatever this host has; the manifest pins every package by tree hash, so the registry's state +# cannot change what is installed. Set JULIA_PKG_SERVER in user.bazelrc (--action_env) to go +# through a mirror. +julia_depot_layer( + name = "depot_layer", + contents = "full", + env = _REACTANT_GPU_ENV, + julia = "@reactantserver_julia//:dist", + manifest = "Manifest.toml", + project = "//:Project.toml", + # The member Project.tomls, which the workspace's [workspace] table needs to instantiate. + srcs = ["//:workspace_projects"], ) # The application at /opt/reactantserver, in the workspace layout the entrypoints expect: the root @@ -307,11 +276,43 @@ for f in entrypoint.node.sh entrypoint.worker.sh healthcheck.node.sh healthcheck ln -s "{app}/docker/$$f" "$$stage/usr/local/bin/$$f" done cp "$$app/config/node.default.yaml" "$$stage/etc/reactantserver/node.yaml" -chmod -R a+rX "$$stage/opt" "$$stage/etc" """.format(app = _APP) + _TAR + """ --directory "$$stage" etc opt usr """, ) +# The precompile caches, so a container starts serving without first compiling ~150 packages. +# +# Built from the image's own Julia, depot and application layers, unpacked into one tree with the +# image's layout, so the caches are for exactly the files the image ships, and compiled for the +# depot path and CPU targets of :image_env, so they load unchanged from /opt. The driver stub layer +# rides along for the build only. +# +# Precompiling is also the first time the whole stack is loaded, so a package that fails to load +# fails the build here rather than a container at start. +julia_compiled_layer( + name = "compiled_layer", + env = _BUILD_LOAD_ENV, + image_env = ":image_env", + julia = "@reactantserver_julia//:dist", + layers = [ + ":julia_layer", + ":depot_layer", + ":app_layer", + ":cuda_stub_layer", + ], + projects = _ENTRY_PROJECT_PATHS, +) + +# The layers in the order they change, least often first, so a push or a pull after a code change +# moves only the application layer (and the caches, when a package's sources change). +_IMAGE_LAYERS = [ + ":debs_layer", + ":julia_layer", + ":depot_layer", + ":compiled_layer", + ":app_layer", +] + # Root, and not by oversight: the entrypoint's GPU-reclaim gate and the supervisor expect it, and a # stale cache (a mounted, edited package) must still be able to precompile into the depot. oci_image( @@ -325,43 +326,53 @@ oci_image( "--", "/usr/local/bin/entrypoint.node.sh", ], - env = _REACTANT_GPU_ENV | { - # Julia is a layer, so nothing else puts it on PATH; the entrypoints exec a bare `julia`. - # The rest is the CUDA base's own PATH, repeated because this key replaces it. - "PATH": _JULIA + "/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", - "JULIA_DEPOT_PATH": _DEPOT_PATH, - "JULIA_PROJECT": _APP, - "REACTANT_WORKSPACE_ROOT": _APP, - "JULIA_SSL_CA_ROOTS_PATH": "/etc/ssl/certs/ca-certificates.crt", - # utility as well as compute: the entrypoint's GPU-reclaim gate runs nvidia-smi. - "NVIDIA_DRIVER_CAPABILITIES": "compute,utility", - "NVIDIA_VISIBLE_DEVICES": "all", - }, + env = ":image_env", # The embedded gateway's gRPC and metrics/health ports. The workers role binds per-worker ports # (base_port+i, metrics_base_port+i) instead; publish the range you configured. exposed_ports = [ "8001/tcp", "8002/tcp", ], - tars = [ - ":debs_layer", - ":depot_layer", - ":compiled_layer", - ":julia_layer", - ":app_layer", - ], + tars = _IMAGE_LAYERS, workdir = _APP, ) +# Every entry project loads from the baked caches with nothing precompiled and no cache rejected, +# checked on the image's layers without a container, so it needs no podman. Manual because it +# builds the whole image; //deploy:image_check covers what only a container shows. +julia_precompile_test( + name = "image_precompile_test", + size = "large", + env = _BUILD_LOAD_ENV, + image_env = ":image_env", + julia = "@reactantserver_julia//:dist", + layers = [l for l in _IMAGE_LAYERS if l != ":debs_layer"] + [":cuda_stub_layer"], + projects = _ENTRY_PROJECT_PATHS, + tags = ["manual"], +) + oci_load( name = "image_load", image = ":image", # Explicit, because rules_oci probes for docker first, and on hosts where a docker CLI is # installed but its socket is unreachable that probe wins and fails. - loader = "@rules_julia_depot//oci:podman.sh", + loader = "podman.sh", repo_tags = ["reactantserver:bazel"], ) +# Push the image as :latest. The repository is given at run time, so this file names no registry +# account; credentials come from the docker config (`docker login`, `podman login --authfile +# ~/.docker/config.json`, or docker/login-action in CI): +# +# bazel run //deploy:image_push -- --repository index.docker.io// +# +# .github/workflows/image.yml runs exactly this. +oci_push( + name = "image_push", + image = ":image", + remote_tags = ["latest"], +) + # Check a loaded image: every entry project loads from the baked caches without precompiling, and # the tools the entrypoint and healthcheck call are present. Needs podman, and the image loaded # first (bazel run //deploy:image_load); runs on a host with or without a GPU. diff --git a/deploy/README.md b/deploy/README.md index 0f12ebe1..776bcaaa 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -9,8 +9,10 @@ Julia. ``` bazel build //deploy:image +bazel test //deploy:image_precompile_test # the image starts without precompiling (no podman) bazel run //deploy:image_load # loads it into podman as localhost/reactantserver:bazel -bazel run //deploy:image_check # the loaded image starts without precompiling +bazel run //deploy:image_check # the loaded image: caches, tini, curl, entrypoints +bazel run //deploy:image_push -- --repository index.docker.io// # push as :latest bazel test //deploy:manifest_current # the Julia lock still matches the workspace Project.toml files bazel run //deploy:relock # re-resolve the Julia lock after a [deps] or [compat] change bazel run //deploy:relock_debs # re-resolve the Ubuntu package lock (curl, tini) @@ -18,6 +20,12 @@ bazel run //deploy:relock_debs # re-resolve the Ubuntu package lock (c Or `make bazel-image`, which builds and loads it. +The Julia-specific parts (the distribution, depot and precompile-cache layers, the image +environment, and the precompile test) are the image rules of +[rules_julia_depot](https://github.com/csvance/rules_julia_depot) (`julia/image.bzl`); this +package adds the Ubuntu packages, the CUDA driver stub the build loads Reactant against, the +application layer, and the image itself, assembled with rules_oci. + Julia, the CUDA base image, and every package are fetched and pinned by Bazel. The build host needs Bazel (the version in `.bazelversion`; Bazelisk picks it up), GNU tar, and a C compiler with `make`, because InterProcessCommunication.jl generates its constants file by compiling a small C program when @@ -65,7 +73,11 @@ against the base. ## Layers +In image order, least often changed first, so a push after a code change moves only the last +layers: + - `debs_layer`: `curl`, `tini`, and their libraries, from the Ubuntu lock. +- `julia_layer`: Julia at `/opt/julia`, its relative symlinks kept. - `depot_layer`: the Julia depot at `/opt/julia-depot`, instantiated from the Julia lock into an empty depot (the General registry is fetched fresh; the lock pins every package by tree hash, so the registry state cannot change what is installed). Set `JULIA_PKG_SERVER` through @@ -73,12 +85,15 @@ against the base. - `compiled_layer`: the precompile caches for every project the image starts Julia in (the supervisor, a worker, the gateway, and the worker-role healthcheck), so a container serves without first compiling about 150 packages. -- `julia_layer`: Julia at `/opt/julia`. - `app_layer`: the workspace at `/opt/reactantserver` (root project plus the Julia lock, member packages without their tests, `docker/`, `config/`), the entrypoints and healthchecks linked into `/usr/local/bin`, and the default node file at `/etc/reactantserver/node.yaml`, all where `docker/Dockerfile` puts them. +`image_env` writes the image's environment: the depot path below, `JULIA_PROJECT`, the portable +`JULIA_CPU_TARGET`, `JULIA_PKG_OFFLINE=true`, Julia's `bin/` ahead of the base's `PATH`, and the +CUDA and NVIDIA variables. + The base is `nvidia/cuda:13.1.2-cudnn-devel-ubuntu24.04`, pinned by digest. It is deliberately fat: Reactant_jll's CUDA 13.1 artifact links cuDNN and cuBLASLt statically but `dlopen`s `libnvJitLink.so.13`, `libnvrtc.so.13`, and `libcupti.so` by soname, and the NVIDIA container @@ -102,10 +117,13 @@ JULIA_DEPOT_PATH=/opt/julia-depot:/opt/reactantserver:/opt/julia/local/share/jul `compiled_layer` unpacks the depot and application layers into one temporary tree with the image's layout and precompiles against the same list with that tree in place of `/opt`, so its caches load unchanged in the image. The loader needs `libcuda.so.1` to load Reactant; the build takes the -driver stub from the CUDA base (`cuda_stub`) and hides the host's GPUs. The caches are compiled for +driver stub from the CUDA base as a build-time layer (`cuda_stub_layer`, unpacked beside the others +but never part of the image), puts it on `LD_LIBRARY_PATH`, and hides the host's GPUs. The caches are compiled for the official Julia build's x86_64 CPU targets, so they load on any x86_64 host, not only CPUs like -the build machine's. `//deploy:image_check` verifies that each entry project loads with no cache -rejected and nothing precompiled. +the build machine's. `//deploy:image_precompile_test` verifies, on the layers and without a +container, that each entry project loads with no cache rejected and nothing precompiled; +`//deploy:image_check` repeats that inside a loaded container and also checks `tini`, `curl`, and +the entrypoints. ## Running it @@ -132,6 +150,16 @@ podman run ... --health-cmd /usr/local/bin/healthcheck.node.sh --health-interval --health-timeout 20s --health-retries 3 --health-start-period 300s localhost/reactantserver:bazel ``` +## Publishing + +`.github/workflows/image.yml` builds the image, runs `//deploy:manifest_current` and +`//deploy:image_precompile_test`, and pushes it to Docker Hub as `:latest`. It runs only when +triggered by hand (Actions, Image, Run workflow). It reads the target from the +`DOCKERHUB_REPOSITORY` repository variable (for example `index.docker.io//`) and logs +in with the `DOCKERHUB_USERNAME` and `DOCKERHUB_TOKEN` secrets. Releases are not named, so each run +replaces `:latest`; the run summary records the pushed digest and commit, so pull by digest to pin +a deployment. + ## Consuming from another module The image and its layers are public. A deployment repository can take a `bazel_dep` on diff --git a/deploy/podman.sh b/deploy/podman.sh new file mode 100755 index 00000000..3ec9525d --- /dev/null +++ b/deploy/podman.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# The loader for //deploy:image_load. rules_oci probes `command -v docker` before podman, and on a +# host where a docker CLI is installed but its socket is unreachable that probe wins and the load +# fails, so the loader is named rather than discovered. +exec podman "$@" From 1aba5f23c2c7aa2bbe2a29d2fb25b6427549e8dc Mon Sep 17 00:00:00 2001 From: Carroll Vance Date: Sun, 4 Oct 2026 20:02:43 +0000 Subject: [PATCH 2/3] Pin rules_julia_depot to its main branch (cead347) The image-layers branch, including the {root} env placeholder (f8d7c22) that the CUDA stub layer relies on, is merged into main as cead347. Pinning the merge commit keeps this module on rules_julia_depot's main line instead of a feature-branch commit. The only commit added since f8d7c22 (d266e89) changes e2e tests only. Co-Authored-By: Claude Opus 5.5 (1M context) --- MODULE.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 1321e8c8..0a317301 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -52,7 +52,7 @@ use_repo(debs, "reactantserver_debs") bazel_dep(name = "rules_julia_depot", version = "0.1.0") git_override( module_name = "rules_julia_depot", - commit = "f8d7c226136e7f02146d80088c02cb59a97096ad", + commit = "cead3479d64e7bf56fac66a15b72e9c4cd45ddc8", remote = "https://github.com/csvance/rules_julia_depot.git", ) From 4e818b0dc387761f424ab70550e8b345f4cdb921 Mon Sep 17 00:00:00 2001 From: Carroll Vance Date: Sun, 4 Oct 2026 20:24:02 +0000 Subject: [PATCH 3/3] Update MODULE.bazel.lock from the first full image build The first full build of //deploy:image against the pinned rules_julia_depot and rules_oci 2.3.3 evaluated a module extension that analysis alone never reached (pybind11_bazel's internal_configure), and Bazel recorded it. Committing it keeps the next build from rewriting the lockfile in a clean checkout. It records no host-specific paths. Co-Authored-By: Claude Opus 5.5 (1M context) --- MODULE.bazel.lock | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 1ab8a82b..7d4e3fce 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -225,6 +225,28 @@ }, "selectedYankedVersions": {}, "moduleExtensions": { + "@@pybind11_bazel+//:internal_configure.bzl%internal_configure_extension": { + "general": { + "bzlTransitiveDigest": "NRXra7941UfmNUyIxnLt82V5hULluVGL2nBsijTl4j4=", + "usagesDigest": "D1r3lfzMuUBFxgG8V6o0bQTLMk3GkaGOaPzw53wrwyw=", + "recordedInputs": [ + "REPO_MAPPING:pybind11_bazel+,bazel_tools bazel_tools", + "FILE:@@pybind11_bazel+//MODULE.bazel e6f4c20442eaa7c90d7190d8dc539d0ab422f95c65a57cc59562170c58ae3d34" + ], + "generatedRepoSpecs": { + "pybind11": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file": "@@pybind11_bazel+//:pybind11-BUILD.bazel", + "strip_prefix": "pybind11-2.12.0", + "urls": [ + "https://github.com/pybind/pybind11/archive/v2.12.0.zip" + ] + } + } + } + } + }, "@@rules_julia_depot+//julia:extensions.bzl%julia": { "general": { "bzlTransitiveDigest": "OsJA58rY4YQkeDkx4AoHFcLUMQpp9niR9zYBH9RsVzU=",