diff --git a/.bazelignore b/.bazelignore index f5510a30..58b789aa 100644 --- a/.bazelignore +++ b/.bazelignore @@ -4,3 +4,5 @@ docs/build lib private examples +# The Python env CondaPkg creates for the export tests; it ships BUILD files of its own. +packages/ReactantServerExport/test/.CondaPkg diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml new file mode 100644 index 00000000..7dbabecb --- /dev/null +++ b/.github/workflows/image.yml @@ -0,0 +1,72 @@ +# Build the locked CUDA node image with Bazel (//deploy:image) and push it to Docker Hub as :latest. +# Triggered by hand only: Actions > Image > Run workflow, or `gh workflow run image.yml --ref `. +# +# Releases are not named yet, so every run overwrites :latest. The image digest is printed in the +# run summary; pull by that digest to pin a deployment. +# +# Needs, in the repository settings: +# variable DOCKERHUB_REPOSITORY the target, e.g. index.docker.io// +# secret DOCKERHUB_USERNAME +# secret DOCKERHUB_TOKEN a Docker Hub access token with write access to that repository +name: Image + +on: + workflow_dispatch: + +# Two runs pushing :latest at once would race; the later one waits instead. +concurrency: + group: image-latest + cancel-in-progress: false + +permissions: + contents: read + +jobs: + image: + runs-on: ubuntu-latest + timeout-minutes: 240 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + # The CUDA base, the depot (Reactant's CUDA artifact included) and the layers built from them + # need tens of gigabytes. The runner's root disk is too small for that once its preinstalled + # toolchains are counted, so those are removed and Bazel keeps everything on /mnt. + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + sudo docker image prune --all --force + sudo mkdir -p /mnt/bazel + sudo chown "$(id -u):$(id -g)" /mnt/bazel + printf '%s\n' \ + 'startup --output_user_root=/mnt/bazel/root' \ + 'common --repository_cache=/mnt/bazel/repository-cache' \ + > user.bazelrc + df -h / /mnt + + # Bazelisk is preinstalled as `bazel` and runs the version in .bazelversion. + - name: Check the Julia lock + run: bazel test //deploy:manifest_current + + # Builds every layer, then loads every entry project from them with nothing precompiled and + # no cache rejected. A failure here stops the run before anything is pushed. + - name: Build the image and check it starts warm + run: bazel test //deploy:image_precompile_test + + - name: Log in to Docker Hub + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Push :latest + env: + REPOSITORY: ${{ vars.DOCKERHUB_REPOSITORY }} + run: | + : "${REPOSITORY:?set the DOCKERHUB_REPOSITORY repository variable}" + bazel run //deploy:image_push -- --repository "$REPOSITORY" + digest="$(jq -r '.manifests[0].digest' bazel-bin/deploy/image/index.json)" + { + echo "Pushed \`$REPOSITORY:latest\`" + echo + echo "Digest: \`$digest\` (commit \`$GITHUB_SHA\`)" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/BUILD.bazel b/BUILD.bazel index f6b37617..1460a5a6 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -3,6 +3,9 @@ package(default_visibility = ["//deploy:__pkg__"]) +# The root project, which //deploy:depot_layer instantiates from the committed deploy/Manifest.toml. +exports_files(["Project.toml"]) + # ReactantServerExport is offline tooling and not a workspace member (see Project.toml); the image # does not carry it. _NOT_IN_IMAGE = ["packages/ReactantServerExport/**"] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ee927b7b..74ec7b13 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -139,14 +139,18 @@ bazel test //deploy:manifest_current The Ubuntu packages the image adds to the CUDA base (`curl`, `tini`) are locked the same way in `deploy/debs.lock.json`; moving the base image digest in `MODULE.bazel` calls for -`bazel run //deploy:relock_debs`. After any change to the image, load it and check that it starts -without precompiling: +`bazel run //deploy:relock_debs`. After any change to the image, check that it starts without +precompiling, on its layers and then in a loaded container: ``` +bazel test //deploy:image_precompile_test bazel run //deploy:image_load bazel run //deploy:image_check ``` +The image is published to Docker Hub as `:latest` by the manually triggered +`.github/workflows/image.yml`; see [`deploy/README.md`](deploy/README.md#publishing). + ## Kaimon gate tools (dev only) `packages/ReactantServer/ext/ReactantServerKaimonGateExt.jl` registers four `rserver_*` GateTools with a running diff --git a/MODULE.bazel b/MODULE.bazel index 18b67312..0a317301 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -10,21 +10,32 @@ module( # build, so the same commit always yields the same package and artifact set. # Bazel 9 moved sh_binary/sh_test out of the core into rules_shell. -bazel_dep(name = "rules_shell", version = "0.7.1") +bazel_dep(name = "rules_shell", version = "0.8.0") # rules_oci assembles OCI images with no daemon; podman is needed only to load and run the result. -bazel_dep(name = "rules_oci", version = "2.2.6") +# +# 2.3.3, the latest release, which has not reached the Bazel Central Registry (2.3.0 is the newest +# there), so it is fetched from the release archive. The fix that matters here is oci_load's: before +# 2.3.1 (#881) it reported success when the tar it streams into the loader failed, so a broken +# load of this multi-gigabyte image looked like a good one. 2.3.1 also makes oci_pull repositories +# resolve their own dependencies under bzlmod (#891). Since 2.3.0 it also depends on a +# current aspect_bazel_lib, so the Bazel 9 workaround of naming aspect_bazel_lib here is gone. +# Drop the override once the registry has 2.3.3 or later. +bazel_dep(name = "rules_oci", version = "2.3.3") +archive_override( + module_name = "rules_oci", + integrity = "sha256-JD6sh+xB1o0dnm0hiEpQ7CjgMfcCsLn83H+Xw/PVOJo=", + strip_prefix = "rules_oci-2.3.3", + urls = ["https://github.com/bazel-contrib/rules_oci/releases/download/v2.3.3/rules_oci-v2.3.3.tar.gz"], +) -# rules_oci 2.2.6 pins aspect_bazel_lib 2.7.2, which registers its tar and zstd toolchains -# against @local_config_platform. Bazel 9 removed that repo, so toolchain resolution fails for -# every target, including ones unrelated to images ("invalid registered toolchain -# '@zstd_toolchains//:all'"). MVS takes the higher version, so naming it here is the whole fix. -bazel_dep(name = "aspect_bazel_lib", version = "2.22.5") +# bsdtar, from the toolchain rules_oci uses (tar.bzl since rules_oci 2.3), reads .deb files (ar +# archives holding zstd or xz tarballs) and the pulled base image's layers, so //deploy needs +# neither dpkg nor ar on the host. +bazel_dep(name = "tar.bzl", version = "0.10.9") -# bsdtar, from the same toolchain rules_oci uses, reads .deb files (ar archives holding zstd or xz -# tarballs) and the pulled base image's layers, so //deploy needs neither dpkg nor ar on the host. -bazel_lib_toolchains = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "toolchains") -use_repo(bazel_lib_toolchains, "bsd_tar_toolchains") +tar = use_extension("@tar.bzl//tar:extensions.bzl", "toolchains") +use_repo(tar, "bsd_tar_toolchains") # The Ubuntu packages the image adds to the CUDA base (curl, tini and their dependencies), each # pinned by snapshot URL and sha256 in deploy/debs.lock.json. Re-resolve with @@ -36,11 +47,12 @@ debs.lock( ) use_repo(debs, "reactantserver_debs") -# The Julia toolchain, the clean-depot image layer and the podman loader. +# The Julia toolchain, and the image rules (julia/image.bzl): the distribution, depot and +# precompile-cache layers, the image environment, and the precompile test. bazel_dep(name = "rules_julia_depot", version = "0.1.0") git_override( module_name = "rules_julia_depot", - commit = "49bc678d037fa967920edccc380a6721a20050ee", + commit = "cead3479d64e7bf56fac66a15b72e9c4cd45ddc8", remote = "https://github.com/csvance/rules_julia_depot.git", ) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 8d0e0e73..7d4e3fce 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -22,9 +22,8 @@ "https://bcr.bazel.build/modules/apple_support/1.24.2/source.json": "2c22c9827093250406c5568da6c54e6fdf0ef06238def3d99c71b12feb057a8d", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/MODULE.bazel": "253d739ba126f62a5767d832765b12b59e9f8d2bc88cc1572f4a73e46eb298ca", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.22.5/MODULE.bazel": "004ba890363d05372a97248c37205ae64b6fa31047629cd2c0895a9d0c7779e8", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.22.5/source.json": "ac2c3213df8f985785f1d0aeb7f0f73d5324e6e67d593d9b9470fb74a25d4a9b", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.21.1/MODULE.bazel": "07e3ce3eaaa50dbd0be7fa0094e36890478937adc780ec53e77fd9fe543af8b1", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.21.1/source.json": "cb7d22ce044efa47c6e251107a35b8a919f5cd35254190d825adff1b7ae21e6e", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "812d2dd42f65dca362152101fbec418029cc8fd34cbad1a2fde905383d705838", "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", @@ -40,6 +39,7 @@ "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", + "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", "https://bcr.bazel.build/modules/bazel_features/1.42.1/MODULE.bazel": "275a59b5406ff18c01739860aa70ad7ccb3cfb474579411decca11c93b951080", "https://bcr.bazel.build/modules/bazel_features/1.42.1/source.json": "fcd4396b2df85f64f2b3bb436ad870793ecf39180f1d796f913cc9276d355309", @@ -64,7 +64,8 @@ "https://bcr.bazel.build/modules/buildozer/8.5.1/MODULE.bazel": "a35d9561b3fc5b18797c330793e99e3b834a473d5fbd3d7d7634aafc9bdb6f8f", "https://bcr.bazel.build/modules/buildozer/8.5.1/source.json": "e3386e6ff4529f2442800dee47ad28d3e6487f36a1f75ae39ae56c70f0cd2fbd", "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/MODULE.bazel": "cdf8cbe5ee750db04b78878c9633cc76e80dcf4416cbe982ac3a9222f80713c8", - "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/source.json": "fa7b512dfcb5eafd90ce3959cf42a2a6fe96144ebbb4b3b3928054895f2afac2", + "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.3/MODULE.bazel": "f1b7bb2dd53e8f2ef984b39485ec8a44e9076dda5c4b8efd2fb4c6a6e856a31d", + "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.3/source.json": "ebe931bfe362e4b41e59ee00a528db6074157ff2ced92eb9e970acab2e1089c9", "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", @@ -81,7 +82,8 @@ "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/source.json": "f448c6e8963fdfa7eb831457df83ad63d3d6355018f6574fb017e8169deb43a9", "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", - "https://bcr.bazel.build/modules/package_metadata/0.0.2/source.json": "e53a759a72488d2c0576f57491ef2da0cf4aab05ac0997314012495935531b73", + "https://bcr.bazel.build/modules/package_metadata/0.0.6/MODULE.bazel": "341dab6f417197494517d54c8e557c0baee1de7aec83543a4fbefe57900acb7e", + "https://bcr.bazel.build/modules/package_metadata/0.0.6/source.json": "9581d8b22db43550ac75ecc314ee4fa0a33400bfdc77d1317d8af6b18dca7756", "https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5", "https://bcr.bazel.build/modules/platforms/0.0.11/MODULE.bazel": "0daefc49732e227caa8bfa834d65dc52e8cc18a2faf80df25e8caea151a9413f", "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", @@ -130,6 +132,7 @@ "https://bcr.bazel.build/modules/rules_cc/0.2.13/MODULE.bazel": "eecdd666eda6be16a8d9dc15e44b5c75133405e820f620a234acc4b1fdc5aa37", "https://bcr.bazel.build/modules/rules_cc/0.2.17/MODULE.bazel": "1849602c86cb60da8613d2de887f9566a6d354a6df6d7009f9d04a14402f9a84", "https://bcr.bazel.build/modules/rules_cc/0.2.17/source.json": "3832f45d145354049137c0090df04629d9c2b5493dc5c2bf46f1834040133a07", + "https://bcr.bazel.build/modules/rules_cc/0.2.4/MODULE.bazel": "1ff1223dfd24f3ecf8f028446d4a27608aa43c3f41e346d22838a4223980b8cc", "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8", @@ -164,8 +167,6 @@ "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d", "https://bcr.bazel.build/modules/rules_license/1.0.0/MODULE.bazel": "a7fda60eefdf3d8c827262ba499957e4df06f659330bbe6cdbdb975b768bb65c", "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb", - "https://bcr.bazel.build/modules/rules_oci/2.2.6/MODULE.bazel": "2ba6ddd679269e00aeffe9ca04faa2d0ca4129650982c9246d0d459fe2da47d9", - "https://bcr.bazel.build/modules/rules_oci/2.2.6/source.json": "94e7decb8f95d9465b0bbea71c65064cd16083be1350c7468f131818641dc4a5", "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", "https://bcr.bazel.build/modules/rules_pkg/1.0.1/MODULE.bazel": "5b1df97dbc29623bccdf2b0dcd0f5cb08e2f2c9050aab1092fd39a41e82686ff", "https://bcr.bazel.build/modules/rules_pkg/1.0.1/source.json": "bd82e5d7b9ce2d31e380dd9f50c111d678c3bdaca190cb76b0e1c71b05e1ba8a", @@ -191,8 +192,8 @@ "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", "https://bcr.bazel.build/modules/rules_shell/0.6.1/MODULE.bazel": "72e76b0eea4e81611ef5452aa82b3da34caca0c8b7b5c0c9584338aa93bae26b", - "https://bcr.bazel.build/modules/rules_shell/0.7.1/MODULE.bazel": "257dd8d667371de804918dfceb86f6ddd2e1b5e025f5d9322878d4a73dc8aa58", - "https://bcr.bazel.build/modules/rules_shell/0.7.1/source.json": "9e6c3ea5766b584f60a44dd36520c4be16a12bb173aad238e018f73083e8feb6", + "https://bcr.bazel.build/modules/rules_shell/0.8.0/MODULE.bazel": "f6a89f1d6a669a26f28fe814503857055d76306b79cfc11d12399af08d0b80ae", + "https://bcr.bazel.build/modules/rules_shell/0.8.0/source.json": "eb53cc815bc503c6683c5fe12d943f98883f81fc22f51403ec8a95610cba4195", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", @@ -200,7 +201,6 @@ "https://bcr.bazel.build/modules/rules_swift/3.1.2/source.json": "e85761f3098a6faf40b8187695e3de6d97944e98abd0d8ce579cb2daf6319a66", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", - "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", "https://bcr.bazel.build/modules/stardoc/0.5.6/MODULE.bazel": "c43dabc564990eeab55e25ed61c07a1aadafe9ece96a4efabb3f8bf9063b71ef", "https://bcr.bazel.build/modules/stardoc/0.6.2/MODULE.bazel": "7060193196395f5dd668eda046ccbeacebfd98efc77fed418dbe2b82ffaa39fd", "https://bcr.bazel.build/modules/stardoc/0.7.0/MODULE.bazel": "05e3d6d30c099b6770e97da986c53bd31844d7f13d41412480ea265ac9e8079c", @@ -210,9 +210,11 @@ "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "5e463fbfba7b1701d957555ed45097d7f984211330106ccd1352c6e0af0dcf91", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/MODULE.bazel": "75aab2373a4bbe2a1260b9bf2a1ebbdbf872d3bd36f80bff058dccd82e89422f", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/source.json": "5fba48bbe0ba48761f9e9f75f92876cafb5d07c0ce059cc7a8027416de94a05b", + "https://bcr.bazel.build/modules/tar.bzl/0.10.9/MODULE.bazel": "4636f5f6a7c34d3d0653634ed95a34560426d548c5301c6c5924c885ef737b46", + "https://bcr.bazel.build/modules/tar.bzl/0.10.9/source.json": "30a5cf0a8281adce53ab1c5a4291e88b2c161e2df1feb6cba4ee50556f1748d7", "https://bcr.bazel.build/modules/tar.bzl/0.2.1/MODULE.bazel": "52d1c00a80a8cc67acbd01649e83d8dd6a9dc426a6c0b754a04fe8c219c76468", "https://bcr.bazel.build/modules/tar.bzl/0.5.1/MODULE.bazel": "7c2eb3dcfc53b0f3d6f9acdfd911ca803eaf92aadf54f8ca6e4c1f3aee288351", - "https://bcr.bazel.build/modules/tar.bzl/0.5.1/source.json": "deed3094f7cc779ed1d37a68403847b0e38d9dd9d931e03cb90825f3368b515f", + "https://bcr.bazel.build/modules/tar.bzl/0.7.0/MODULE.bazel": "cc1acd85da33c80e430b65219a620d54d114628df24a618c3a5fa0b65e988da9", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/yq.bzl/0.1.1/MODULE.bazel": "9039681f9bcb8958ee2c87ffc74bdafba9f4369096a2b5634b88abc0eaefa072", "https://bcr.bazel.build/modules/yq.bzl/0.1.1/source.json": "2d2bad780a9f2b9195a4a370314d2c17ae95eaa745cefc2e12fbc49759b15aa3", @@ -223,6 +225,28 @@ }, "selectedYankedVersions": {}, "moduleExtensions": { + "@@pybind11_bazel+//:internal_configure.bzl%internal_configure_extension": { + "general": { + "bzlTransitiveDigest": "NRXra7941UfmNUyIxnLt82V5hULluVGL2nBsijTl4j4=", + "usagesDigest": "D1r3lfzMuUBFxgG8V6o0bQTLMk3GkaGOaPzw53wrwyw=", + "recordedInputs": [ + "REPO_MAPPING:pybind11_bazel+,bazel_tools bazel_tools", + "FILE:@@pybind11_bazel+//MODULE.bazel e6f4c20442eaa7c90d7190d8dc539d0ab422f95c65a57cc59562170c58ae3d34" + ], + "generatedRepoSpecs": { + "pybind11": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "build_file": "@@pybind11_bazel+//:pybind11-BUILD.bazel", + "strip_prefix": "pybind11-2.12.0", + "urls": [ + "https://github.com/pybind/pybind11/archive/v2.12.0.zip" + ] + } + } + } + } + }, "@@rules_julia_depot+//julia:extensions.bzl%julia": { "general": { "bzlTransitiveDigest": "OsJA58rY4YQkeDkx4AoHFcLUMQpp9niR9zYBH9RsVzU=", @@ -302,172 +326,6 @@ } } }, - "@@rules_oci+//oci:extensions.bzl%oci": { - "general": { - "bzlTransitiveDigest": "TO0sjSNkDsRNtKiwgPJW4ogKM/yK77iip7GpQtVYt44=", - "usagesDigest": "DQiiJQoJaA2MzN61knQBry/pa4YyOviydpfVI5wg2jM=", - "recordedInputs": [ - "REPO_MAPPING:aspect_bazel_lib+,bazel_tools bazel_tools", - "REPO_MAPPING:bazel_features+,bazel_tools bazel_tools", - "REPO_MAPPING:rules_oci+,aspect_bazel_lib aspect_bazel_lib+", - "REPO_MAPPING:rules_oci+,bazel_features bazel_features+", - "REPO_MAPPING:rules_oci+,bazel_skylib bazel_skylib+" - ], - "generatedRepoSpecs": { - "reactantserver_cuda_base_linux_amd64": { - "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_pull", - "attributes": { - "www_authenticate_challenges": {}, - "scheme": "https", - "registry": "index.docker.io", - "repository": "nvidia/cuda", - "identifier": "sha256:08d6facb6a72f918beb3ffe5b57cff72402a203f44428831262e95c7466d4188", - "platform": "linux/amd64", - "target_name": "reactantserver_cuda_base_linux_amd64", - "bazel_tags": [] - } - }, - "reactantserver_cuda_base": { - "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_alias", - "attributes": { - "target_name": "reactantserver_cuda_base", - "www_authenticate_challenges": {}, - "scheme": "https", - "registry": "index.docker.io", - "repository": "nvidia/cuda", - "identifier": "sha256:08d6facb6a72f918beb3ffe5b57cff72402a203f44428831262e95c7466d4188", - "platforms": { - "@@platforms//cpu:x86_64": "@reactantserver_cuda_base_linux_amd64" - }, - "bzlmod_repository": "reactantserver_cuda_base", - "reproducible": true - } - }, - "oci_crane_darwin_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "darwin_amd64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_darwin_arm64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "darwin_arm64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_arm64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_arm64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_armv6": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_armv6", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_i386": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_i386", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_s390x": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_s390x", - "crane_version": "v0.18.0" - } - }, - "oci_crane_linux_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "linux_amd64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_windows_armv6": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "windows_armv6", - "crane_version": "v0.18.0" - } - }, - "oci_crane_windows_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", - "attributes": { - "platform": "windows_amd64", - "crane_version": "v0.18.0" - } - }, - "oci_crane_toolchains": { - "repoRuleId": "@@rules_oci+//oci/private:toolchains_repo.bzl%toolchains_repo", - "attributes": { - "toolchain_type": "@rules_oci//oci:crane_toolchain_type", - "toolchain": "@oci_crane_{platform}//:crane_toolchain" - } - }, - "oci_regctl_darwin_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "darwin_amd64" - } - }, - "oci_regctl_darwin_arm64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "darwin_arm64" - } - }, - "oci_regctl_linux_arm64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "linux_arm64" - } - }, - "oci_regctl_linux_s390x": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "linux_s390x" - } - }, - "oci_regctl_linux_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "linux_amd64" - } - }, - "oci_regctl_windows_amd64": { - "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", - "attributes": { - "platform": "windows_amd64" - } - }, - "oci_regctl_toolchains": { - "repoRuleId": "@@rules_oci+//oci/private:toolchains_repo.bzl%toolchains_repo", - "attributes": { - "toolchain_type": "@rules_oci//oci:regctl_toolchain_type", - "toolchain": "@oci_regctl_{platform}//:regctl_toolchain" - } - } - }, - "moduleExtensionMetadata": { - "explicitRootModuleDirectDeps": [ - "reactantserver_cuda_base", - "reactantserver_cuda_base_linux_amd64" - ], - "explicitRootModuleDirectDevDeps": [], - "useAllRepos": "NO", - "reproducible": false - } - } - }, "@@rules_python+//python/extensions:config.bzl%config": { "general": { "bzlTransitiveDigest": "dzD8Q2YmrP3fz8saWLHPmlwPLO91ImtTmP/c9JKTStM=", @@ -665,57 +523,6 @@ } } } - }, - "@@tar.bzl+//tar:extensions.bzl%toolchains": { - "general": { - "bzlTransitiveDigest": "/2afh6fPjq/rcyE/jztQDK3ierehmFFngfvmqyRv72M=", - "usagesDigest": "maF8qsAIqeH1ey8pxP0gNZbvJt34kLZvTFeQ0ntrJVA=", - "recordedInputs": [], - "generatedRepoSpecs": { - "bsd_tar_toolchains": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:toolchain.bzl%tar_toolchains_repo", - "attributes": { - "user_repository_name": "bsd_tar_toolchains" - } - }, - "bsd_tar_toolchains_darwin_amd64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "bsd_tar_toolchains_darwin_arm64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "bsd_tar_toolchains_linux_amd64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "bsd_tar_toolchains_linux_arm64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "bsd_tar_toolchains_windows_amd64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "windows_amd64" - } - }, - "bsd_tar_toolchains_windows_arm64": { - "repoRuleId": "@@tar.bzl+//tar/toolchain:platforms.bzl%bsdtar_binary_repo", - "attributes": { - "platform": "windows_arm64" - } - } - } - } } }, "facts": {}, diff --git a/deploy/BUILD.bazel b/deploy/BUILD.bazel index 888efbd5..f95fe62c 100644 --- a/deploy/BUILD.bazel +++ b/deploy/BUILD.bazel @@ -1,4 +1,12 @@ -load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load") +load( + "@rules_julia_depot//julia:image.bzl", + "julia_compiled_layer", + "julia_depot_layer", + "julia_dist_layer", + "julia_image_env", + "julia_precompile_test", +) +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") load("@rules_shell//shell:sh_binary.bzl", "sh_binary") load("@rules_shell//shell:sh_test.bzl", "sh_test") @@ -9,18 +17,25 @@ package(default_visibility = ["//visibility:public"]) # The ReactantServer node image, built from the committed deploy/Manifest.toml. # # bazel build //deploy:image -# bazel run //deploy:image_load # loads it into podman as reactantserver:bazel -# bazel run //deploy:image_check # the loaded image starts without precompiling -# bazel test //deploy:manifest_current # the manifest still matches the workspace -# bazel run //deploy:relock # re-resolve deploy/Manifest.toml after a Project change -# bazel run //deploy:relock_debs # re-resolve deploy/debs.lock.json (curl, tini) +# bazel test //deploy:image_precompile_test # the image starts without precompiling (no podman) +# bazel run //deploy:image_load # loads it into podman as reactantserver:bazel +# bazel run //deploy:image_check # the loaded image: caches, tini, curl, entrypoints +# bazel run //deploy:image_push -- --repository index.docker.io// # pushes :latest +# bazel test //deploy:manifest_current # the manifest still matches the workspace +# bazel run //deploy:relock # re-resolve deploy/Manifest.toml after a Project change +# bazel run //deploy:relock_debs # re-resolve deploy/debs.lock.json (curl, tini) +# +# The Julia-specific layers (the distribution, the depot, the precompile caches), the image's +# environment and the precompile test come from @rules_julia_depot//julia:image.bzl. What stays here +# is what is ours: the Ubuntu packages, the CUDA driver stub the build loads Reactant against, the +# application layer, and the image itself. # # WHY A SECOND MANIFEST. The workspace root's Manifest.toml is gitignored on purpose: development # and CI resolve fresh so compat bumps are picked up. An image must not: this manifest is the # production lock, moved deliberately with //deploy:relock and reviewed like any other change. # It is a manifest for the ROOT Project.toml, and it lives here only so the gitignore can stay as -# it is. Every action stages it beside the root project and the member Project.tomls in a -# temporary tree, because Pkg resolves a manifest next to its (real, symlink-resolved) project. +# it is. The depot layer stages it beside the root project and the member Project.tomls, because +# Pkg resolves a manifest next to its (real, symlink-resolved) project. # CUDA userspace the Reactant_jll artifact is selected for. Must agree with the base image # (@reactantserver_cuda_base, MODULE.bazel): Reactant_jll's platform augmentation reads these at @@ -31,21 +46,14 @@ _REACTANT_GPU_ENV = { "REACTANT_GPU_VERSION": "13.1", } -# Where the image keeps Julia's state, and the order Julia searches it. The workspace root is a -# depot too, and not by accident: Julia records a precompiled package's source files relative to -# the depot that holds them ("@depot/..."), which is what lets a cache built in a temporary tree -# load from /opt. Packages outside every depot are recorded by absolute path and would be stale in -# the image, so the workspace root is listed as a depot to make the workspace's own packages -# relocatable like the registry ones. Julia's bundled depots follow, for the stdlib caches. -# //deploy:compiled_layer builds against exactly this list, with the build tree in place of /opt. -_DEPOT = "/opt/julia-depot" +# Where the image keeps the application. Julia and the depot take image.bzl's defaults, +# /opt/julia and /opt/julia-depot. _APP = "/opt/reactantserver" -_JULIA = "/opt/julia" -_DEPOT_PATH = ":".join([_DEPOT, _APP, _JULIA + "/local/share/julia", _JULIA + "/share/julia"]) -# The projects the image starts Julia in, each precompiled by //deploy:compiled_layer and checked -# by //deploy:image_check. A package's cache depends on the active project's preferences (the -# worker's LocalPreferences.toml configures Reactant), so each entry point needs its own. +# The projects the image starts Julia in, relative to _APP, each precompiled by +# //deploy:compiled_layer and checked by //deploy:image_precompile_test. A package's cache depends +# on the active project's preferences (the worker's LocalPreferences.toml configures Reactant), so +# each entry point needs its own. # packages/ReactantServerNode the supervisor (docker/entrypoint.node.sh) # packages/ReactantServer a worker (spawned by the supervisor, or entrypoint.worker.sh) # packages/ReactantServerGateway the embedded gateway (spawned by the supervisor) @@ -57,11 +65,18 @@ _ENTRY_PROJECTS = [ ".", ] -# The CPU targets the caches are compiled for: the official Julia build's x86_64 set, so the caches -# load on any x86_64 host rather than only on CPUs like the build machine's. Julia rejects a cache -# built for a CPU it is not running on and recompiles, which is the first-start cost this layer -# exists to remove. -_CPU_TARGET = "generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1)" +_ENTRY_PROJECT_PATHS = [_APP if p == "." else _APP + "/" + p for p in _ENTRY_PROJECTS] + +# Where the CUDA driver stub sits in the build-time tree. Never in the image; see :cuda_stub_layer. +_CUDA_STUB_DIR = "/build/cuda-stub" + +# What Julia needs to load the stack on a build host: the driver stub in place of libcuda.so.1, the +# host's GPUs hidden so a build never touches them, and the artifact selection. `{root}` is the +# tree the layers are unpacked into (see image.bzl). +_BUILD_LOAD_ENV = _REACTANT_GPU_ENV | { + "CUDA_VISIBLE_DEVICES": "", + "LD_LIBRARY_PATH": "{root}" + _CUDA_STUB_DIR, +} _STAGE = """ workspace_path() { @@ -87,9 +102,10 @@ stage_workspace() { } """ -# The layers write deterministic tars: sorted, epoch mtimes, root-owned by number. Without these a -# layer's digest changes on every build and nothing downstream can be cached or compared. -_TAR = "tar --create --file \"$@\" --sort=name --mtime=@0 --owner=0 --group=0 --numeric-owner" +# The layers written here are deterministic tars in the same form as image.bzl's: sorted, epoch +# mtimes, root-owned by number, permissions normalised. Without these a layer's digest changes on +# every build and nothing downstream can be cached or compared. +_TAR = "LC_ALL=C tar --create --file \"$@\" --format=gnu --sort=name --mtime=@0 --owner=0 --group=0 --numeric-owner --mode='u=rwX,go=rX'" # The Ubuntu packages the image adds to the base (curl, tini and the libraries curl needs), from # the pinned downloads in deploy/debs.lock.json, unpacked the way dpkg would lay them out. @@ -134,16 +150,18 @@ done ) # The CUDA driver stub, libcuda.so, taken from the pinned CUDA base itself so it always matches the -# base's CUDA release. Build time only: the CUDA build of libReactantExtra.so lists libcuda.so.1 as -# NEEDED, so loading Reactant (which precompiling does) fails on a host without an NVIDIA driver, -# and would load the host's real driver on one that has it. The stub answers every call with "no -# device" and never reaches the image; a container gets the real driver from its host. +# base's CUDA release, as a layer at _CUDA_STUB_DIR. BUILD TIME ONLY: it is among the layers of +# :compiled_layer and :image_precompile_test and never among the image's. The CUDA build of +# libReactantExtra.so lists libcuda.so.1 as NEEDED, so loading Reactant (which precompiling does) +# fails on a host without an NVIDIA driver, and would load the host's real driver on one that has +# it. The stub answers every call with "no device"; a container gets the real driver from its host. # -# The layers are searched in manifest order and the last copy wins, as an overlay would resolve it. +# The base's layers are searched in manifest order and the last copy wins, as an overlay would +# resolve it. genrule( - name = "cuda_stub", + name = "cuda_stub_layer", srcs = ["@reactantserver_cuda_base_linux_amd64"], - outs = ["cuda_stub/libcuda.so.1"], + outs = ["reactantserver-cuda-stub.tar"], cmd = """ set -euo pipefail layout=$(location @reactantserver_cuda_base_linux_amd64) @@ -151,128 +169,79 @@ blobs="$$layout/blobs/sha256" member=usr/local/cuda-{gpu_version}/targets/x86_64-linux/lib/stubs/libcuda.so manifest="$$(grep -o 'sha256:[0-9a-f]*' "$$layout/index.json" | head -1 | cut -d: -f2)" found="$$(mktemp -d)" -trap 'rm -rf "$$found"' EXIT +stage="$$(mktemp -d)" +trap 'rm -rf "$$found" "$$stage"' EXIT for layer in $$(tr ',' '\\n' < "$$blobs/$$manifest" | sed -n '/"layers"/,$$p' | grep -o 'sha256:[0-9a-f]*' | cut -d: -f2); do $(BSDTAR_BIN) -xf "$$blobs/$$layer" -C "$$found" "$$member" 2>/dev/null || true done [ -f "$$found/$$member" ] || {{ echo "FAILED: $$member is not in the base image" >&2; exit 1; }} -cp "$$found/$$member" "$@" -""".format(gpu_version = _REACTANT_GPU_ENV["REACTANT_GPU_VERSION"]), +mkdir -p "$$stage{stub_dir}" +cp "$$found/$$member" "$$stage{stub_dir}/libcuda.so.1" +""".format( + gpu_version = _REACTANT_GPU_ENV["REACTANT_GPU_VERSION"], + stub_dir = _CUDA_STUB_DIR, + ) + _TAR + """ --directory "$$stage" {top} +""".format(top = _CUDA_STUB_DIR.split("/")[1]), toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], tools = ["@bsd_tar_toolchains//:resolved_toolchain"], ) -# The depot, instantiated from the manifest into an empty tree and tarred at /opt/julia-depot. +# The image's Julia layout and environment, declared once: the oci_image takes the file this +# writes, and the compiled layer and the precompile test take the same facts, so the caches are +# built for, and checked against, exactly the environment the image runs with. # -# FULL, not artifacts-only: the image loads its packages from source (with the caches of -# //deploy:compiled_layer), not from a sysimage, so the packages themselves are the code. +# The workspace root is a depot too (extra_depots), and not by accident: Julia records a +# precompiled package's source files relative to the depot that holds them ("@depot/..."), which is +# what lets a cache built in a temporary tree load from /opt. Packages outside every depot are +# recorded by absolute path and would be stale in the image, so the workspace root is listed to make +# the workspace's own packages relocatable like the registry ones. The search order comes out as # -# The source depot handed to image_depot.sh is EMPTY, so the General registry is fetched into the -# clean depot rather than copied from whatever this host has; the manifest pins every package by -# tree hash, so the registry's state cannot change what is installed. Set JULIA_PKG_SERVER in -# user.bazelrc (--action_env) to go through a mirror. -genrule( - name = "depot_layer", - srcs = [ - "Manifest.toml", - "//:workspace_projects", - "@reactantserver_julia//:dist", - "@reactantserver_julia//:bin/julia", - ], - outs = ["reactantserver-depot.tar"], - cmd = _STAGE + """ -set -euo pipefail -export JULIA_BIN="$$(cd "$$(dirname $(location @reactantserver_julia//:bin/julia))" && pwd)/julia" -stage="$$(mktemp -d)" -src_depot="$$(mktemp -d)" -trap 'rm -rf "$$stage" "$$src_depot"' EXIT -stage_workspace "$$stage" -export JULIA_DEPOT_PATH="$$src_depot" -export REACTANT_GPU={gpu} REACTANT_GPU_VERSION={gpu_version} -JULIA_DEPOT_CONTENTS=full \\ - $(location @rules_julia_depot//julia:image_depot.sh) "$$stage" "$@" -""".format( - gpu = _REACTANT_GPU_ENV["REACTANT_GPU"], - gpu_version = _REACTANT_GPU_ENV["REACTANT_GPU_VERSION"], - ), - tags = ["requires-network"], # packages, artifacts and the registry from the package server - tools = ["@rules_julia_depot//julia:image_depot.sh"], -) - -# The precompile caches, so a container starts serving without first compiling ~150 packages. +# /opt/julia-depot:/opt/reactantserver:/opt/julia/local/share/julia:/opt/julia/share/julia # -# Built from the depot and application layers themselves, unpacked into one tree with the image's -# layout, so the caches are for exactly the files the image ships. Julia validates a cache against -# its sources by content, and records them relative to their depot (see _DEPOT_PATH), so caches -# compiled here under /opt load unchanged from /opt. The CUDA stub stands in for the driver -# while Reactant loads; GPUs are hidden so the build never touches the host's. +# The caches are compiled for the official Julia build's x86_64 CPU targets (image.bzl's default), +# so they load on any x86_64 host rather than only on CPUs like the build machine's. # -# Precompiling is also the first time the whole stack is loaded, so a package that fails to load -# fails the build here rather than a container at start. -genrule( - name = "compiled_layer", - srcs = [ - ":app_layer", - ":cuda_stub", - ":depot_layer", - "@reactantserver_julia//:bin/julia", - "@reactantserver_julia//:dist", - ], - outs = ["reactantserver-compiled.tar"], - cmd = """ -set -euo pipefail -julia="$$(readlink -f $(location @reactantserver_julia//:bin/julia))" -julia_prefix="$$(cd "$$(dirname "$$julia")/.." && pwd)" -root="$$(mktemp -d)" -stub="$$(mktemp -d)" -# Pkg leaves installed packages read-only, which rm -rf alone cannot delete. -trap 'chmod -R u+w "$$root"; rm -rf "$$root" "$$stub"' EXIT -tar --extract --file $(location :depot_layer) --directory "$$root" --no-same-owner -tar --extract --file $(location :app_layer) --directory "$$root" --no-same-owner -cp $(location :cuda_stub) "$$stub/libcuda.so.1" -# _DEPOT_PATH, with the build tree in place of /opt and the toolchain's Julia in place of /opt/julia. -export JULIA_DEPOT_PATH="$$root{depot}:$$root{app}:$$julia_prefix/local/share/julia:$$julia_prefix/share/julia" -export JULIA_CPU_TARGET='{cpu_target}' -export JULIA_PKG_OFFLINE=true -export CUDA_VISIBLE_DEVICES= -export LD_LIBRARY_PATH="$$stub" -export REACTANT_GPU={gpu} REACTANT_GPU_VERSION={gpu_version} -for p in {projects}; do - echo "==> precompiling $$p" - "$$julia" --startup-file=no --project="$$root{app}/$$p" -e 'using Pkg; Pkg.precompile(; strict = true)' -done -""".format( - app = _APP, - cpu_target = _CPU_TARGET, - depot = _DEPOT, - gpu = _REACTANT_GPU_ENV["REACTANT_GPU"], - gpu_version = _REACTANT_GPU_ENV["REACTANT_GPU_VERSION"], - projects = " ".join(_ENTRY_PROJECTS), - ) + _TAR + """ --directory "$$root" {compiled} -""".format(compiled = _DEPOT.lstrip("/") + "/compiled"), +# PATH becomes /opt/julia/bin:$PATH, which rules_oci expands against the CUDA base's own PATH. +# JULIA_PKG_OFFLINE is set too: nothing in the image resolves or installs, and a stale cache (a +# mounted, edited package) still precompiles into the depot offline. +julia_image_env( + name = "image_env", + env = _REACTANT_GPU_ENV | { + "REACTANT_WORKSPACE_ROOT": _APP, + "JULIA_SSL_CA_ROOTS_PATH": "/etc/ssl/certs/ca-certificates.crt", + # utility as well as compute: the entrypoint's GPU-reclaim gate runs nvidia-smi. + "NVIDIA_DRIVER_CAPABILITIES": "compute,utility", + "NVIDIA_VISIBLE_DEVICES": "all", + }, + extra_depots = [_APP], + project = _APP, ) -# Julia as a layer at /opt/julia rather than as the base, because the base has to be CUDA's. -genrule( +# Julia at /opt/julia rather than as the base, because the base has to be CUDA's. The +# distribution's own relative symlinks are kept, so its libraries are not shipped twice. +julia_dist_layer( name = "julia_layer", - srcs = [ - "@reactantserver_julia//:dist", - "@reactantserver_julia//:bin/julia", - ], - outs = ["reactantserver-julia.tar"], - cmd = """ -set -euo pipefail -root="$$(cd "$$(dirname $(location @reactantserver_julia//:bin/julia))/.." && pwd)" -stage="$$(mktemp -d)" -trap 'rm -rf "$$stage"' EXIT -mkdir -p "$$stage/opt" -# -L, NOT plain -a: an external repo is a tree of symlinks into the execroot, and copying the -# links would ship /opt/julia/bin/julia pointing at a path that exists only on the build host. -cp -aL "$$root" "$$stage/opt/julia" -chmod -R u+w "$$stage/opt/julia" -rm -f "$$stage/opt/julia/BUILD.bazel" "$$stage/opt/julia/WORKSPACE" "$$stage/opt/julia/REPO.bazel" -""" + _TAR + """ --directory "$$stage" opt -""", + julia = "@reactantserver_julia//:dist", +) + +# The depot, instantiated from the manifest into an empty tree at /opt/julia-depot. +# +# FULL, not artifacts-only: the image loads its packages from source (with the caches of +# //deploy:compiled_layer), not from a sysimage, so the packages themselves are the code. +# +# No `depot`, so the General registry is fetched into the clean depot rather than copied from +# whatever this host has; the manifest pins every package by tree hash, so the registry's state +# cannot change what is installed. Set JULIA_PKG_SERVER in user.bazelrc (--action_env) to go +# through a mirror. +julia_depot_layer( + name = "depot_layer", + contents = "full", + env = _REACTANT_GPU_ENV, + julia = "@reactantserver_julia//:dist", + manifest = "Manifest.toml", + project = "//:Project.toml", + # The member Project.tomls, which the workspace's [workspace] table needs to instantiate. + srcs = ["//:workspace_projects"], ) # The application at /opt/reactantserver, in the workspace layout the entrypoints expect: the root @@ -307,11 +276,43 @@ for f in entrypoint.node.sh entrypoint.worker.sh healthcheck.node.sh healthcheck ln -s "{app}/docker/$$f" "$$stage/usr/local/bin/$$f" done cp "$$app/config/node.default.yaml" "$$stage/etc/reactantserver/node.yaml" -chmod -R a+rX "$$stage/opt" "$$stage/etc" """.format(app = _APP) + _TAR + """ --directory "$$stage" etc opt usr """, ) +# The precompile caches, so a container starts serving without first compiling ~150 packages. +# +# Built from the image's own Julia, depot and application layers, unpacked into one tree with the +# image's layout, so the caches are for exactly the files the image ships, and compiled for the +# depot path and CPU targets of :image_env, so they load unchanged from /opt. The driver stub layer +# rides along for the build only. +# +# Precompiling is also the first time the whole stack is loaded, so a package that fails to load +# fails the build here rather than a container at start. +julia_compiled_layer( + name = "compiled_layer", + env = _BUILD_LOAD_ENV, + image_env = ":image_env", + julia = "@reactantserver_julia//:dist", + layers = [ + ":julia_layer", + ":depot_layer", + ":app_layer", + ":cuda_stub_layer", + ], + projects = _ENTRY_PROJECT_PATHS, +) + +# The layers in the order they change, least often first, so a push or a pull after a code change +# moves only the application layer (and the caches, when a package's sources change). +_IMAGE_LAYERS = [ + ":debs_layer", + ":julia_layer", + ":depot_layer", + ":compiled_layer", + ":app_layer", +] + # Root, and not by oversight: the entrypoint's GPU-reclaim gate and the supervisor expect it, and a # stale cache (a mounted, edited package) must still be able to precompile into the depot. oci_image( @@ -325,43 +326,53 @@ oci_image( "--", "/usr/local/bin/entrypoint.node.sh", ], - env = _REACTANT_GPU_ENV | { - # Julia is a layer, so nothing else puts it on PATH; the entrypoints exec a bare `julia`. - # The rest is the CUDA base's own PATH, repeated because this key replaces it. - "PATH": _JULIA + "/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", - "JULIA_DEPOT_PATH": _DEPOT_PATH, - "JULIA_PROJECT": _APP, - "REACTANT_WORKSPACE_ROOT": _APP, - "JULIA_SSL_CA_ROOTS_PATH": "/etc/ssl/certs/ca-certificates.crt", - # utility as well as compute: the entrypoint's GPU-reclaim gate runs nvidia-smi. - "NVIDIA_DRIVER_CAPABILITIES": "compute,utility", - "NVIDIA_VISIBLE_DEVICES": "all", - }, + env = ":image_env", # The embedded gateway's gRPC and metrics/health ports. The workers role binds per-worker ports # (base_port+i, metrics_base_port+i) instead; publish the range you configured. exposed_ports = [ "8001/tcp", "8002/tcp", ], - tars = [ - ":debs_layer", - ":depot_layer", - ":compiled_layer", - ":julia_layer", - ":app_layer", - ], + tars = _IMAGE_LAYERS, workdir = _APP, ) +# Every entry project loads from the baked caches with nothing precompiled and no cache rejected, +# checked on the image's layers without a container, so it needs no podman. Manual because it +# builds the whole image; //deploy:image_check covers what only a container shows. +julia_precompile_test( + name = "image_precompile_test", + size = "large", + env = _BUILD_LOAD_ENV, + image_env = ":image_env", + julia = "@reactantserver_julia//:dist", + layers = [l for l in _IMAGE_LAYERS if l != ":debs_layer"] + [":cuda_stub_layer"], + projects = _ENTRY_PROJECT_PATHS, + tags = ["manual"], +) + oci_load( name = "image_load", image = ":image", # Explicit, because rules_oci probes for docker first, and on hosts where a docker CLI is # installed but its socket is unreachable that probe wins and fails. - loader = "@rules_julia_depot//oci:podman.sh", + loader = "podman.sh", repo_tags = ["reactantserver:bazel"], ) +# Push the image as :latest. The repository is given at run time, so this file names no registry +# account; credentials come from the docker config (`docker login`, `podman login --authfile +# ~/.docker/config.json`, or docker/login-action in CI): +# +# bazel run //deploy:image_push -- --repository index.docker.io// +# +# .github/workflows/image.yml runs exactly this. +oci_push( + name = "image_push", + image = ":image", + remote_tags = ["latest"], +) + # Check a loaded image: every entry project loads from the baked caches without precompiling, and # the tools the entrypoint and healthcheck call are present. Needs podman, and the image loaded # first (bazel run //deploy:image_load); runs on a host with or without a GPU. diff --git a/deploy/README.md b/deploy/README.md index 0f12ebe1..776bcaaa 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -9,8 +9,10 @@ Julia. ``` bazel build //deploy:image +bazel test //deploy:image_precompile_test # the image starts without precompiling (no podman) bazel run //deploy:image_load # loads it into podman as localhost/reactantserver:bazel -bazel run //deploy:image_check # the loaded image starts without precompiling +bazel run //deploy:image_check # the loaded image: caches, tini, curl, entrypoints +bazel run //deploy:image_push -- --repository index.docker.io// # push as :latest bazel test //deploy:manifest_current # the Julia lock still matches the workspace Project.toml files bazel run //deploy:relock # re-resolve the Julia lock after a [deps] or [compat] change bazel run //deploy:relock_debs # re-resolve the Ubuntu package lock (curl, tini) @@ -18,6 +20,12 @@ bazel run //deploy:relock_debs # re-resolve the Ubuntu package lock (c Or `make bazel-image`, which builds and loads it. +The Julia-specific parts (the distribution, depot and precompile-cache layers, the image +environment, and the precompile test) are the image rules of +[rules_julia_depot](https://github.com/csvance/rules_julia_depot) (`julia/image.bzl`); this +package adds the Ubuntu packages, the CUDA driver stub the build loads Reactant against, the +application layer, and the image itself, assembled with rules_oci. + Julia, the CUDA base image, and every package are fetched and pinned by Bazel. The build host needs Bazel (the version in `.bazelversion`; Bazelisk picks it up), GNU tar, and a C compiler with `make`, because InterProcessCommunication.jl generates its constants file by compiling a small C program when @@ -65,7 +73,11 @@ against the base. ## Layers +In image order, least often changed first, so a push after a code change moves only the last +layers: + - `debs_layer`: `curl`, `tini`, and their libraries, from the Ubuntu lock. +- `julia_layer`: Julia at `/opt/julia`, its relative symlinks kept. - `depot_layer`: the Julia depot at `/opt/julia-depot`, instantiated from the Julia lock into an empty depot (the General registry is fetched fresh; the lock pins every package by tree hash, so the registry state cannot change what is installed). Set `JULIA_PKG_SERVER` through @@ -73,12 +85,15 @@ against the base. - `compiled_layer`: the precompile caches for every project the image starts Julia in (the supervisor, a worker, the gateway, and the worker-role healthcheck), so a container serves without first compiling about 150 packages. -- `julia_layer`: Julia at `/opt/julia`. - `app_layer`: the workspace at `/opt/reactantserver` (root project plus the Julia lock, member packages without their tests, `docker/`, `config/`), the entrypoints and healthchecks linked into `/usr/local/bin`, and the default node file at `/etc/reactantserver/node.yaml`, all where `docker/Dockerfile` puts them. +`image_env` writes the image's environment: the depot path below, `JULIA_PROJECT`, the portable +`JULIA_CPU_TARGET`, `JULIA_PKG_OFFLINE=true`, Julia's `bin/` ahead of the base's `PATH`, and the +CUDA and NVIDIA variables. + The base is `nvidia/cuda:13.1.2-cudnn-devel-ubuntu24.04`, pinned by digest. It is deliberately fat: Reactant_jll's CUDA 13.1 artifact links cuDNN and cuBLASLt statically but `dlopen`s `libnvJitLink.so.13`, `libnvrtc.so.13`, and `libcupti.so` by soname, and the NVIDIA container @@ -102,10 +117,13 @@ JULIA_DEPOT_PATH=/opt/julia-depot:/opt/reactantserver:/opt/julia/local/share/jul `compiled_layer` unpacks the depot and application layers into one temporary tree with the image's layout and precompiles against the same list with that tree in place of `/opt`, so its caches load unchanged in the image. The loader needs `libcuda.so.1` to load Reactant; the build takes the -driver stub from the CUDA base (`cuda_stub`) and hides the host's GPUs. The caches are compiled for +driver stub from the CUDA base as a build-time layer (`cuda_stub_layer`, unpacked beside the others +but never part of the image), puts it on `LD_LIBRARY_PATH`, and hides the host's GPUs. The caches are compiled for the official Julia build's x86_64 CPU targets, so they load on any x86_64 host, not only CPUs like -the build machine's. `//deploy:image_check` verifies that each entry project loads with no cache -rejected and nothing precompiled. +the build machine's. `//deploy:image_precompile_test` verifies, on the layers and without a +container, that each entry project loads with no cache rejected and nothing precompiled; +`//deploy:image_check` repeats that inside a loaded container and also checks `tini`, `curl`, and +the entrypoints. ## Running it @@ -132,6 +150,16 @@ podman run ... --health-cmd /usr/local/bin/healthcheck.node.sh --health-interval --health-timeout 20s --health-retries 3 --health-start-period 300s localhost/reactantserver:bazel ``` +## Publishing + +`.github/workflows/image.yml` builds the image, runs `//deploy:manifest_current` and +`//deploy:image_precompile_test`, and pushes it to Docker Hub as `:latest`. It runs only when +triggered by hand (Actions, Image, Run workflow). It reads the target from the +`DOCKERHUB_REPOSITORY` repository variable (for example `index.docker.io//`) and logs +in with the `DOCKERHUB_USERNAME` and `DOCKERHUB_TOKEN` secrets. Releases are not named, so each run +replaces `:latest`; the run summary records the pushed digest and commit, so pull by digest to pin +a deployment. + ## Consuming from another module The image and its layers are public. A deployment repository can take a `bazel_dep` on diff --git a/deploy/podman.sh b/deploy/podman.sh new file mode 100755 index 00000000..3ec9525d --- /dev/null +++ b/deploy/podman.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# The loader for //deploy:image_load. rules_oci probes `command -v docker` before podman, and on a +# host where a docker CLI is installed but its socket is unreachable that probe wins and the load +# fails, so the loader is named rather than discovered. +exec podman "$@"