diff --git a/.buildkite/_forge.rayci.yml b/.buildkite/_forge.rayci.yml index d5ad4ad8e44..0ad0c14c7c3 100644 --- a/.buildkite/_forge.rayci.yml +++ b/.buildkite/_forge.rayci.yml @@ -4,6 +4,18 @@ steps: - name: forge wanda: ci/docker/forge.wanda.yaml + # Bazel 8+ release binaries need glibc >= 2.25, but Ray builds its Linux + # wheels inside manylinux2014 (CentOS 7, glibc 2.17) so that they keep running + # on glibc 2.17 systems. This step compiles Bazel from source inside + # manylinux2014 and ships it with a JDK 21 that also runs on glibc 2.17 + # (ci/docker/bazel-glibc217.Dockerfile). Every image that runs Bazel inside + # manylinux2014 depends on it. Wanda caches it, so it only rebuilds when + # .bazelversion changes. + # TODO: upgrade the wheel build image from manylinux2014 to manylinux_2_28 + # (glibc 2.28, which raises the user glibc floor to 2.28). The official Bazel + # binary then runs there, so remove this step, the bazel-glibc217 depends_on + # edges and the bazel-glibc217 COPY blocks in the ray-core, ray-cpp-core, + # ray-java and manylinux-retag Dockerfiles. - name: bazel-glibc217-x86_64 label: "wanda: bazel for manylinux2014 (x86_64)" wanda: ci/docker/bazel-glibc217.wanda.yaml @@ -13,6 +25,7 @@ steps: - name: manylinux-x86_64 wanda: ci/docker/manylinux.wanda.yaml + # TODO: drop after the move to manylinux_2_28 (see _forge.rayci.yml). depends_on: bazel-glibc217-x86_64 env_file: rayci.env env: diff --git a/.buildkite/_wheel-build.rayci.yml b/.buildkite/_wheel-build.rayci.yml index 2c272e1a478..f5935ceb2e8 100644 --- a/.buildkite/_wheel-build.rayci.yml +++ b/.buildkite/_wheel-build.rayci.yml @@ -4,6 +4,7 @@ steps: - name: ray-core-build label: "wanda: core binary parts py{{array.python}} (x86_64)" wanda: ci/docker/ray-core.wanda.yaml + # TODO: drop after the move to manylinux_2_28 (see _forge.rayci.yml). depends_on: bazel-glibc217-x86_64 array: python: @@ -33,6 +34,7 @@ steps: - name: ray-java-build label: "wanda: java build (x86_64)" wanda: ci/docker/ray-java.wanda.yaml + # TODO: drop after the move to manylinux_2_28 (see _forge.rayci.yml). depends_on: bazel-glibc217-x86_64 env_file: rayci.env env: diff --git a/.buildkite/build.rayci.yml b/.buildkite/build.rayci.yml index 5a292afd964..7c50345b67c 100644 --- a/.buildkite/build.rayci.yml +++ b/.buildkite/build.rayci.yml @@ -30,6 +30,7 @@ steps: - name: ray-cpp-core-build label: "wanda: cpp core (x86_64)" wanda: ci/docker/ray-cpp-core.wanda.yaml + # TODO: drop after the move to manylinux_2_28 (see _forge.rayci.yml). depends_on: bazel-glibc217-x86_64 env_file: rayci.env env: diff --git a/.buildkite/linux_aarch64.rayci.yml b/.buildkite/linux_aarch64.rayci.yml index b6d26613bc4..fae0e7e32bd 100644 --- a/.buildkite/linux_aarch64.rayci.yml +++ b/.buildkite/linux_aarch64.rayci.yml @@ -10,6 +10,9 @@ steps: wanda: ci/docker/forge.aarch64.wanda.yaml instance_type: builder-arm64 + # Bazel built for glibc 2.17, for the images that run Bazel inside + # manylinux2014. See the x86_64 step in _forge.rayci.yml for why. + # TODO: remove after the move to manylinux_2_28 (see _forge.rayci.yml). - name: bazel-glibc217-aarch64 label: "wanda: bazel for manylinux2014 (aarch64)" wanda: ci/docker/bazel-glibc217.wanda.yaml @@ -20,6 +23,7 @@ steps: - name: manylinux-aarch64 wanda: ci/docker/manylinux.wanda.yaml + # TODO: drop after the move to manylinux_2_28 (see _forge.rayci.yml). depends_on: bazel-glibc217-aarch64 env_file: rayci.env env: @@ -29,6 +33,7 @@ steps: - name: ray-java-build-aarch64 label: "wanda: java build (aarch64)" wanda: ci/docker/ray-java.wanda.yaml + # TODO: drop after the move to manylinux_2_28 (see _forge.rayci.yml). depends_on: bazel-glibc217-aarch64 tags: - release_wheels @@ -196,6 +201,7 @@ steps: - name: ray-core-build-aarch64 label: "wanda: core binary parts py{{array.python}} (aarch64)" wanda: ci/docker/ray-core.wanda.yaml + # TODO: drop after the move to manylinux_2_28 (see _forge.rayci.yml). depends_on: bazel-glibc217-aarch64 env_file: rayci.env array: @@ -270,6 +276,7 @@ steps: - name: ray-cpp-core-build-aarch64 label: "wanda: cpp core (aarch64)" wanda: ci/docker/ray-cpp-core.wanda.yaml + # TODO: drop after the move to manylinux_2_28 (see _forge.rayci.yml). depends_on: bazel-glibc217-aarch64 env_file: rayci.env env: diff --git a/ci/docker/manylinux-retag.Dockerfile b/ci/docker/manylinux-retag.Dockerfile index 66da26948dc..5a3ec2a64dd 100644 --- a/ci/docker/manylinux-retag.Dockerfile +++ b/ci/docker/manylinux-retag.Dockerfile @@ -15,6 +15,7 @@ FROM rayproject/manylinux2014:${MANYLINUX_VERSION}-jdk-${HOSTTYPE} # Bazel's own Linux releases need glibc >= 2.25 since Bazel 8, and this image is # glibc 2.17. Run the copy built for it (ci/docker/bazel-glibc217.Dockerfile) # instead. The build calls bazelisk, and /usr/local/bin/bazel links to it. +# TODO: remove after the move to manylinux_2_28 (see .buildkite/_forge.rayci.yml). COPY --from=bazel /bazel /opt/bazel-glibc217/bazel COPY --from=bazel /jdk21 /opt/bazel-glibc217/jdk21 RUN sudo mv /usr/local/bin/bazelisk /usr/local/bin/bazelisk.real \ diff --git a/ci/docker/ray-core.Dockerfile b/ci/docker/ray-core.Dockerfile index d5cbdac5935..887117b1d21 100644 --- a/ci/docker/ray-core.Dockerfile +++ b/ci/docker/ray-core.Dockerfile @@ -25,6 +25,7 @@ WORKDIR /home/forge/ray # Bazel's own Linux releases need glibc >= 2.25 since Bazel 8, and this image is # glibc 2.17. Run the copy built for it (ci/docker/bazel-glibc217.Dockerfile) # instead. The build calls bazelisk, and /usr/local/bin/bazel links to it. +# TODO: remove after the move to manylinux_2_28 (see .buildkite/_forge.rayci.yml). COPY --from=bazel /bazel /opt/bazel-glibc217/bazel COPY --from=bazel /jdk21 /opt/bazel-glibc217/jdk21 RUN sudo mv /usr/local/bin/bazelisk /usr/local/bin/bazelisk.real \ diff --git a/ci/docker/ray-cpp-core.Dockerfile b/ci/docker/ray-cpp-core.Dockerfile index 3f94e382bc2..57062534366 100644 --- a/ci/docker/ray-cpp-core.Dockerfile +++ b/ci/docker/ray-cpp-core.Dockerfile @@ -28,6 +28,7 @@ WORKDIR /home/forge/ray # Bazel's own Linux releases need glibc >= 2.25 since Bazel 8, and this image is # glibc 2.17. Run the copy built for it (ci/docker/bazel-glibc217.Dockerfile) # instead. The build calls bazelisk, and /usr/local/bin/bazel links to it. +# TODO: remove after the move to manylinux_2_28 (see .buildkite/_forge.rayci.yml). COPY --from=bazel /bazel /opt/bazel-glibc217/bazel COPY --from=bazel /jdk21 /opt/bazel-glibc217/jdk21 RUN sudo mv /usr/local/bin/bazelisk /usr/local/bin/bazelisk.real \ diff --git a/ci/docker/ray-java.Dockerfile b/ci/docker/ray-java.Dockerfile index 1ef17014c36..a24b2a069c4 100644 --- a/ci/docker/ray-java.Dockerfile +++ b/ci/docker/ray-java.Dockerfile @@ -24,6 +24,7 @@ WORKDIR /home/forge/ray # Bazel's own Linux releases need glibc >= 2.25 since Bazel 8, and this image is # glibc 2.17. Run the copy built for it (ci/docker/bazel-glibc217.Dockerfile) # instead. The build calls bazelisk, and /usr/local/bin/bazel links to it. +# TODO: remove after the move to manylinux_2_28 (see .buildkite/_forge.rayci.yml). COPY --from=bazel /bazel /opt/bazel-glibc217/bazel COPY --from=bazel /jdk21 /opt/bazel-glibc217/jdk21 RUN sudo mv /usr/local/bin/bazelisk /usr/local/bin/bazelisk.real \