Add RHEL / manylinux build tutorial

Build_Guide/RHEL_Manylinux/Dockerfile.base.rhel

@@ -0,0 +1,94 @@
+# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# Reconstructs, from 100% public sources, the EL8 / glibc-2.28 (manylinux_2_28)
+# CUDA/cuDNN/TensorRT base image (plus OS build-dependencies) that
+# `build.py --target-platform=rhel` expects but does not install itself -- so you can
+# reproduce a RHEL/manylinux Triton build without an internal, unpublished base image.
+# Rocky Linux 8 (not UBI8) because build.py's -devel deps need EPEL + PowerTools.
+#
+# EXAMPLE ONLY -- not officially supported; equivalent, not identical, to the base
+# NVIDIA uses for the released manylinux artifacts. See this tutorial's README.
+#
+#   docker build -f Dockerfile.base.rhel \
+#     --build-arg BASE_IMAGE=nvidia/cuda:13.2.1-cudnn-devel-rockylinux8 \
+#     --build-arg TENSORRT_VERSION=10.16.1.11-1.cuda13.2 -t triton-manylinux-base:local .
+
+# The base CUDA/cuDNN image; pin to the target release (e.g. nvidia/cuda:13.0.3-...-rockylinux8 for 25.08).
+ARG BASE_IMAGE=nvidia/cuda:13.2.1-cudnn-devel-rockylinux8
+FROM ${BASE_IMAGE}
+
+# EPEL + PowerTools, then the OS packages build.py needs that a stock CUDA image lacks:
+# gcc-toolset-13 (C++17 default; system gcc 8.5 is too old), python3.12 headers (pybind11),
+# build utils incl. patch (ORT dep fetch), and CPython build deps (_ctypes/_bz2/_lzma).
+RUN set -eux; \
+    dnf install -y dnf-plugins-core epel-release; \
+    dnf config-manager --set-enabled powertools || dnf config-manager --set-enabled crb || true; \
+    dnf install -y \
+        gcc-toolset-13 python3.12 python3.12-devel python3.12-pip \
+        patch which file diffutils make \
+        libffi-devel bzip2-devel xz-devel zlib-devel; \
+    dnf clean all
+
+# Make gcc-toolset-13 the default compiler (CC/CXX so nested CMake builds honor it, not the
+# system gcc 8.5); point ORT's build at cuDNN (CUDNN_HOME); pre-create the pyenv symlink dir.
+ENV PATH=/opt/rh/gcc-toolset-13/root/usr/bin:${PATH} \
+    LD_LIBRARY_PATH=/opt/rh/gcc-toolset-13/root/usr/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} \
+    CC=/opt/rh/gcc-toolset-13/root/usr/bin/gcc \
+    CXX=/opt/rh/gcc-toolset-13/root/usr/bin/g++ \
+    CUDNN_HOME=/usr
+
+# TensorRT from the cuda-rhel8 repo that the base image already ships in /etc/yum.repos.d/.
+# Empty installs newest TensorRT for the base CUDA, otherwise pin to release version.
+ARG TENSORRT_VERSION=
+RUN set -eux; \
+    if [ -n "${TENSORRT_VERSION}" ]; then \
+        dnf install -y "tensorrt-devel-${TENSORRT_VERSION}"; \
+        dnf install -y python3-dnf-plugin-versionlock; \
+        dnf versionlock add "tensorrt*" "libnvinfer*"; \
+    else \
+        dnf install -y tensorrt-devel; \
+    fi; \
+    dnf clean all
+
+# auditwheel/patchelf retag the tritonserver wheels manylinux_2_28 (else linux_x86_64). The rhel
+# buildbase compiles with a fresh pyenv python AND reshuffles the default python3.12, so a plain
+# `pip install auditwheel` ends up importable by neither interpreter at wheel-repair time. Install
+# both in an isolated venv and put them on PATH -- self-contained, so the swap can't orphan them,
+# and stock build.py's manylinux repair step works without patching build.py.
+RUN mkdir -p /opt/python && \
+    python3.12 -m venv --copies /opt/auditwheel-venv && \
+    /opt/auditwheel-venv/bin/pip install --no-cache-dir auditwheel patchelf && \
+    ln -sf /opt/auditwheel-venv/bin/auditwheel /usr/local/bin/auditwheel && \
+    ln -sf /opt/auditwheel-venv/bin/patchelf   /usr/local/bin/patchelf
+
+# The final runtime image (FROM this base) installs the tritonserver/tritonfrontend wheels with a
+# bare `pip install <whl>`. Rocky 8's system `pip` maps to python3.6, which can't install cp312 wheels,
+# so point `/usr/local/bin/pip` at pip3.12 (from the python3.12-pip package installed above).
+RUN ln -sf /usr/bin/pip3.12 /usr/local/bin/pip
+
+# Fail loudly if a bad version pin left the CUDA/TensorRT/compiler/pip pieces missing.
+RUN set -eux; rpm -q tensorrt-devel; nvcc --version >/dev/null; g++ --version | head -1; pip --version

Build_Guide/RHEL_Manylinux/Dockerfile.pytorch-runtime.rhel

@@ -0,0 +1,63 @@
+# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# Completes the `tritonserver` image from Step 3 so the PyTorch backend can SERVE, not just
+# build. ONLY needed for the pytorch backend; the other backends serve from the Step 3 image
+# as-is.
+#
+# The pytorch backend runs a Python "stub" process at model load whose `import torch` /
+# `import numpy` must resolve under the pyenv Python the backend was compiled against (at
+# /opt/pyenv_build, not the system python3.12). Building the `python` backend already had
+# build.py provision that interpreter AND numpy into this image (its `if "python" in
+# FLAGS.backend` runtime block). All that's left is torch itself -- the framework build.py
+# never bundles -- so we add it here, into that same interpreter.
+#
+# If you build WITHOUT the python backend, build.py won't provision the interpreter; you then
+# also need to COPY it from tritonserver_buildbase and `pip install numpy` here first.
+#
+#   docker build -f Dockerfile.pytorch-runtime.rhel -t tritonserver-pytorch:example .
+
+FROM tritonserver
+
+# Match the libtorch the backend extracted from the Step 2 image (CUDA 13.2 -> cu132).
+# Keep this in sync with Dockerfile.pytorch.rhel's TORCH_VERSION -- a mismatch would ABI-break.
+ARG TORCH_VERSION=2.13.0
+ARG TORCH_INDEX_URL=https://download.pytorch.org/whl/cu132
+
+# Point at whichever 3.12.<patch> build.py provisioned, so nothing below hardcodes the patch.
+RUN ln -sfn "$(ls -d /opt/pyenv_build/versions/3.12.* | head -1)" /opt/pyenv_build/versions/3.12
+
+# Add torch into the pyenv Python build.py already set up (via the python backend).
+RUN /opt/pyenv_build/versions/3.12/bin/python -m pip install --no-cache-dir \
+      "torch==${TORCH_VERSION}" --index-url "${TORCH_INDEX_URL}"
+
+# The extracted libtorch links cuSPARSELt, NCCL and nvshmem (the last via libtorch_nvshmem.so,
+# copied in by NVSHMEM=ON), none of which the stock CUDA base ships. All three are bundled inside
+# the torch wheel installed above (site-packages/nvidia/*/lib), so point the loader at just those
+# three dirs -- version-matched to the libtorch, and without shadowing the base's own CUDA libs.
+RUN SP=/opt/pyenv_build/versions/3.12/lib/python3.12/site-packages/nvidia; \
+    printf '%s\n' "$SP/cusparselt/lib" "$SP/nccl/lib" "$SP/nvshmem/lib" \
+      > /etc/ld.so.conf.d/torch-cuda.conf && ldconfig

Build_Guide/RHEL_Manylinux/Dockerfile.pytorch.rhel

@@ -0,0 +1,85 @@
+# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# Public reconstruction of the internal PyTorch manylinux "wheel image"
+# (dl/dgx/pytorch:<ver>-py3-wheel-*) that build.py's `--image=pytorch,...` path extracts a
+# prebuilt libtorch from -- i.e. a manylinux_2_28 image with a public torch wheel installed,
+# mirroring what the internal CI does (install torch into the wheel image, then --image=pytorch).
+# ONLY needed for the pytorch backend. EXAMPLE ONLY -- equivalent, not identical, to the
+# internal wheel image. This tutorial builds without torchvision (see the README's Step 3);
+# wiring it up from public sources is an untested path here.
+#
+#   docker build -f Dockerfile.pytorch.rhel \
+#     --build-arg TORCH_INDEX_URL=https://download.pytorch.org/whl/cu132 \
+#     -t triton-manylinux-pytorch:example .
+
+# manylinux_2_28 == AlmaLinux 8 / glibc 2.28, matching the base-image ABI and providing the
+# /opt/_internal/cpython-3.12.* layout the pytorch_backend copies libtorch from.
+ARG BASE_IMAGE=quay.io/pypa/manylinux_2_28_x86_64
+FROM ${BASE_IMAGE}
+
+# Pin torch to the release's version; the CUDA channel must match the base (13.2 -> cu132).
+# Keep this in sync with Dockerfile.pytorch-runtime.rhel's TORCH_VERSION -- the wheel here
+# is the source build.py extracts libtorch from, and the runtime installs the same version
+# to get matching bundled cuSPARSELt/NCCL/nvshmem. A mismatch would ABI-break at server load.
+ARG TORCH_VERSION=2.13.0
+ARG TORCH_INDEX_URL=https://download.pytorch.org/whl/cu132
+ENV PATH=/opt/python/cp312-cp312/bin:${PATH}
+
+RUN set -eux; \
+    python -m pip install --upgrade pip; \
+    python -m pip install --index-url "${TORCH_INDEX_URL}" \
+        "torch==${TORCH_VERSION}"; \
+    # The rhel pytorch_backend copies from a HARDCODED /opt/_internal/cpython-3.12.1 path;
+    # symlink it to whatever 3.12 patch this image actually ships so the `docker cp` resolves.
+    if [ ! -e /opt/_internal/cpython-3.12.1 ]; then \
+        ln -s "$(ls -d /opt/_internal/cpython-3.12.*)" /opt/_internal/cpython-3.12.1; \
+    fi; \
+    # OS libs the backend copies from /usr/lib64 (libjpeg.so.62 / libpng16.so.16).
+    dnf install -y libjpeg-turbo libpng; \
+    dnf clean all; \
+    python -c "import torch; print('torch', torch.__version__, torch.__path__[0] + '/lib')"
+
+# The rhel pytorch_backend's --image=pytorch path does a FIXED set of `docker cp`s that
+# NVIDIA's from-source torch image satisfies but a public wheel doesn't. Backfill them so the
+# (hardcoded, unconditional) copies succeed:
+#  - 10 MKL libs + libbackend_with_compiler.so + libaoti_custom_ops.so: this torch links no MKL
+#    (verified via ldd) and omits the two demo/AOTI libs, so empty stub .so's satisfy the copy
+#    and the patchelf that follows, and are never dlopen'd at runtime.
+#  - LICENSE + jit/codegen: taken from the wheel (the codegen headers already ship in torch/include).
+RUN set -eux; \
+    P=/opt/_internal/cpython-3.12.1; \
+    TL="$P/lib/python3.12/site-packages/torch/lib"; \
+    SP="$P/lib/python3.12/site-packages"; \
+    stub() { gcc -shared -fPIC -x c -o "$1" /dev/null; }; \
+    for m in avx2 avx512 core def gnu_thread intel_lp64 intel_thread rt sequential vml_def; do \
+        stub "$P/lib/libmkl_${m}.so.1"; \
+    done; \
+    stub "$TL/libbackend_with_compiler.so"; \
+    stub "$TL/libaoti_custom_ops.so"; \
+    mkdir -p /opt/pytorch/pytorch/torch/csrc/jit; \
+    cp "$(find "$SP" -path '*/torch-*.dist-info/licenses/LICENSE' | head -1)" /opt/pytorch/pytorch/LICENSE; \
+    cp -r "$SP/torch/include/torch/csrc/jit/codegen" /opt/pytorch/pytorch/torch/csrc/jit/codegen