Skip to content

docs: Update projects #21

docs: Update projects

docs: Update projects #21

# SPDX-FileCopyrightText: 2026 The RISE Project
# SPDX-License-Identifier: MIT
#
# This workflow is based on: https://github.com/run-ai/runai-model-streamer/blob/master/Makefile
# and https://github.com/run-ai/runai-model-streamer/blob/master/.devcontainer/Dockerfile
---
name: Build runai-model-streamer-s3 wheels (riscv64)
on:
workflow_dispatch:
inputs:
version:
description: 'Version glob to (re)build; empty builds every version of docs/packages/runai-model-streamer-s3.yaml not released yet'
required: false
default: ''
pull_request:
branches: [main]
paths:
- '.github/workflows/build-runai-model-streamer-s3.yml'
- 'docs/packages/runai-model-streamer-s3.yaml'
- 'patches/runai-model-streamer-s3/**'
push:
branches: [main]
paths:
- '.github/workflows/build-runai-model-streamer-s3.yml'
- 'docs/packages/runai-model-streamer-s3.yaml'
- 'patches/runai-model-streamer-s3/**'
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
permissions:
contents: read
env:
MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64
BAZEL_VERSION: '7.6.1'
RULES_PYTHON_VERSION: '0.33.2'
RULES_JAVA_VERSION: '7.6.5'
OPENSSL_VERSION: '3.5.0'
CURL_VERSION: '8.13.0'
ZLIB_VERSION: '1.3.1'
AWS_SDK_CPP_VERSION: '1.11.584'
# dl.min.io no longer serves server binaries for any arch; built from the last release tag.
MINIO_VERSION: 'RELEASE.2025-10-15T17-29-55Z'
jobs:
setup:
uses: $/.github/workflows/_setup.yml
with:
package: runai-model-streamer-s3
version: ${{ inputs.version }}
bazel:
needs: [setup]
if: needs.setup.outputs.versions != '[]'
name: Bootstrap bazel (riscv64)
runs-on: ubuntu-24.04-riscv
timeout-minutes: 120
steps:
- name: Restore bazel binary
id: cache
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: bazel-bin
key: bazel-${{ env.BAZEL_VERSION }}-manylinux_riscv64
- name: Bootstrap bazel ${{ env.BAZEL_VERSION }}
if: steps.cache.outputs.cache-hit != 'true'
run: |
mkdir -p bazel-bin
docker run --rm -i --network=host \
-v "${GITHUB_WORKSPACE}:/work" \
-w /work \
-e BAZEL_VERSION="${BAZEL_VERSION}" \
-e RULES_PYTHON_VERSION="${RULES_PYTHON_VERSION}" \
-e RULES_JAVA_VERSION="${RULES_JAVA_VERSION}" \
"${MANYLINUX_RISCV64_IMAGE}" \
bash <<'SCRIPT'
set -eux
dnf install -y --setopt=install_weak_deps=False java-21-openjdk-devel zip unzip
JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")")"
export JAVA_HOME
# rules_python 0.33.2 has no riscv64 PLATFORMS entry (bazelbuild/bazel#23018).
mkdir -p /tmp/rules_python
curl -fsSLo /tmp/rules_python.tar.gz "https://github.com/bazel-contrib/rules_python/releases/download/${RULES_PYTHON_VERSION}/rules_python-${RULES_PYTHON_VERSION}.tar.gz"
tar -xzf /tmp/rules_python.tar.gz -C /tmp/rules_python --strip-components=1
sed -i 's|fail("No platform declared for host OS {} on arch {}".format(os_name, arch))|return "x86_64-unknown-linux-gnu"|' \
/tmp/rules_python/python/private/toolchains_repo.bzl
# rules_java 7.x maps riscv64 to a stray-colon include path for jni_md.h.
mkdir -p /tmp/rules_java
curl -fsSLo /tmp/rules_java.tar.gz "https://github.com/bazelbuild/rules_java/releases/download/${RULES_JAVA_VERSION}/rules_java-${RULES_JAVA_VERSION}.tar.gz"
tar -xzf /tmp/rules_java.tar.gz -C /tmp/rules_java
sed -i 's|\[":include/linux"\]|["include/linux"]|g' /tmp/rules_java/toolchains/BUILD
mkdir -p /tmp/bazel-src
cd /tmp/bazel-src
curl -fsSLo dist.zip "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-dist.zip"
unzip -q dist.zip
EXTRA_BAZEL_ARGS="--tool_java_runtime_version=local_jdk \
--override_module=rules_python=/tmp/rules_python \
--override_module=rules_java=/tmp/rules_java" \
bash ./compile.sh
install -m 0755 output/bazel /work/bazel-bin/bazel
SCRIPT
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: bazel-${{ env.BAZEL_VERSION }}-riscv64
path: bazel-bin/bazel
if-no-files-found: error
minio:
needs: [setup]
if: needs.setup.outputs.versions != '[]'
name: Build MinIO (riscv64)
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version: 'stable'
cache: false
- name: Build MinIO ${{ env.MINIO_VERSION }}
env:
CGO_ENABLED: '0'
GOOS: linux
GOARCH: riscv64
run: |
go install "github.com/minio/minio@${MINIO_VERSION}"
mkdir -p minio
cp "$(go env GOPATH)/bin/linux_riscv64/minio" minio/minio
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: minio-riscv64
path: minio/minio
if-no-files-found: error
build_wheel:
name: Build runai-model-streamer-s3 ${{ matrix.version }} py3-none-manylinux_riscv64
needs: [setup, bazel]
if: needs.setup.outputs.versions != '[]'
runs-on: ubuntu-24.04-riscv
timeout-minutes: 300
strategy:
fail-fast: false
matrix:
version: ${{ fromJSON(needs.setup.outputs.versions) }}
env:
RUNAI_MODEL_STREAMER_VERSION: ${{ matrix.version }}
steps:
- name: Checkout runai-model-streamer ${{ env.RUNAI_MODEL_STREAMER_VERSION }}
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: run-ai/runai-model-streamer
ref: ${{ env.RUNAI_MODEL_STREAMER_VERSION }}
path: runai-model-streamer
persist-credentials: false
- name: Checkout python-wheels
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
path: python-wheels
persist-credentials: false
- name: Patch runai-model-streamer source
working-directory: runai-model-streamer
run: git apply -v ../python-wheels/patches/runai-model-streamer-s3/${{ env.RUNAI_MODEL_STREAMER_VERSION }}/*.patch
- name: Download bazel
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: bazel-${{ env.BAZEL_VERSION }}-riscv64
path: bazel-bin
- name: Restore OpenSSL, curl, zlib and the AWS SDK
id: deps-cache
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: deps
key: runai-model-streamer-s3-deps-manylinux_2_39_riscv64-${{ hashFiles('python-wheels/.github/workflows/build-runai-model-streamer-s3.yml') }}
- name: Build OpenSSL, curl, zlib and the AWS SDK
if: steps.deps-cache.outputs.cache-hit != 'true'
run: |
mkdir -p deps/riscv64-ssl deps/riscv64-curl deps/riscv64-zlib deps/riscv64-aws deps/licenses
docker run --rm -i --network=host \
-v "${GITHUB_WORKSPACE}:/work" \
-v "${GITHUB_WORKSPACE}/deps/riscv64-ssl:/opt/riscv64-ssl" \
-v "${GITHUB_WORKSPACE}/deps/riscv64-curl:/opt/riscv64-curl" \
-v "${GITHUB_WORKSPACE}/deps/riscv64-zlib:/opt/riscv64-zlib" \
-v "${GITHUB_WORKSPACE}/deps/riscv64-aws:/opt/riscv64-aws" \
-w /tmp \
-e OPENSSL_VERSION="${OPENSSL_VERSION}" \
-e CURL_VERSION="${CURL_VERSION}" \
-e ZLIB_VERSION="${ZLIB_VERSION}" \
-e AWS_SDK_CPP_VERSION="${AWS_SDK_CPP_VERSION}" \
"${MANYLINUX_RISCV64_IMAGE}" \
bash <<'SCRIPT'
set -eux
# OpenSSL's Configure needs perl modules the image's minimal perl-interpreter lacks.
dnf install -y --setopt=install_weak_deps=False perl
LICENSES=/work/deps/licenses
curl -fsSL -o "openssl-${OPENSSL_VERSION}.tar.gz" "https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz"
tar zxf "openssl-${OPENSSL_VERSION}.tar.gz"
cd "openssl-${OPENSSL_VERSION}"
./Configure no-ssl2 no-zlib no-shared no-comp no-dynamic-engine enable-ec_nistp_64_gcc_128 \
--prefix=/opt/riscv64-ssl --openssldir=/opt/riscv64-ssl --libdir=lib linux64-riscv64
make -j"$(nproc)" > /dev/null
make install_sw > /dev/null
cp LICENSE.txt "$LICENSES/LICENSE.openssl"
cd /tmp
curl -fsSL -o "curl-${CURL_VERSION}.tar.gz" "https://github.com/curl/curl/releases/download/curl-${CURL_VERSION//./_}/curl-${CURL_VERSION}.tar.gz"
tar zxf "curl-${CURL_VERSION}.tar.gz"
cd "curl-${CURL_VERSION}"
# Keep libcurl.a to the libraries the @aws link line names, whatever -devel the image carries.
CPPFLAGS=-I/opt/riscv64-ssl/include \
LDFLAGS=-L/opt/riscv64-ssl/lib \
CFLAGS="-fPIC" \
./configure --disable-shared --without-libpsl --with-openssl=/opt/riscv64-ssl --prefix=/opt/riscv64-curl \
--without-brotli --without-zstd --without-nghttp2 --without-libidn2
make -j"$(nproc)" > /dev/null
make install > /dev/null
cp COPYING "$LICENSES/LICENSE.curl"
cd /tmp
curl -fsSL -o "zlib-${ZLIB_VERSION}.tar.gz" "https://github.com/madler/zlib/releases/download/v${ZLIB_VERSION}/zlib-${ZLIB_VERSION}.tar.gz"
tar zxf "zlib-${ZLIB_VERSION}.tar.gz"
cd "zlib-${ZLIB_VERSION}"
# Upstream's Ubuntu gcc defaults to PIE, which is what lets its non-PIC libz.a link into a .so.
CFLAGS="-fPIC" ./configure --static --shared --prefix=/opt/riscv64-zlib
make -j"$(nproc)" > /dev/null
make install > /dev/null
cp LICENSE "$LICENSES/LICENSE.zlib"
cd /tmp
git clone --recurse-submodules --shallow-submodules --depth 1 --branch "${AWS_SDK_CPP_VERSION}" https://github.com/aws/aws-sdk-cpp /tmp/aws-sdk-cpp
cd /tmp/aws-sdk-cpp
# GNUInstallDirs would pick lib64 on this RHEL-family image; the @aws link line expects lib.
cmake -B build \
-DBUILD_ONLY="s3-crt" \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-DCMAKE_PREFIX_PATH="/opt/riscv64-ssl;/opt/riscv64-curl" \
-DCMAKE_INSTALL_PREFIX="/opt/riscv64-aws" \
-DCMAKE_INSTALL_LIBDIR=lib \
-DFORCE_SHARED_CRT=OFF \
-DBUILD_SHARED_LIBS=OFF \
-DENABLE_TESTING=OFF \
-DAUTORUN_UNIT_TESTS=OFF \
-DZLIB_LIBRARY="/opt/riscv64-zlib/lib/libz.a" \
-DZLIB_INCLUDE_DIR="/opt/riscv64-zlib/include" \
-Dcrypto_LIBRARY="/opt/riscv64-ssl/lib/libcrypto.a" \
-Dcrypto_INCLUDE_DIR="/opt/riscv64-ssl/include" \
-DCURL_LIBRARY="/opt/riscv64-curl/lib/libcurl.a" \
-DCURL_INCLUDE_DIR="/opt/riscv64-curl/include" \
-DOPENSSL_CRYPTO_LIBRARY="/opt/riscv64-ssl/lib/libssl.a" \
-DOPENSSL_INCLUDE_DIR="/opt/riscv64-ssl/include"
cmake --build build -j"$(nproc)"
cmake --install build
for dir in /tmp/aws-sdk-cpp /tmp/aws-sdk-cpp/crt/aws-crt-cpp /tmp/aws-sdk-cpp/crt/aws-crt-cpp/crt/*; do
name="$(basename "$dir")"
if [ "$name" = aws-lc ]; then
continue
fi
if [ -f "$dir/LICENSE" ]; then
cp "$dir/LICENSE" "$LICENSES/LICENSE.$name"
fi
if [ -f "$dir/NOTICE" ]; then
cp "$dir/NOTICE" "$LICENSES/LICENSE.$name.NOTICE"
fi
done
ls -l /opt/riscv64-aws/lib "$LICENSES"
SCRIPT
- name: Save OpenSSL, curl, zlib and the AWS SDK
if: steps.deps-cache.outputs.cache-hit != 'true'
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: deps
key: ${{ steps.deps-cache.outputs.cache-primary-key }}
- name: Build the S3 streamer library and package the wheel
run: |
mkdir -p dist
docker run --rm -i --network=host \
-v "${GITHUB_WORKSPACE}:/work" \
-v "${GITHUB_WORKSPACE}/deps/riscv64-ssl:/opt/riscv64-ssl" \
-v "${GITHUB_WORKSPACE}/deps/riscv64-curl:/opt/riscv64-curl" \
-v "${GITHUB_WORKSPACE}/deps/riscv64-zlib:/opt/riscv64-zlib" \
-v "${GITHUB_WORKSPACE}/deps/riscv64-aws:/opt/riscv64-aws" \
-w /work/runai-model-streamer \
-e PACKAGE_VERSION="${RUNAI_MODEL_STREAMER_VERSION}" \
"${MANYLINUX_RISCV64_IMAGE}" \
bash <<'SCRIPT'
set -eux
# cpp/toolchain links -l:libstdc++.a, which on this image is its own CRB package.
dnf install -y --setopt=install_weak_deps=False java-21-openjdk-devel libstdc++-static
JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")")"
export JAVA_HOME
install -m 0755 /work/bazel-bin/bazel /usr/local/bin/bazel
git config --global --add safe.directory '*'
# cpp/toolchain/rules.bzl expects the Debian cross-toolchain names; this image's gcc is native.
for tool in gcc g++ ar ld cpp gcov nm objdump strip; do
native="$(command -v "$tool" || true)"
[ -n "$native" ] && ln -sf "$native" "/usr/bin/riscv64-linux-gnu-$tool"
done
cd cpp
bazel build s3:libstreamers3.so --config=riscv64
cd ../py/runai_model_streamer_s3
cp /work/deps/licenses/LICENSE.* .
/opt/python/cp312-cp312/bin/pip install -q setuptools wheel
/opt/python/cp312-cp312/bin/python setup.py bdist_wheel --plat-name manylinux_2_39_riscv64
cp dist/*.whl /work/dist/
SCRIPT
- name: Verify the wheel ships a riscv64 shared object and its licences
run: |
python3 - dist/*.whl <<'EOF'
import sys, zipfile
with zipfile.ZipFile(sys.argv[1]) as zf:
names = zf.namelist()
so = next(n for n in names if n.endswith("libstreamer/lib/libstreamers3.so"))
header = zf.read(so)[:20]
assert header[:4] == b"\x7fELF", header
assert header[18] == 0xF3, header # EM_RISCV
licenses = {n.rsplit("/", 1)[-1] for n in names if ".dist-info/licenses/" in n}
print(sorted(licenses))
for dep in ("openssl", "curl", "zlib", "aws-sdk-cpp", "aws-crt-cpp", "s2n", "aws-c-common"):
assert f"LICENSE.{dep}" in licenses, dep
EOF
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: runai-model-streamer-s3-${{ matrix.version }}-py3-none-manylinux_riscv64
path: dist/*.whl
if-no-files-found: error
test_wheel:
name: Test runai-model-streamer-s3 ${{ matrix.version }} on Python ${{ matrix.python-version }}
needs: [setup, build_wheel, minio]
if: needs.setup.outputs.versions != '[]'
runs-on: ubuntu-24.04-riscv
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
version: ${{ fromJSON(needs.setup.outputs.versions) }}
python-version: ['3.12', '3.13', '3.14']
env:
RUNAI_MODEL_STREAMER_VERSION: ${{ matrix.version }}
steps:
- name: Checkout runai-model-streamer ${{ env.RUNAI_MODEL_STREAMER_VERSION }}
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: run-ai/runai-model-streamer
ref: ${{ env.RUNAI_MODEL_STREAMER_VERSION }}
path: runai-model-streamer
persist-credentials: false
- name: Download wheel
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: runai-model-streamer-s3-${{ env.RUNAI_MODEL_STREAMER_VERSION }}-py3-none-manylinux_riscv64
path: wheelhouse
- name: Download MinIO
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: minio-riscv64
path: minio
- name: Install Python
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
with:
python-version: ${{ matrix.python-version }}
activate-environment: true
enable-cache: false
- name: Install the wheel and upstream's test dependencies
env:
UV_EXTRA_INDEX_URL: https://pypi.riseproject.dev/simple/
UV_INDEX_STRATEGY: unsafe-best-match
# requirements.dev's numpy==1.24.4 has no riscv64 wheel and no longer builds on Python >=3.12.
run: |
echo "numpy>=1.26,<3" > "$RUNNER_TEMP/numpy-override.txt"
uv pip install --override "$RUNNER_TEMP/numpy-override.txt" \
-r runai-model-streamer/py/runai_model_streamer/requirements.dev \
pytest \
"runai-model-streamer[s3]==${RUNAI_MODEL_STREAMER_VERSION}" \
wheelhouse/*.whl
- name: Run the S3 package's unit tests against the installed wheel
run: |
site="$(python -c 'import runai_model_streamer_s3, os; print(os.path.dirname(os.path.dirname(runai_model_streamer_s3.__file__)))')"
cd "$site"
python -m unittest discover -b runai_model_streamer_s3
- name: Run upstream's S3 tests against MinIO
working-directory: runai-model-streamer/tests
env:
MINIO_ROOT_USER: minioadmin
MINIO_ROOT_PASSWORD: minioadmin
RUNAI_STREAMER_S3_USE_VIRTUAL_ADDRESSING: '0'
AWS_BUCKET: test-bucket
AWS_ENDPOINT_URL: http://127.0.0.2:9000
AWS_ACCESS_KEY_ID: minioadmin
AWS_SECRET_ACCESS_KEY: minioadmin
run: |
chmod +x "$GITHUB_WORKSPACE/minio/minio"
mkdir -p "$RUNNER_TEMP/minio/test-bucket"
"$GITHUB_WORKSPACE/minio/minio" server "$RUNNER_TEMP/minio" --address ":9000" &
trap 'kill $!' EXIT
pytest -v s3/
gpl_sources:
needs: [setup]
if: needs.setup.outputs.versions != '[]'
strategy:
fail-fast: false
matrix:
version: ${{ fromJSON(needs.setup.outputs.versions) }}
name: Collect GPL sources for runai-model-streamer-s3 ${{ matrix.version }}
runs-on: ubuntu-24.04-riscv
env:
RUNAI_MODEL_STREAMER_VERSION: ${{ matrix.version }}
steps:
- name: Checkout python-wheels
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: ./actions/collect-gpl-sources
with:
image: ${{ env.MANYLINUX_RISCV64_IMAGE }}
packages: gcc
output: gpl-sources.tar
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: runai-model-streamer-s3-${{ env.RUNAI_MODEL_STREAMER_VERSION }}-gpl-sources
path: gpl-sources.tar
if-no-files-found: error
publish:
name: Publish runai-model-streamer-s3 ${{ matrix.version }}
needs: [setup, build_wheel, test_wheel, gpl_sources]
if: needs.setup.outputs.versions != '[]'
strategy:
fail-fast: false
matrix:
version: ${{ fromJSON(needs.setup.outputs.versions) }}
permissions:
contents: write
pull-requests: write
uses: $/.github/workflows/_publish-wheel.yml
secrets:
app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }}
with:
artifact-pattern: runai-model-streamer-s3-${{ matrix.version }}-py3-none-manylinux_riscv64
gpl-sources-artifact: runai-model-streamer-s3-${{ matrix.version }}-gpl-sources
gpl-sources-description: gcc