diff --git a/.github/workflows/ascend-npu.yml b/.github/workflows/ascend-npu.yml new file mode 100644 index 0000000..a928c5f --- /dev/null +++ b/.github/workflows/ascend-npu.yml @@ -0,0 +1,64 @@ +name: Ascend NPU Host Regression + +on: + workflow_dispatch: + inputs: + verify_engine: + description: Require the already-managed inference engine health endpoint + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: sage-mate-ascend-npu-host + cancel-in-progress: false + +jobs: + host-regression: + if: >- + github.repository == 'SAGE-Research/sage-mate' && + github.ref == 'refs/heads/main' && + github.event_name == 'workflow_dispatch' + environment: ascend-npu + runs-on: + group: sage-mate-ascend + labels: + - self-hosted + - Linux + - ARM64 + - ascend + - sage-mate-ephemeral + timeout-minutes: 20 + steps: + - name: Check out the trusted main revision + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + ref: ${{ github.sha }} + + - name: Verify trusted dispatch context + env: + EXPECTED_SHA: ${{ github.sha }} + run: | + test "$GITHUB_EVENT_NAME" = workflow_dispatch + test "$GITHUB_REF" = refs/heads/main + test "$GITHUB_REPOSITORY" = SAGE-Research/sage-mate + test "$(git rev-parse HEAD)" = "$EXPECTED_SHA" + test "${SAGE_ASCEND_CI_EPHEMERAL:-}" = 1 + + - name: Run non-destructive Ascend host regression + env: + VERIFY_ENGINE: ${{ inputs.verify_engine }} + REPORT_PATH: ${{ runner.temp }}/ascend-host-report.json + run: bash tools/verify_ascend_ci_host.sh + + - name: Upload public-safe host report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: ascend-host-regression-${{ github.run_id }} + if-no-files-found: error + retention-days: 14 + path: ${{ runner.temp }}/ascend-host-report.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f794bd..59b110f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## v4.6.31 - 2026-08-15 + +- Added a trusted-main-only Ascend host regression workflow backed by a + workflow-restricted, one-job ephemeral runner instead of exposing the NPU + host to public pull-request code. +- Added non-destructive ARM64/NPU, Docker device-binding, graph-mode, app + health, runner cleanup, and public-safe report gates plus an operator wrapper + that registers, dispatches, watches, and removes one runner automatically. + ## v4.6.30 - 2026-08-15 - Fixed returning-visitor first paint so the welcome identity and three diff --git a/README.md b/README.md index d974736..b53a2ba 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,9 @@ curl -s http://127.0.0.1:55601/healthz ## CI 覆盖 - `ubuntu-latest`: lint、frontend 静态契约、Firefox 多视口布局回归、pytest、Linux CPU 一键安装检查。 -- `ubuntu-latest`: Ascend 启动脚本语法和部署契约检查；真实 NPU 部署需在受控 Ascend 主机上单独执行。 +- `ubuntu-latest`: Ascend 启动脚本语法和部署契约检查。 +- 真实 NPU 主机回归：仅从 `main` 人工触发 `.github/workflows/ascend-npu.yml`，由 + workflow-restricted 的一次性 runner 执行；公共 PR 永远不会直接使用 NPU 宿主机。 ## 入口 diff --git a/docs/deployment.md b/docs/deployment.md index 3597e06..1d1ab68 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -27,6 +27,29 @@ In particular, an Ascend launch requires `VLLM_ENGINE_MODEL_PATH` and release installer can choose currently idle NPUs, while the lower-level launcher deliberately fails instead of guessing. +### Trusted one-shot Ascend CI + +Pull requests always use GitHub-hosted runners. The public repository must not +attach a persistent NPU host to `pull_request` jobs. Hardware regression is a +separate `workflow_dispatch` workflow on `main`, routed through the +workflow-restricted `sage-mate-ascend` runner group and the `ascend-npu` +environment. + +An operator starts one ephemeral runner and dispatches one job with: + +```bash +SAGE_ASCEND_CI_CONTAINER_IMAGE= \ + tools/run_ascend_ci_once.sh +``` + +The job checks the trusted revision, Ascend device/control nodes, `npu-smi`, +non-interactive Docker access, container device binding, graph-mode launcher +contracts, and application health. It does not start or replace the managed +model service. Set `VERIFY_ENGINE=true` only when an already-managed engine is +expected to be healthy. The runner deregisters after its single job and its +temporary work directory is removed; diagnostics are retained under the +operator's XDG cache directory. + The loopback addresses and ports shown in `.env.example` are editable sample configuration, not values embedded in generated systemd units. To move the deployment, copy its secret material separately and generate a new `.env` for the destination host. diff --git a/pyproject.toml b/pyproject.toml index e84868d..009589a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "sage-mate" -version = "4.6.30" +version = "4.6.31" description = "Sage Mate multi-profile local and hosted assistant built on SAGE and vllm-hust" readme = "README.md" license = {file = "LICENSE"} diff --git a/src/sage_faculty_twin/__init__.py b/src/sage_faculty_twin/__init__.py index 4b2e090..ab6a8a6 100644 --- a/src/sage_faculty_twin/__init__.py +++ b/src/sage_faculty_twin/__init__.py @@ -1,5 +1,5 @@ """Sage Mate application package.""" -__version__ = "4.6.30" +__version__ = "4.6.31" __all__ = ["__version__"] diff --git a/tests/test_ascend_ci_contract.py b/tests/test_ascend_ci_contract.py new file mode 100644 index 0000000..e03de93 --- /dev/null +++ b/tests/test_ascend_ci_contract.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import re +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_ascend_workflow_is_manual_main_only_and_read_only() -> None: + workflow = (ROOT / ".github" / "workflows" / "ascend-npu.yml").read_text( + encoding="utf-8" + ) + trigger_block = workflow.split("permissions:", 1)[0] + assert "workflow_dispatch:" in trigger_block + assert "pull_request:" not in trigger_block + assert "push:" not in trigger_block + assert "contents: read" in workflow + assert "github.ref == 'refs/heads/main'" in workflow + assert "github.event_name == 'workflow_dispatch'" in workflow + assert "environment: ascend-npu" in workflow + assert "group: sage-mate-ascend" in workflow + assert "- sage-mate-ephemeral" in workflow + + +def test_ascend_workflow_pins_third_party_actions() -> None: + workflow = (ROOT / ".github" / "workflows" / "ascend-npu.yml").read_text( + encoding="utf-8" + ) + references = re.findall(r"^\s*uses:\s*\S+@([^\s]+)$", workflow, re.MULTILINE) + assert references + assert all(re.fullmatch(r"[0-9a-f]{40}", revision) for revision in references) + + +def test_runner_launcher_requires_ephemeral_restricted_group() -> None: + launcher = (ROOT / "tools" / "run_ascend_ci_once.sh").read_text(encoding="utf-8") + assert "--ephemeral" in launcher + assert "restricted_to_workflows == true" in launcher + assert 'visibility == "selected"' in launcher + assert "@refs/heads/main" in launcher + assert 'orgs/$organization/actions/runners/registration-token' in launcher + assert 'runner-groups/$group_id/repositories' in launcher + assert "runner remained registered" in launcher + + +def test_host_probe_is_non_destructive_and_keeps_graph_mode() -> None: + probe = (ROOT / "tools" / "verify_ascend_ci_host.sh").read_text(encoding="utf-8") + assert "workflow_dispatch" in probe + assert "refs/heads/main" in probe + assert "--network none" in probe + assert "VLLM_ENGINE_ENFORCE_EAGER=0" in probe + assert "--enforce-eager" in probe + assert '"result":"failed"' in probe + assert "systemctl start" not in probe + assert "systemctl restart" not in probe diff --git a/tools/run_ascend_ci_once.sh b/tools/run_ascend_ci_once.sh new file mode 100755 index 0000000..ba9b315 --- /dev/null +++ b/tools/run_ascend_ci_once.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +set -euo pipefail + +repository="${SAGE_ASCEND_CI_REPOSITORY:-SAGE-Research/sage-mate}" +organization="${repository%%/*}" +runner_group="${SAGE_ASCEND_CI_RUNNER_GROUP:-sage-mate-ascend}" +workflow="${SAGE_ASCEND_CI_WORKFLOW:-ascend-npu.yml}" +container_image="${SAGE_ASCEND_CI_CONTAINER_IMAGE:-}" +verify_engine="${VERIFY_ENGINE:-false}" +runner_cache_root="${XDG_CACHE_HOME:-$HOME/.cache}/sage-mate-actions-runner" +runner_runtime="$(mktemp -d "${TMPDIR:-/tmp}/sage-mate-actions-runner.XXXXXX")" +runner_name="sage-mate-ascend-$(hostname -s)-$$" +listener_pid="" + +fail() { + echo "ERROR: $*" >&2 + exit 1 +} + +cleanup() { + if [[ -n "$listener_pid" ]] && kill -0 "$listener_pid" 2>/dev/null; then + kill "$listener_pid" 2>/dev/null || true + wait "$listener_pid" 2>/dev/null || true + fi + if [[ -d "$runner_runtime/_diag" ]]; then + mkdir -p "$runner_cache_root/diag" + cp -a "$runner_runtime/_diag/." "$runner_cache_root/diag/" 2>/dev/null || true + fi + rm -rf -- "$runner_runtime" +} +trap cleanup EXIT INT TERM + +command -v gh >/dev/null || fail "gh is required" +command -v curl >/dev/null || fail "curl is required" +command -v jq >/dev/null || fail "jq is required" +command -v sha256sum >/dev/null || fail "sha256sum is required" +[[ -n "$container_image" ]] || fail "SAGE_ASCEND_CI_CONTAINER_IMAGE is required" +[[ "$(uname -m)" == "aarch64" ]] || fail "this runner package requires aarch64" + +group_json="$(gh api "orgs/$organization/actions/runner-groups")" +group_id="$(jq -r \ + --arg group "$runner_group" \ + --arg workflow "${repository}/.github/workflows/${workflow}@refs/heads/main" \ + '.runner_groups[] | select(.name == $group) | + select(.visibility == "selected") | + select(.allows_public_repositories == true) | + select(.restricted_to_workflows == true) | + select(.selected_workflows | index($workflow)) | .id' \ + <<<"$group_json")" +[[ "$group_id" =~ ^[0-9]+$ ]] || fail "runner group is not restricted to the trusted main workflow" +gh api "orgs/$organization/actions/runner-groups/$group_id/repositories" \ + --jq ".repositories[] | select(.full_name == \"$repository\") | .full_name" \ + | grep -qx "$repository" || fail "runner group is not restricted to $repository" + +release_json="$(gh api repos/actions/runner/releases/latest)" +runner_tag="$(jq -r '.tag_name' <<<"$release_json")" +runner_version="${runner_tag#v}" +asset_name="actions-runner-linux-arm64-${runner_version}.tar.gz" +asset_url="$(jq -r --arg name "$asset_name" '.assets[] | select(.name == $name) | .url' <<<"$release_json")" +asset_digest="$(jq -r --arg name "$asset_name" '.assets[] | select(.name == $name) | .digest' <<<"$release_json")" +[[ -n "$asset_url" && "$asset_url" != "null" ]] || fail "latest ARM64 runner asset not found" +[[ "$asset_digest" == sha256:* ]] || fail "runner release has no SHA-256 digest" + +archive="$runner_runtime/$asset_name" +gh api -H 'Accept: application/octet-stream' "$asset_url" >"$archive" +echo "${asset_digest#sha256:} $archive" | sha256sum --check --status || fail "runner archive checksum mismatch" +tar -xzf "$archive" -C "$runner_runtime" + +registration_token="$(gh api --method POST "orgs/$organization/actions/runners/registration-token" --jq .token)" +( + cd "$runner_runtime" + ./config.sh \ + --url "https://github.com/$organization" \ + --token "$registration_token" \ + --runnergroup "$runner_group" \ + --name "$runner_name" \ + --labels 'ascend,sage-mate-ephemeral' \ + --work _work \ + --ephemeral \ + --unattended +) + +( + cd "$runner_runtime" + SAGE_ASCEND_CI_EPHEMERAL=1 \ + SAGE_ASCEND_CI_CONTAINER_IMAGE="$container_image" \ + ./run.sh +) & +listener_pid=$! + +for _ in $(seq 1 30); do + if gh api "orgs/$organization/actions/runners" \ + --jq ".runners[] | select(.name == \"$runner_name\" and .status == \"online\") | .name" \ + | grep -qx "$runner_name"; then + break + fi + sleep 1 +done +gh api "orgs/$organization/actions/runners" \ + --jq ".runners[] | select(.name == \"$runner_name\" and .status == \"online\") | .name" \ + | grep -qx "$runner_name" || fail "ephemeral runner did not become online" + +dispatch_started="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +gh workflow run "$workflow" --repo "$repository" --ref main -f "verify_engine=$verify_engine" +run_id="" +for _ in $(seq 1 30); do + run_id="$(gh run list --repo "$repository" --workflow "$workflow" --event workflow_dispatch --branch main \ + --limit 10 --json databaseId,createdAt \ + --jq ".[] | select(.createdAt >= \"$dispatch_started\") | .databaseId" | head -n 1)" + [[ -n "$run_id" ]] && break + sleep 1 +done +[[ -n "$run_id" ]] || fail "dispatched workflow run was not found" + +echo "Watching Ascend workflow run $run_id" +gh run watch "$run_id" --repo "$repository" --exit-status --interval 10 +wait "$listener_pid" +listener_pid="" + +if gh api "orgs/$organization/actions/runners" --jq ".runners[] | select(.name == \"$runner_name\") | .name" | grep -q .; then + fail "ephemeral runner remained registered after its job" +fi +echo "Ascend one-shot CI completed and runner deregistered: run=$run_id" diff --git a/tools/verify_ascend_ci_host.sh b/tools/verify_ascend_ci_host.sh new file mode 100755 index 0000000..1ee863a --- /dev/null +++ b/tools/verify_ascend_ci_host.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +report_path="${REPORT_PATH:-${RUNNER_TEMP:-/tmp}/ascend-host-report.json}" +app_health_url="${SAGE_ASCEND_CI_APP_HEALTH_URL:-http://127.0.0.1:55601/health}" +engine_health_url="${SAGE_ASCEND_CI_ENGINE_HEALTH_URL:-http://127.0.0.1:18001/health}" +verify_engine="${VERIFY_ENGINE:-false}" +container_image="${SAGE_ASCEND_CI_CONTAINER_IMAGE:-}" + +mkdir -p "$(dirname "$report_path")" +printf '{"schema_version":1,"result":"started"}\n' >"$report_path" + +finalize_failure_report() { + local status=$? + if (( status != 0 )); then + printf '{"schema_version":1,"result":"failed"}\n' >"$report_path" + fi + trap - EXIT + exit "$status" +} +trap finalize_failure_report EXIT + +fail() { + echo "ERROR: $*" >&2 + exit 1 +} + +[[ "${GITHUB_EVENT_NAME:-}" == "workflow_dispatch" ]] || fail "hardware regression requires workflow_dispatch" +[[ "${GITHUB_REF:-}" == "refs/heads/main" ]] || fail "hardware regression requires refs/heads/main" +[[ "${SAGE_ASCEND_CI_EPHEMERAL:-}" == "1" ]] || fail "runner is not marked ephemeral" +[[ -n "$container_image" ]] || fail "SAGE_ASCEND_CI_CONTAINER_IMAGE is required" +[[ "$(uname -m)" == "aarch64" ]] || fail "Ascend runner must be aarch64" + +command -v npu-smi >/dev/null || fail "npu-smi is unavailable" +command -v curl >/dev/null || fail "curl is unavailable" +command -v python3 >/dev/null || fail "python3 is unavailable" + +mapfile -t npu_devices < <(find /dev -maxdepth 1 -type c -name 'davinci[0-9]*' -printf '%f\n' | sort -V) +(( ${#npu_devices[@]} > 0 )) || fail "no Ascend NPU device nodes found" +for control_device in /dev/davinci_manager /dev/devmm_svm /dev/hisi_hdc; do + [[ -c "$control_device" ]] || fail "missing control device $control_device" +done +npu-smi info >/dev/null + +docker_cmd=(docker) +if ! docker info >/dev/null 2>&1; then + docker_cmd=(sudo -n docker) + "${docker_cmd[@]}" info >/dev/null 2>&1 || fail "Docker is unavailable without interactive elevation" +fi +"${docker_cmd[@]}" image inspect "$container_image" >/dev/null + +probe_device="/dev/${npu_devices[0]}" +"${docker_cmd[@]}" run --rm --network none \ + --entrypoint /bin/sh \ + --device "$probe_device" \ + --device /dev/davinci_manager \ + --device /dev/devmm_svm \ + --device /dev/hisi_hdc \ + "$container_image" \ + -c "test -c '$probe_device' && test -c /dev/davinci_manager && test -c /dev/devmm_svm && test -c /dev/hisi_hdc" + +for script in \ + tools/run_vllm_engine.sh \ + tools/lock_sage_mate_engine.sh \ + tools/verify_sage_mate_engine.sh \ + tools/retry_deploy_vllm_ascend_until_success.sh; do + bash -n "$repo_root/$script" +done +grep -q 'VLLM_ENGINE_ENFORCE_EAGER=0' "$repo_root/tools/lock_sage_mate_engine.sh" +if grep -Eq -- '(^|[[:space:]])--enforce-eager([[:space:]]|$)' \ + "$repo_root/tools/run_vllm_engine.sh" \ + "$repo_root/tools/lock_sage_mate_engine.sh"; then + fail "formal Ascend launcher enables --enforce-eager" +fi + +app_health="$(curl --fail --silent --show-error --max-time 15 "$app_health_url")" +python3 -c 'import json,sys; data=json.load(sys.stdin); assert data.get("status") == "ok"' <<<"$app_health" + +engine_status="not_requested" +if [[ "$verify_engine" == "true" ]]; then + curl --fail --silent --show-error --max-time 15 "$engine_health_url" >/dev/null + engine_status="healthy" +fi + +REPORT_PATH="$report_path" \ +APP_HEALTH="$app_health" \ +ENGINE_STATUS="$engine_status" \ +NPU_COUNT="${#npu_devices[@]}" \ +CONTAINER_IMAGE="$container_image" \ +python3 - <<'PY' +import json +import os +from pathlib import Path + +health = json.loads(os.environ["APP_HEALTH"]) +report = { + "schema_version": 1, + "result": "passed", + "architecture": "aarch64", + "npu_device_count": int(os.environ["NPU_COUNT"]), + "container_image": os.environ["CONTAINER_IMAGE"], + "container_device_binding": "passed", + "graph_mode_contract": "passed", + "app_health": { + "status": health.get("status"), + "app_version": health.get("app_version"), + }, + "engine_health": os.environ["ENGINE_STATUS"], +} +Path(os.environ["REPORT_PATH"]).write_text( + json.dumps(report, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", +) +PY + +trap - EXIT + +echo "Ascend host regression passed: ${#npu_devices[@]} device(s), image=$container_image" +echo "Report: $report_path" diff --git a/uv.lock b/uv.lock index bbd94d4..e00ba58 100644 --- a/uv.lock +++ b/uv.lock @@ -2613,7 +2613,7 @@ wheels = [ [[package]] name = "sage-mate" -version = "4.6.30" +version = "4.6.31" source = { editable = "." } dependencies = [ { name = "cloudpickle" },