From ede5d7dff1a41d30201d57b9601fdf5273831b2d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 20:37:55 -0500 Subject: [PATCH 1/6] fix(install): install mliport into .venv-nep and finish NEP docs integration (#61) P0: _nep_steps() installed only calorine, ASE and numpy, so the verify stage failed with FileNotFoundError on .venv-nep/bin/mliport (the inventory stage only worked because the bootstrap script leaks PYTHONPATH). The NEP step is now one resolver run with editable ./mliport[analysis-all] plus the pinned calorine==4.1, built from the same _uv_pip / build_package_source_args path as the other backends. This also makes --source china/offline/custom effective for NEP: the old PIP_EXTRA_INDEX_URL env was stripped by the managed source profiles and profile.index_url does not exist. P1: README.md/README_CN.md gain NEP install, environment table, example and limits; the generated environment table no longer prints "calorine==4.1 (12.9)" (NumPy/ASE resolved by the installer vs CUDA Toolkit 12.9 used to build gpumd); the manual NEP install route includes mliport; docs state that doctor reads MLIPORT_GPUMD_COMMAND (not NEP_GPUMD_COMMAND) and that a GPU-host install without gpumd stops at the closing doctor. Tests: plan-level NEP tests (editable install, pin, order, mirror/offline/custom, clean scope, isolation, other backends unchanged), docs consistency tests, and a gated real-install suite (MLIPORT_REAL_INSTALL=1) run by the new installer-real-nep CI job. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01GYQHUEKJ91XQC4Hzj11Q1X --- .github/workflows/test.yml | 31 ++ CHANGELOG.md | 19 + README.md | 37 +- README_CN.md | 28 +- docs/backends/nep.md | 21 +- docs/backends/nep.zh.md | 18 +- docs/installation.md | 33 +- docs/installation.zh.md | 28 +- mliport/README.md | 37 +- mliport/mliport/install/compatibility.py | 32 +- mliport/mliport/install/docs.py | 2 +- mliport/mliport/install/plan.py | 60 ++- scripts/generate_docs.py | 6 +- scripts/install_mliport.sh | 2 +- .../mliport/mliport/test_install_nep_real.py | 442 ++++++++++++++++++ tests/mliport/mliport/test_install_plan.py | 122 +++++ .../mliport/test_install_ux_contract.py | 45 ++ 17 files changed, 894 insertions(+), 69 deletions(-) create mode 100644 tests/mliport/mliport/test_install_nep_real.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 42189179..7f54488e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -102,3 +102,34 @@ jobs: raise SystemExit("direct-force models must be rejected") print("capability wheel smoke ok on", sys.version) PY + + # Really executes scripts/install_mliport.sh for the NEP backend (network, + # uv, a fresh .venv-nep) and runs doctor from the result. Plan-level checks + # live in tests/mliport/mliport/test_install_plan.py and run everywhere. + installer-real-nep: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v5 + - uses: astral-sh/setup-uv@v7 + with: + python-version: "3.12" + enable-cache: true + cache-dependency-glob: "mliport/pyproject.toml" + - name: Install pytest only + run: | + uv venv .venv-ci --python 3.12 + uv pip install --python .venv-ci/bin/python pytest + - name: Real NEP install, doctor, reinstall, mirror, offline + env: + MLIPORT_REAL_INSTALL: "1" + MLIPORT_REAL_INSTALL_LOGDIR: ${{ runner.temp }}/nep-install-logs + run: | + .venv-ci/bin/python -m pytest tests/mliport/mliport/test_install_nep_real.py -v -rA -p no:cacheprovider + - name: Upload install logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: nep-install-logs + path: ${{ runner.temp }}/nep-install-logs + if-no-files-found: warn diff --git a/CHANGELOG.md b/CHANGELOG.md index f7e73612..1ea333c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ ## Unreleased +NEP installer and documentation integration (tracking #61): + +- **`--engines nep` now installs mliport into `.venv-nep`.** The NEP install step + only installed `calorine`, ASE and NumPy, so the verify stage failed with + `FileNotFoundError` on `.venv-nep/bin/mliport`. It is now one resolver run with + an editable `./mliport[analysis-all]` plus the pinned `calorine==4.1`, built from + the same `_uv_pip` and package-source arguments as the other backends. The NEP + step previously set `PIP_EXTRA_INDEX_URL` (which the installer strips for managed + profiles) and read a non-existent `profile.index_url`, so `--source china*` and + `--source offline` were silently ignored for NEP; they are honoured now. NEP stays + opt-in and isolated in `.venv-nep`; the plans of the other backends are unchanged. + The installer still does not build `gpumd`. +- **Docs.** `README.md`/`README_CN.md` describe the NEP backend (install, environment + table, example, limits). The generated environment table no longer prints + `calorine==4.1 (12.9)`: for NEP the NumPy/ASE dependency is resolved by the + installer and 12.9 is the CUDA Toolkit used to build `gpumd`. The manual NEP + install route now includes mliport, and the docs state that `doctor` finds + `gpumd` on `PATH` or via `MLIPORT_GPUMD_COMMAND` (not `NEP_GPUMD_COMMAND`). + Internal (no user-visible behaviour change): - **Shared runner safety helpers.** The numerical-safety gates that MD already diff --git a/README.md b/README.md index d3d2dd9c..45e8a2ea 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ **English** | [简体中文](README_CN.md) -Run machine-learned interatomic potentials — MACE, DPA, GRACE and UMA — from one +Run machine-learned interatomic potentials — MACE, DPA, GRACE, UMA and NEP — from one command line, one terminal UI and one Python API. Give it a structure and a checkpoint; it relaxes, runs molecular dynamics, finds NEB barriers, searches adsorbate configurations on a slab or analyzes a trajectory, and writes the files a VASP user expects: `OUTCAR`, `OSZICAR`, @@ -14,10 +14,12 @@ your chemistry depends on the checkpoint you choose. What you get: -- **The same workflow on four backends.** Single point, fixed- and variable-cell +- **The same workflow on every backend.** Single point, fixed- and variable-cell relaxation, NVE/NVT MD (Langevin, Bussi, Nosé–Hoover chain), NEB and CI-NEB, MD-based configuration search for an adsorbate on a slab, batch sweeps and a job - queue, from the CLI, an INCAR-style file, the TUI or Python. + queue, from the CLI, an INCAR-style file, the TUI or Python. NEP's native `gpumd` + runtime covers only single point, relaxation and MD (see + [NEP](docs/backends/nep.md)). - **No silent guessing.** There is no default backend, and a misspelled option, or one that belongs to another backend, stops the run. Configuration errors fail closed. @@ -37,7 +39,7 @@ The full documentation is at (English | run my first calculation | [Quickstart](docs/quickstart.md) | | check a model on a system with a known answer | [Test systems](docs/test-systems.md) | | install one or more backends, on GPU or CPU | [Installation](docs/installation.md) | -| understand what MACE, DPA, GRACE and UMA are and pick one | [Backends](docs/backends/index.md), [Models](docs/models.md) | +| understand what MACE, DPA, GRACE, UMA and NEP are and pick one | [Backends](docs/backends/index.md), [Models](docs/models.md) | | set options through the CLI, an INCAR-style file or `settings.ini` | [Configuration](docs/configuration.md) | | single point, relaxation, MD, NEB, configuration search, batch, queue | [Workflows](docs/workflows/index.md) | | MSD, diffusion, conductivity, RDF, spectra, Arrhenius | [Analysis](docs/analysis/index.md) | @@ -60,6 +62,9 @@ cd mliport # or CPU only ./scripts/install_mliport.sh --engines mace dpa grace uma --device cpu + +# NEP is opt-in (add it explicitly, alone or with the others) +./scripts/install_mliport.sh --engines nep --device cpu ``` The installer pins the upstream backend versions, installs the analysis extras and @@ -76,6 +81,15 @@ together with CPU, offline and proxy setups and uninstalling. | DPA | `.venv-dpa` | `bin/mliport-dpa` | | GRACE | `.venv-grace` | `bin/mliport-grace` | | UMA | `.venv` | `bin/mliport-uma` | +| NEP | `.venv-nep` | `bin/mliport-nep` | + +NEP is not part of the default engine list. The installer puts the pinned `calorine` +and an editable mliport into `.venv-nep`, which is enough for the CPU ASE runtime +(`CPUNEP`, float64). The native runtime and `GPUNEP` also need a `gpumd` executable, +which the installer does **not** build (it needs your CUDA toolkit and a matching +`-arch=sm_XX`). On a GPU host, `--device auto|cuda` makes the closing `doctor` demand `gpumd`, so +use `--device cpu` until you have built it; see [docs/backends/nep.md](docs/backends/nep.md) for the build and for +`mliport doctor --engine nep`. Run mliport from the environment of the backend you want, or through its launcher: @@ -99,6 +113,7 @@ path with `--model`; a local directory works offline. | DPA | `DPA-3.1-3M.pt` | [deepmodelingcommunity/DPA-3.1-3M](https://huggingface.co/deepmodelingcommunity/DPA-3.1-3M) | | GRACE | `GRACE-2L-OMAT-medium-base` | [AMS-ICAMS-RUB/grace-foundation-models](https://huggingface.co/AMS-ICAMS-RUB/grace-foundation-models) | | UMA | `uma-s-1p2.pt` | [facebook/UMA](https://huggingface.co/facebook/UMA) | +| NEP | your own `nep.txt` | trained for your elements and conditions; mliport ships and downloads none | The hashes the validation suite uses are pinned in [`validation/science/model_manifest.json`](validation/science/model_manifest.json). @@ -135,6 +150,7 @@ fails; mliport never falls back to another model on its own. .venv-dpa/bin/mliport sp LGPS.vasp --model DPA-3.1-3M.pt --model-type dpa --task bulk --head Omat24 --device cuda .venv-grace/bin/mliport sp LGPS.vasp --model GRACE-2L-OMAT-medium-base --model-type grace --task bulk --device cuda .venv/bin/mliport sp LGPS.vasp --model uma-s-1p2.pt --model-type uma --task omat --device cuda +.venv-nep/bin/mliport sp LGPS.vasp --model nep.txt --model-type nep --nep-runtime ase --device cpu ``` | Engine | `--model-type` | Task / head | Good to know | @@ -143,6 +159,7 @@ fails; mliport never falls back to another model on its own. | DPA | `dpa` | `bulk`, `--head Omat24` | multitask checkpoint; a missing head is an error, not a guess | | GRACE | `grace` | `bulk` | TensorFlow runtime, CPU and GPU | | UMA | `uma` (`fairchem` is an alias) | `omat` | `--task omol` for molecules; non-commercial research license | +| NEP | `nep` + required `--nep-runtime ase\|gpumd` | `bulk` (PBC only) | runtime is never inferred; `ase` = Calorine (`cpu` float64, `cuda` float32 via one `gpumd` process per call), `gpumd` = one native process per run (float32) | DPA and GRACE cannot change GPU after their framework has loaded, so when `CUDA_VISIBLE_DEVICES` is unset the CLI restarts the command in a fresh process with @@ -255,8 +272,8 @@ Options for each task are in [docs/analysis/index.md](docs/analysis/index.md). ## Configuration -The backend is always explicit (`--model-type mace|dpa|grace|uma` plus `--model`, or -an alias). Configuration is strict: unknown and cross-backend keys are errors, and +The backend is always explicit (`--model-type mace|dpa|grace|uma|nep` plus `--model`, +or an alias; `nep` also needs `--nep-runtime`). Configuration is strict: unknown and cross-backend keys are errors, and `--lenient-config` downgrades them for exploratory runs. `mliport config` inspects settings; a project `settings.ini` can define aliases, and `--settings PATH` wins over the user and project files. Each run writes `resolved_config.json` with the @@ -273,6 +290,14 @@ entry point (API, direct CLI, INCAR, TUI) can express is recorded in `validation/science/capability_matrix.json`; a `no` or `partial` there means that entry point cannot say it, and nothing is translated silently. +NEP limits: you supply the `nep.txt`, and mliport cannot say whether it fits your +material. The native `gpumd` runtime has no CPU path and refuses NEB, batch and +confsearch instead of switching to the ASE runtime; DFT-D3 is rejected; `qNEP`, +`TNEP` and `nep4-vdw` models are unsupported; partial PBC is refused. NEP is not part +of the validation campaign summarised below: its real-hardware evidence is a V100 with +GPUMD 5.9.1 and CUDA 12.9, and other GPUs and toolkits are unverified. Details: +[docs/backends/nep.md](docs/backends/nep.md). + Model accuracy belongs to the upstream checkpoint. Short MD runs check the workflow, not equilibration. NEB's pre-flight shows the forces match the energy, which says nothing about how accurate the barrier is. Transport analyses fail closed with an diff --git a/README_CN.md b/README_CN.md index e1b7cc63..a60f46a1 100644 --- a/README_CN.md +++ b/README_CN.md @@ -2,7 +2,7 @@ [English](README.md) | **简体中文** -mliport 用一套命令行、终端界面(TUI)和 Python API 运行四种机器学习势:MACE、DPA、GRACE 和 UMA。 +mliport 用一套命令行、终端界面(TUI)和 Python API 运行五种机器学习势:MACE、DPA、GRACE、UMA 和 NEP。 准备好结构和模型文件,就可以做结构优化、分子动力学、NEB 势垒、slab 上吸附分子的构型搜索和轨迹分析。 输出沿用 VASP 的文件格式:`OUTCAR`、`OSZICAR`、`CONTCAR`、`XDATCAR`。 @@ -11,9 +11,9 @@ POTCAR。模型对你的体系准不准,取决于你选的模型文件。 主要功能: -- **四个后端,用法一致。** 单点、结构优化(固定或可变晶胞)、NVE/NVT 分子动力学(Langevin、Bussi、 +- **各后端用法一致。** 单点、结构优化(固定或可变晶胞)、NVE/NVT 分子动力学(Langevin、Bussi、 Nosé–Hoover 链)、NEB 和 CI-NEB、slab 吸附构型搜索、批量计算和任务队列。可以用命令行、INCAR 风格 - 的输入文件、TUI 或 Python 调用。 + 的输入文件、TUI 或 Python 调用。NEP 的原生 `gpumd` 运行时只支持单点、结构优化和 MD(见 [NEP](docs/backends/nep.zh.md))。 - **不做默认猜测。** 没有默认后端。选项拼错,或者用了别的后端的选项,程序会报错停下。 - **完整的计算记录。** 每次运行都记录模型文件和它的 SHA-256、请求的设备和实际使用的设备、最终生效 的配置以及 mliport 版本。 @@ -31,7 +31,7 @@ POTCAR。模型对你的体系准不准,取决于你选的模型文件。 | 跑第一个计算 | [快速上手](docs/quickstart.zh.md) | | 在答案已知的体系上检验一个模型 | [测试体系](docs/test-systems.zh.md) | | 安装一个或多个后端(GPU 或 CPU) | [安装](docs/installation.zh.md) | -| 了解 MACE、DPA、GRACE、UMA 是什么并选一个 | [后端](docs/backends/index.zh.md)、[模型](docs/models.zh.md) | +| 了解 MACE、DPA、GRACE、UMA、NEP 是什么并选一个 | [后端](docs/backends/index.zh.md)、[模型](docs/models.zh.md) | | 通过 CLI、INCAR 风格文件或 `settings.ini` 设置选项 | [配置](docs/configuration.zh.md) | | 单点、弛豫、MD、NEB、构型搜索、批量、队列 | [工作流](docs/workflows/index.zh.md) | | MSD、扩散、电导率、RDF、谱、Arrhenius | [分析](docs/analysis/index.zh.md) | @@ -54,6 +54,9 @@ cd mliport # 或者只用 CPU ./scripts/install_mliport.sh --engines mace dpa grace uma --device cpu + +# NEP 默认不装,需要显式加上(可单独装,也可和其他后端一起装) +./scripts/install_mliport.sh --engines nep --device cpu ``` 安装脚本会锁定各后端的上游版本,装好分析用的依赖,最后对每个后端运行一次 `mliport doctor`。四个后端 @@ -67,6 +70,13 @@ GPU 要用不同的安装包,见 [docs/installation.md](docs/installation.md) | DPA | `.venv-dpa` | `bin/mliport-dpa` | | GRACE | `.venv-grace` | `bin/mliport-grace` | | UMA | `.venv` | `bin/mliport-uma` | +| NEP | `.venv-nep` | `bin/mliport-nep` | + +NEP 不在默认后端列表里。安装器会在 `.venv-nep` 里装好锁定版本的 `calorine` 和 editable 的 mliport,这足够 +运行 CPU 的 ASE 运行时(`CPUNEP`,float64)。原生运行时和 `GPUNEP` 还需要 `gpumd` 可执行文件,安装器**不会** +编译它(要用你的 CUDA toolkit 和匹配的 `-arch=sm_XX`)。在有 GPU 的机器上,`--device auto|cuda` 会让结尾的 `doctor` 要求 `gpumd`, +编译好之前请用 `--device cpu`;编译方法和 `mliport doctor --engine nep` 见 +[docs/backends/nep.zh.md](docs/backends/nep.zh.md)。 用哪个后端,就在哪个后端的环境里运行 mliport,或者用它的启动脚本: @@ -89,6 +99,7 @@ mliport 不会自动下载模型。请从上游下载,再用 `--model` 传入 | DPA | `DPA-3.1-3M.pt` | [deepmodelingcommunity/DPA-3.1-3M](https://huggingface.co/deepmodelingcommunity/DPA-3.1-3M) | | GRACE | `GRACE-2L-OMAT-medium-base` | [AMS-ICAMS-RUB/grace-foundation-models](https://huggingface.co/AMS-ICAMS-RUB/grace-foundation-models) | | UMA | `uma-s-1p2.pt` | [facebook/UMA](https://huggingface.co/facebook/UMA) | +| NEP | 你自己的 `nep.txt` | 针对你的元素和条件训练;mliport 不附带也不下载 | 验证时用的模型哈希记录在 [`validation/science/model_manifest.json`](validation/science/model_manifest.json)。 模型的许可证以上游为准,UMA 只能用于非商业研究。Hugging Face 镜像和 SOCKS 代理的设置见 @@ -119,6 +130,7 @@ mliport 不会自动下载模型。请从上游下载,再用 `--model` 传入 .venv-dpa/bin/mliport sp LGPS.vasp --model DPA-3.1-3M.pt --model-type dpa --task bulk --head Omat24 --device cuda .venv-grace/bin/mliport sp LGPS.vasp --model GRACE-2L-OMAT-medium-base --model-type grace --task bulk --device cuda .venv/bin/mliport sp LGPS.vasp --model uma-s-1p2.pt --model-type uma --task omat --device cuda +.venv-nep/bin/mliport sp LGPS.vasp --model nep.txt --model-type nep --nep-runtime ase --device cpu ``` | 引擎 | `--model-type` | Task / head | 说明 | @@ -127,6 +139,7 @@ mliport 不会自动下载模型。请从上游下载,再用 `--model` 传入 | DPA | `dpa` | `bulk`、`--head Omat24` | 多任务模型,必须指定 head | | GRACE | `grace` | `bulk` | 基于 TensorFlow,支持 CPU 和 GPU | | UMA | `uma`(也可写 `fairchem`) | `omat` | 分子用 `--task omol`;仅限非商业研究 | +| NEP | `nep`,另需 `--nep-runtime ase\|gpumd` | `bulk`(仅周期性体系) | 运行时从不自动推断;`ase` 为 Calorine(`cpu` 为 float64,`cuda` 为 float32,每次调用起一个 `gpumd` 进程),`gpumd` 为每次运行一个原生进程(float32) | DPA 和 GRACE 的框架加载后就不能再换 GPU。没有设置 `CUDA_VISIBLE_DEVICES` 时,命令行会新开一个设好 GPU 的进程来运行;在 Python 里调用时,请在导入这些框架之前设置 `CUDA_VISIBLE_DEVICES`。详见 @@ -225,7 +238,7 @@ result = run_single_point("LGPS.vasp", "model.model", model_type="mace", device= ## 配置 -后端必须明确指定(`--model-type mace|dpa|grace|uma` 加 `--model`,或者用模型别名)。配置检查是严格的: +后端必须明确指定(`--model-type mace|dpa|grace|uma|nep` 加 `--model`,或者用模型别名;`nep` 还需要 `--nep-runtime`)。配置检查是严格的: 不认识的参数和别的后端的参数都会报错,试算时可以用 `--lenient-config` 降为警告。`mliport config` 用来查看 设置;项目里的 `settings.ini` 可以定义别名,`--settings PATH` 指定的文件优先于用户和项目配置。每次运行都 会写出 `resolved_config.json`,记录每个参数的取值和来源。 @@ -239,6 +252,11 @@ INCAR 风格输入只支持常用参数。API、命令行、INCAR 和 TUI 各自 `validation/science/capability_matrix.json` 里;标为 `no` 或 `partial` 的项在该入口无法设置,程序也不会 自动换成别的设置。 +NEP 的限制:`nep.txt` 要你自己提供,mliport 无法判断它是否适合你的材料。原生 `gpumd` 运行时没有 CPU 路径, +遇到 NEB、批量和 confsearch 会直接拒绝,不会改用 ASE 运行时;DFT-D3 会被拒绝;不支持 `qNEP`、`TNEP` 和 +`nep4-vdw` 模型;部分周期性边界会被拒绝。NEP 不在下面这份验证汇总的范围内:它在真实硬件上的证据是一台 V100 +配 GPUMD 5.9.1 和 CUDA 12.9,其他 GPU 和 toolkit 没有验证过。详见 [docs/backends/nep.zh.md](docs/backends/nep.zh.md)。 + 模型精度由上游模型决定。短时间 MD 只用来检查流程能否跑通,不代表体系已经平衡。NEB 预检通过只说明力和 能量自洽,不代表势垒准确。采样不足时,输运分析会报告 insufficient-sampling,不给出数值。 diff --git a/docs/backends/nep.md b/docs/backends/nep.md index 420fcd8b..2d608684 100644 --- a/docs/backends/nep.md +++ b/docs/backends/nep.md @@ -28,7 +28,7 @@ the precision behind your back. | Environment | Python | Runtime stack | |---|---|---| -| `.venv-nep` | 3.10, 3.11, 3.12, 3.13 | `calorine==4.1` (numpy 12.9) | +| `.venv-nep` | 3.10, 3.11, 3.12, 3.13 | `calorine==4.1` (NumPy/ASE, resolved by the installer; gpumd built with CUDA Toolkit 12.9) | ```bash ./scripts/install_mliport.sh --engines nep --device auto @@ -42,11 +42,15 @@ the precision behind your back. |---|---|---|---|---|---| -NEP lives in its own environment because Calorine is a compiled extension: +NEP lives in its own environment because Calorine is a compiled extension. The +installer above creates it with the pinned `calorine` **and** an editable +mliport, so `.venv-nep/bin/mliport` exists. By hand, the equivalent is one +resolver run from the repository root (an environment without mliport has no +`mliport` command and cannot run `doctor`): ```bash uv venv .venv-nep --python 3.12 -uv pip install --python .venv-nep/bin/python calorine ase +uv pip install --python .venv-nep/bin/python -e './mliport[analysis-all]' calorine==4.1 ``` The native runtime additionally needs the `gpumd` executable, which is *not* @@ -60,13 +64,20 @@ source /path/to/cuda/env.sh # provides nvcc + libcublas/libcusolver/lib make CUDA_ARCH="-arch=sm_70" -j16 # sm_70 = V100; pick your own architecture ``` -Then point mliport at it with `NEP_GPUMD_COMMAND` / `--gpumd-command`, or put it -on `PATH`. Verify both runtimes at once: +mliport was verified with GPUMD 5.9.1 and CUDA 12.9 on a V100 (`sm_70`); other +GPUs and toolkits are unverified (see the table of `-arch` values in the +installer's compatibility registry). Then point mliport at the binary with +`NEP_GPUMD_COMMAND` / `--gpumd-command`, or put it on `PATH`. `mliport doctor` +looks for `gpumd` on `PATH` or in the `MLIPORT_GPUMD_COMMAND` environment +variable (it does not read `NEP_GPUMD_COMMAND`). Verify both runtimes at once: ```bash .venv-nep/bin/mliport doctor --engine nep --device cuda:0 ``` +Without `gpumd`, `--device cpu` still passes (Calorine `CPUNEP` needs none) and +`--device cuda` fails with a message that names the missing executable. + `doctor --engine nep` reports the Calorine/ASE versions, the resolved `gpumd` path and SHA-256, and the first lines of its version banner. It does **not** load a model: a NEP run additionally needs your `nep.txt`. diff --git a/docs/backends/nep.zh.md b/docs/backends/nep.zh.md index 97928d66..37db8b91 100644 --- a/docs/backends/nep.zh.md +++ b/docs/backends/nep.zh.md @@ -23,7 +23,7 @@ mliport 不会推断 runtime、不会从设备反推 runtime,也不会在两 | 环境 | Python | 运行时 | |---|---|---| -| `.venv-nep` | 3.10, 3.11, 3.12, 3.13 | `calorine==4.1` (numpy 12.9) | +| `.venv-nep` | 3.10, 3.11, 3.12, 3.13 | `calorine==4.1`(NumPy/ASE 由安装器解析;gpumd 需用 CUDA Toolkit 12.9 编译) | ```bash ./scripts/install_mliport.sh --engines nep --device auto @@ -37,11 +37,14 @@ mliport 不会推断 runtime、不会从设备反推 runtime,也不会在两 |---|---|---|---|---|---| -Calorine 是编译扩展,因此 NEP 使用独立环境: +Calorine 是编译扩展,因此 NEP 使用独立环境。上面的安装脚本会在该环境里装入固定版本的 +`calorine` **和**可编辑安装的 mliport,所以 `.venv-nep/bin/mliport` 一定存在。手动安装时, +等价做法是在仓库根目录用同一次解析装好两者(不含 mliport 的环境没有 `mliport` 命令,也无法 +运行 `doctor`): ```bash uv venv .venv-nep --python 3.12 -uv pip install --python .venv-nep/bin/python calorine ase +uv pip install --python .venv-nep/bin/python -e './mliport[analysis-all]' calorine==4.1 ``` 原生路径还需要 `gpumd` 可执行文件。mliport 安装器**不**自动构建它(需要与本机 GPU 匹配 @@ -54,13 +57,18 @@ source /path/to/cuda/env.sh # 提供 nvcc 与 libcublas/libcusolver/lib make CUDA_ARCH="-arch=sm_70" -j16 # sm_70 = V100,按你的 GPU 选择 ``` -然后用 `NEP_GPUMD_COMMAND` / `--gpumd-command` 指向它,或把它放进 `PATH`。 -一次性检查两条路径: +mliport 在 V100(`sm_70`)上用 GPUMD 5.9.1 与 CUDA 12.9 验证过;其他 GPU 和工具包版本未经 +验证(各架构的 `-arch` 取值见安装器兼容性注册表)。然后用 `NEP_GPUMD_COMMAND` / +`--gpumd-command` 指向该二进制,或把它放进 `PATH`。`mliport doctor` 在 `PATH` 或环境变量 +`MLIPORT_GPUMD_COMMAND` 里查找 `gpumd`(它不读取 `NEP_GPUMD_COMMAND`)。一次性检查两条路径: ```bash .venv-nep/bin/mliport doctor --engine nep --device cuda:0 ``` +没有 `gpumd` 时,`--device cpu` 仍然通过(Calorine `CPUNEP` 不需要它),`--device cuda` 会失败, +并在报错里点明缺少可执行文件。 + `doctor --engine nep` 会报告 Calorine/ASE 版本、解析到的 `gpumd` 路径与 SHA-256,以及其 版本横幅首行。它**不会**加载模型:真正跑 NEP 还需要你的 `nep.txt`。 diff --git a/docs/installation.md b/docs/installation.md index 3a1f08fd..029f3b42 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,7 +1,7 @@ # Installation mliport does not live in one big environment. Each backend (MACE, DPA, GRACE, -UMA) pulls in its own deep-learning stack, and those stacks do not always get +UMA, and the opt-in NEP) pulls in its own stack, and those stacks do not always get along, so the installer builds **one environment per backend** and installs mliport into each. The `mliport` you run from `.venv-mace` can only run MACE; the one in `.venv` can only run UMA, and so on. Pick the backends you need, @@ -30,6 +30,26 @@ On a machine without one: ./scripts/install_mliport.sh --engines mace dpa grace uma --device cpu ``` +NEP is opt-in: it is not in the default engine list, because its native runtime +also needs a `gpumd` binary that you build yourself (see +[NEP](backends/nep.md)). Add it explicitly: + +```bash +./scripts/install_mliport.sh --engines nep --device cpu # CPU ASE runtime (Calorine CPUNEP) +./scripts/install_mliport.sh --engines nep --device auto # GPU machine; the closing doctor needs your gpumd build +``` + +Like every other backend, `.venv-nep` gets the pinned `calorine` and an editable +mliport in one resolver run, so `.venv-nep/bin/mliport` exists and `doctor` can +run. The installer does not compile `gpumd`. With `--device cpu` the install +needs no `gpumd`: the CPU ASE runtime works immediately. With `--device cuda`, or +`--device auto` on a machine with an NVIDIA GPU, the closing `doctor` checks the +CUDA path and fails naming `gpumd` until one is on `PATH` or in +`MLIPORT_GPUMD_COMMAND`. The environment is already installed by then, but the +`bin/mliport-nep` launcher is only written after every step succeeds, so either +build `gpumd` and re-run the installer, or install with `--device cpu` / +`--skip-doctor` (and call `.venv-nep/bin/mliport` directly in the meantime). + Pass fewer engines if you only need some. `--dry-run` prints every step without executing anything. Unless you give `--skip-doctor`, the installer runs `mliport doctor` in each environment when it finishes. The analysis, transport @@ -53,10 +73,15 @@ any scientific option for you. | UMA (FAIRChem) | `.venv` | 3.11, 3.12, 3.13 | `fairchem-core==2.21.0` (torch 2.8.0) | | MACE | `.venv-mace` | 3.10, 3.11, 3.12, 3.13 | `mace-torch==0.3.16` (torch 2.8.0) | | DPA (DeepMD-kit) | `.venv-dpa` | 3.10, 3.11, 3.12, 3.13 | `deepmd-kit==3.1.3` (torch 2.10.0) | -| NEP (Calorine / GPUMD) | `.venv-nep` | 3.10, 3.11, 3.12, 3.13 | `calorine==4.1` (numpy 12.9) | +| NEP (Calorine / GPUMD) | `.venv-nep` | 3.10, 3.11, 3.12, 3.13 | `calorine==4.1` (NumPy/ASE, resolved by the installer; gpumd built with CUDA Toolkit 12.9) | | GRACE | `.venv-grace` | 3.10, 3.11, 3.12, 3.13 | `tensorpotential==0.6.0` (tensorflow 2.20.0) | +For NEP the parenthesis names two different things: Calorine runs on NumPy and +ASE, which the installer resolves together with mliport (neither is pinned +separately), while `CUDA Toolkit 12.9` is the toolkit the separately built +`gpumd` binary was verified with. They are not one version. + The `Python` column is what the installer will accept for that backend: the intersection of the Python versions mliport supports, the backend's own `requires-python`, and the Python range of the framework wheels it needs. The @@ -76,6 +101,7 @@ installer never quietly picks a different version. .venv-dpa/bin/mliport doctor --engine dpa --device auto .venv-grace/bin/mliport doctor --engine grace --device auto .venv/bin/mliport doctor --engine uma --device auto +.venv-nep/bin/mliport doctor --engine nep --device cpu # if you installed NEP ``` Each command exits 0 when the required checks pass and ends with a line such as @@ -93,6 +119,7 @@ Call the CLI from the environment that owns the backend: .venv-dpa/bin/mliport sp structure.cif --model model.pt --model-type dpa --head Omat24 .venv-grace/bin/mliport sp structure.cif --model saved_model --model-type grace .venv/bin/mliport sp structure.cif --model uma.pt --model-type uma +.venv-nep/bin/mliport sp structure.cif --model nep.txt --model-type nep --nep-runtime ase --device cpu ``` The launchers in `./bin/` run exactly these commands. On Windows each @@ -167,7 +194,7 @@ installer from fetching a managed interpreter. A four-engine GPU install takes roughly 25–35 GB, most of it PyTorch and TensorFlow; a CPU install is smaller. To remove a backend, delete its -environment directory (`.venv-mace`, `.venv-dpa`, `.venv-grace` or `.venv`). +environment directory (`.venv-mace`, `.venv-dpa`, `.venv-grace`, `.venv-nep` or `.venv`). The repository and your model files are untouched, and the `./bin/mliport-*` launchers go with the `bin/` directory. Run the installer again to recreate an environment. diff --git a/docs/installation.zh.md b/docs/installation.zh.md index 68d76bb7..29157317 100644 --- a/docs/installation.zh.md +++ b/docs/installation.zh.md @@ -1,6 +1,6 @@ # 安装 -四个后端(MACE、DPA、GRACE、UMA)各自依赖不同版本的深度学习框架,放在同一个环境里经常 +各后端(MACE、DPA、GRACE、UMA,以及需要显式选择的 NEP)各自依赖不同的运行时栈,放在同一个环境里经常 冲突。所以安装脚本给**每个后端单独建一个环境**,每个环境里各装一份 mliport。`.venv-mace` 里的 `mliport` 只能跑 MACE,`.venv` 里的只能跑 UMA,其余类推。需要哪个后端就装哪个,用的 时候从对应环境里调用 `mliport`。 @@ -27,6 +27,22 @@ cd mliport ./scripts/install_mliport.sh --engines mace dpa grace uma --device cpu ``` +NEP 需要显式选择:它不在默认引擎列表里,因为它的原生路径还需要你自己编译的 `gpumd` +(见 [NEP](backends/nep.md)): + +```bash +./scripts/install_mliport.sh --engines nep --device cpu # CPU 的 ASE 路径(Calorine CPUNEP) +./scripts/install_mliport.sh --engines nep --device auto # 有 GPU 的机器;结尾的 doctor 需要你编译的 gpumd +``` + +与其他后端一样,`.venv-nep` 在同一次依赖解析里装入固定版本的 `calorine` 和可编辑安装的 +mliport,所以 `.venv-nep/bin/mliport` 一定存在,`doctor` 才能运行。安装脚本不会编译 +`gpumd`。用 `--device cpu` 安装时不需要 `gpumd`,CPU 的 ASE 路径装好即可使用。用 `--device cuda`,或者在 +有 NVIDIA GPU 的机器上用 `--device auto`,结尾的 `doctor` 会检查 CUDA 路径,在 `PATH` 或 +`MLIPORT_GPUMD_COMMAND` 里找不到 `gpumd` 时会报错并点名 `gpumd`。此时环境已经装好,但 +`bin/mliport-nep` 启动器要等所有步骤都成功才会写出;可以编译好 `gpumd` 后重新运行安装脚本,也可以改用 +`--device cpu` 或 `--skip-doctor` 安装(期间直接用 `.venv-nep/bin/mliport`)。 + 只要部分后端时,`--engines` 后面少写几个就行。`--dry-run` 只打印要执行的步骤,不真正安装。 安装结束时脚本会在每个环境里运行一次 `mliport doctor`(加 `--skip-doctor` 可以跳过)。分析、 输运和电解质相关的依赖会一起装好,不用另外安装。 @@ -48,10 +64,14 @@ cd mliport | UMA (FAIRChem) | `.venv` | 3.11, 3.12, 3.13 | `fairchem-core==2.21.0` (torch 2.8.0) | | MACE | `.venv-mace` | 3.10, 3.11, 3.12, 3.13 | `mace-torch==0.3.16` (torch 2.8.0) | | DPA (DeepMD-kit) | `.venv-dpa` | 3.10, 3.11, 3.12, 3.13 | `deepmd-kit==3.1.3` (torch 2.10.0) | -| NEP (Calorine / GPUMD) | `.venv-nep` | 3.10, 3.11, 3.12, 3.13 | `calorine==4.1` (numpy 12.9) | +| NEP (Calorine / GPUMD) | `.venv-nep` | 3.10, 3.11, 3.12, 3.13 | `calorine==4.1`(NumPy/ASE 由安装器解析;gpumd 需用 CUDA Toolkit 12.9 编译) | | GRACE | `.venv-grace` | 3.10, 3.11, 3.12, 3.13 | `tensorpotential==0.6.0` (tensorflow 2.20.0) | +NEP 一行括号里的两个东西不是同一个版本:Calorine 运行在 NumPy 和 ASE 之上,二者由安装器与 +mliport 一起解析(不单独锁定);`CUDA Toolkit 12.9` 则是单独编译的 `gpumd` 二进制所验证过的 +工具包版本。 + `Python` 一列是该后端可用的 Python 版本,取 mliport、后端本身和所需框架三者支持范围的交集。 这张表由兼容性注册表(`mliport/mliport/install/compatibility.py`)自动生成,具体的版本锁定 也写在那里。 @@ -67,6 +87,7 @@ cd mliport .venv-dpa/bin/mliport doctor --engine dpa --device auto .venv-grace/bin/mliport doctor --engine grace --device auto .venv/bin/mliport doctor --engine uma --device auto +.venv-nep/bin/mliport doctor --engine nep --device cpu # 若安装了 NEP ``` 必需的检查都通过时,命令以 0 退出,最后一行类似 `Environment checks passed for MACE on CUDA.`。 @@ -83,6 +104,7 @@ cd mliport .venv-dpa/bin/mliport sp structure.cif --model model.pt --model-type dpa --head Omat24 .venv-grace/bin/mliport sp structure.cif --model saved_model --model-type grace .venv/bin/mliport sp structure.cif --model uma.pt --model-type uma +.venv-nep/bin/mliport sp structure.cif --model nep.txt --model-type nep --nep-runtime ase --device cpu ``` `./bin/` 里的启动脚本执行的就是这些命令。Windows 上每个环境里有一个 `Scripts\mliport.exe`, @@ -144,7 +166,7 @@ Turing 之前的 GPU 依赖 CUDA 12.6 版本,而 PyTorch 从 2.15 起不再提 ## 磁盘占用与卸载 四个后端都装 GPU 版大约占 25–35 GB,大部分是 PyTorch 和 TensorFlow;CPU 版小一些。要删掉 -某个后端,直接删除它的环境目录(`.venv-mace`、`.venv-dpa`、`.venv-grace` 或 `.venv`)即可, +某个后端,直接删除它的环境目录(`.venv-mace`、`.venv-dpa`、`.venv-grace`、`.venv-nep` 或 `.venv`)即可, 仓库和模型文件不受影响。`./bin/mliport-*` 启动脚本可以连同 `bin/` 目录一起删。需要时重新 运行安装脚本即可恢复。 diff --git a/mliport/README.md b/mliport/README.md index c5f72b9c..e1838798 100644 --- a/mliport/README.md +++ b/mliport/README.md @@ -4,7 +4,7 @@ **English** | [简体中文](https://github.com/hydrogen1222/mliport/blob/main/README_CN.md) -Run machine-learned interatomic potentials — MACE, DPA, GRACE and UMA — from one +Run machine-learned interatomic potentials — MACE, DPA, GRACE, UMA and NEP — from one command line, one terminal UI and one Python API. Give it a structure and a checkpoint; it relaxes, runs molecular dynamics, finds NEB barriers, searches adsorbate configurations on a slab or analyzes a trajectory, and writes the files a VASP user expects: `OUTCAR`, `OSZICAR`, @@ -16,10 +16,12 @@ your chemistry depends on the checkpoint you choose. What you get: -- **The same workflow on four backends.** Single point, fixed- and variable-cell +- **The same workflow on every backend.** Single point, fixed- and variable-cell relaxation, NVE/NVT MD (Langevin, Bussi, Nosé–Hoover chain), NEB and CI-NEB, MD-based configuration search for an adsorbate on a slab, batch sweeps and a job - queue, from the CLI, an INCAR-style file, the TUI or Python. + queue, from the CLI, an INCAR-style file, the TUI or Python. NEP's native `gpumd` + runtime covers only single point, relaxation and MD (see + [NEP](https://github.com/hydrogen1222/mliport/blob/main/docs/backends/nep.md)). - **No silent guessing.** There is no default backend, and a misspelled option, or one that belongs to another backend, stops the run. Configuration errors fail closed. @@ -39,7 +41,7 @@ The full documentation is at (English | run my first calculation | [Quickstart](https://github.com/hydrogen1222/mliport/blob/main/docs/quickstart.md) | | check a model on a system with a known answer | [Test systems](https://github.com/hydrogen1222/mliport/blob/main/docs/test-systems.md) | | install one or more backends, on GPU or CPU | [Installation](https://github.com/hydrogen1222/mliport/blob/main/docs/installation.md) | -| understand what MACE, DPA, GRACE and UMA are and pick one | [Backends](https://github.com/hydrogen1222/mliport/blob/main/docs/backends/index.md), [Models](https://github.com/hydrogen1222/mliport/blob/main/docs/models.md) | +| understand what MACE, DPA, GRACE, UMA and NEP are and pick one | [Backends](https://github.com/hydrogen1222/mliport/blob/main/docs/backends/index.md), [Models](https://github.com/hydrogen1222/mliport/blob/main/docs/models.md) | | set options through the CLI, an INCAR-style file or `settings.ini` | [Configuration](https://github.com/hydrogen1222/mliport/blob/main/docs/configuration.md) | | single point, relaxation, MD, NEB, configuration search, batch, queue | [Workflows](https://github.com/hydrogen1222/mliport/blob/main/docs/workflows/index.md) | | MSD, diffusion, conductivity, RDF, spectra, Arrhenius | [Analysis](https://github.com/hydrogen1222/mliport/blob/main/docs/analysis/index.md) | @@ -62,6 +64,9 @@ cd mliport # or CPU only ./scripts/install_mliport.sh --engines mace dpa grace uma --device cpu + +# NEP is opt-in (add it explicitly, alone or with the others) +./scripts/install_mliport.sh --engines nep --device cpu ``` The installer pins the upstream backend versions, installs the analysis extras and @@ -78,6 +83,15 @@ together with CPU, offline and proxy setups and uninstalling. | DPA | `.venv-dpa` | `bin/mliport-dpa` | | GRACE | `.venv-grace` | `bin/mliport-grace` | | UMA | `.venv` | `bin/mliport-uma` | +| NEP | `.venv-nep` | `bin/mliport-nep` | + +NEP is not part of the default engine list. The installer puts the pinned `calorine` +and an editable mliport into `.venv-nep`, which is enough for the CPU ASE runtime +(`CPUNEP`, float64). The native runtime and `GPUNEP` also need a `gpumd` executable, +which the installer does **not** build (it needs your CUDA toolkit and a matching +`-arch=sm_XX`). On a GPU host, `--device auto|cuda` makes the closing `doctor` demand `gpumd`, so +use `--device cpu` until you have built it; see [docs/backends/nep.md](https://github.com/hydrogen1222/mliport/blob/main/docs/backends/nep.md) for the build and for +`mliport doctor --engine nep`. Run mliport from the environment of the backend you want, or through its launcher: @@ -101,6 +115,7 @@ path with `--model`; a local directory works offline. | DPA | `DPA-3.1-3M.pt` | [deepmodelingcommunity/DPA-3.1-3M](https://huggingface.co/deepmodelingcommunity/DPA-3.1-3M) | | GRACE | `GRACE-2L-OMAT-medium-base` | [AMS-ICAMS-RUB/grace-foundation-models](https://huggingface.co/AMS-ICAMS-RUB/grace-foundation-models) | | UMA | `uma-s-1p2.pt` | [facebook/UMA](https://huggingface.co/facebook/UMA) | +| NEP | your own `nep.txt` | trained for your elements and conditions; mliport ships and downloads none | The hashes the validation suite uses are pinned in [`validation/science/model_manifest.json`](https://github.com/hydrogen1222/mliport/blob/main/validation/science/model_manifest.json). @@ -137,6 +152,7 @@ fails; mliport never falls back to another model on its own. .venv-dpa/bin/mliport sp LGPS.vasp --model DPA-3.1-3M.pt --model-type dpa --task bulk --head Omat24 --device cuda .venv-grace/bin/mliport sp LGPS.vasp --model GRACE-2L-OMAT-medium-base --model-type grace --task bulk --device cuda .venv/bin/mliport sp LGPS.vasp --model uma-s-1p2.pt --model-type uma --task omat --device cuda +.venv-nep/bin/mliport sp LGPS.vasp --model nep.txt --model-type nep --nep-runtime ase --device cpu ``` | Engine | `--model-type` | Task / head | Good to know | @@ -145,6 +161,7 @@ fails; mliport never falls back to another model on its own. | DPA | `dpa` | `bulk`, `--head Omat24` | multitask checkpoint; a missing head is an error, not a guess | | GRACE | `grace` | `bulk` | TensorFlow runtime, CPU and GPU | | UMA | `uma` (`fairchem` is an alias) | `omat` | `--task omol` for molecules; non-commercial research license | +| NEP | `nep` + required `--nep-runtime ase\|gpumd` | `bulk` (PBC only) | runtime is never inferred; `ase` = Calorine (`cpu` float64, `cuda` float32 via one `gpumd` process per call), `gpumd` = one native process per run (float32) | DPA and GRACE cannot change GPU after their framework has loaded, so when `CUDA_VISIBLE_DEVICES` is unset the CLI restarts the command in a fresh process with @@ -257,8 +274,8 @@ Options for each task are in [docs/analysis/index.md](https://github.com/hydroge ## Configuration -The backend is always explicit (`--model-type mace|dpa|grace|uma` plus `--model`, or -an alias). Configuration is strict: unknown and cross-backend keys are errors, and +The backend is always explicit (`--model-type mace|dpa|grace|uma|nep` plus `--model`, +or an alias; `nep` also needs `--nep-runtime`). Configuration is strict: unknown and cross-backend keys are errors, and `--lenient-config` downgrades them for exploratory runs. `mliport config` inspects settings; a project `settings.ini` can define aliases, and `--settings PATH` wins over the user and project files. Each run writes `resolved_config.json` with the @@ -275,6 +292,14 @@ entry point (API, direct CLI, INCAR, TUI) can express is recorded in `validation/science/capability_matrix.json`; a `no` or `partial` there means that entry point cannot say it, and nothing is translated silently. +NEP limits: you supply the `nep.txt`, and mliport cannot say whether it fits your +material. The native `gpumd` runtime has no CPU path and refuses NEB, batch and +confsearch instead of switching to the ASE runtime; DFT-D3 is rejected; `qNEP`, +`TNEP` and `nep4-vdw` models are unsupported; partial PBC is refused. NEP is not part +of the validation campaign summarised below: its real-hardware evidence is a V100 with +GPUMD 5.9.1 and CUDA 12.9, and other GPUs and toolkits are unverified. Details: +[docs/backends/nep.md](https://github.com/hydrogen1222/mliport/blob/main/docs/backends/nep.md). + Model accuracy belongs to the upstream checkpoint. Short MD runs check the workflow, not equilibration. NEB's pre-flight shows the forces match the energy, which says nothing about how accurate the barrier is. Transport analyses fail closed with an diff --git a/mliport/mliport/install/compatibility.py b/mliport/mliport/install/compatibility.py index 6b16019a..7e823d9d 100644 --- a/mliport/mliport/install/compatibility.py +++ b/mliport/mliport/install/compatibility.py @@ -695,7 +695,31 @@ def python_versions_for(engine: str) -> tuple[str, ...]: return tuple(v for v in SUPPORTED_PYTHON_VERSIONS if v in constraint) -def backend_environment_rows() -> list[dict[str, str]]: +def runtime_stack_text(engine: str, lang: str = "en") -> str: + """Human-readable runtime stack of one backend for the generated docs. + + For torch/TensorFlow backends the version is the pinned framework wheel. + NEP has no framework wheel: Calorine sits on NumPy/ASE (resolved by the + installer, not pinned here), and the only version in its arch profiles is + the **CUDA Toolkit** the separately built ``gpumd`` binary targets. Those + two must never be rendered as one "numpy 12.9". + """ + backend = BACKENDS[engine] + versions = "/".join(framework_versions(backend)) + if backend.framework == "numpy": + if lang == "zh": + return ( + f"`{backend.requirement}`(NumPy/ASE 由安装器解析;" + f"gpumd 需用 CUDA Toolkit {versions} 编译)" + ) + return ( + f"`{backend.requirement}` (NumPy/ASE, resolved by the installer; " + f"gpumd built with CUDA Toolkit {versions})" + ) + return f"`{backend.requirement}` ({backend.framework} {versions})" + + +def backend_environment_rows(lang: str = "en") -> list[dict[str, str]]: """One row per backend for the generated documentation matrix.""" rows: list[dict[str, str]] = [] for engine, backend in BACKENDS.items(): @@ -706,10 +730,7 @@ def backend_environment_rows() -> list[dict[str, str]]: "label": backend.label, "environment": backend.venv_name, "python": ", ".join(versions), - "runtime": ( - f"`{backend.requirement}` " - f"({backend.framework} {'/'.join(framework_versions(backend))})" - ), + "runtime": runtime_stack_text(engine, lang), } ) return rows @@ -723,4 +744,5 @@ def backend_environment_rows() -> list[dict[str, str]]: "backend_environment_rows", "framework_versions", "python_versions_for", + "runtime_stack_text", ] diff --git a/mliport/mliport/install/docs.py b/mliport/mliport/install/docs.py index d40c0c61..2a678611 100644 --- a/mliport/mliport/install/docs.py +++ b/mliport/mliport/install/docs.py @@ -43,7 +43,7 @@ def render_backend_matrix(lang: str = "en") -> str: "| " + " | ".join(header) + " |", "|---|---|---|---|", ] - for row in backend_environment_rows(): + for row in backend_environment_rows(lang): lines.append( f"| {row['label']} | `{row['environment']}` | {row['python']} " f"| {row['runtime']} |" diff --git a/mliport/mliport/install/plan.py b/mliport/mliport/install/plan.py index ce70f870..321f77bc 100644 --- a/mliport/mliport/install/plan.py +++ b/mliport/mliport/install/plan.py @@ -295,35 +295,43 @@ def _nep_steps( """NEP (Calorine) install steps. Calorine is a compiled C++ extension (numpy + ASE), not a deep-learning - framework, so there is no framework wheel to pin. The native ``gpumd`` - binary is a separate build that mliport deliberately does NOT perform - automatically: it needs a matching CUDA toolkit and an architecture flag, - and a silently mis-architected binary would fail at run time rather than - install time. The plan therefore installs the ASE runtime and records the - prerequisite for the native runtime in a warning, which ``doctor --engine - nep`` re-checks afterwards. + framework, so there is no framework wheel to pin. Like every other + backend, the isolated environment gets the pinned backend package **and** + an editable mliport in a single resolver run: the ``verify`` stage calls + ``/bin/mliport doctor``, which cannot exist unless mliport itself is + installed into that environment. One resolve (rather than separate + ``calorine`` / ``mliport`` installs) keeps ASE and NumPy consistent between + the two; NumPy is deliberately left to the resolver because neither mliport + nor Calorine pins it. The source profile is applied exactly as for the + other backends (mirror index, ``--offline``), so ``--source china`` / + ``offline`` / ``custom`` keep their promised semantics. + + The native ``gpumd`` binary is a separate build that mliport deliberately + does NOT perform automatically: it needs a matching CUDA toolkit and an + architecture flag, and a silently mis-architected binary would fail at run + time rather than install time. The plan therefore installs the ASE runtime + and records the prerequisite for the native runtime in a warning, which + ``doctor --engine nep`` re-checks afterwards. """ python = f"{backend.venv_name}/bin/python" - steps: list[InstallStep] = [ + argv = _uv_pip( + profile, + python, + "-e", + "./mliport[analysis-all]", + *backend.install_packages(), + ) + argv += build_package_source_args(profile) + return [ InstallStep( stage="pip", - description=f"Install {backend.label} ASE runtime (calorine + ASE)", - argv=[ - "uv", - "pip", - "install", - "--python", - python, - backend.requirement, - "ase>=3.23", - "numpy", - ], - env={"PIP_EXTRA_INDEX_URL": profile.index_url} - if getattr(profile, "index_url", None) - else {}, + description=( + f"Install {backend.label} ASE runtime " + "(pinned calorine + editable mliport)" + ), + argv=argv, ) ] - return steps def generate_plan( @@ -464,8 +472,10 @@ def generate_plan( plan.warnings.append( "NEP: the native runtime needs a separately built `gpumd` " "binary (CUDA toolkit + matching -arch=sm_XX). mliport does not " - "build it automatically; set NEP_GPUMD_COMMAND afterwards and " - "confirm with `mliport doctor --engine nep`." + "build it automatically. Without it only the CPU ASE runtime " + "(`NEP_RUNTIME=ase` on cpu) works. After building, set " + "NEP_GPUMD_COMMAND for mliport runs and make `gpumd` visible to " + "`mliport doctor --engine nep` (on PATH or MLIPORT_GPUMD_COMMAND)." ) plan.steps.extend(_nep_steps(backend, src, cpu=is_cpu)) elif engine == "grace": diff --git a/scripts/generate_docs.py b/scripts/generate_docs.py index 373bc025..541a24fb 100644 --- a/scripts/generate_docs.py +++ b/scripts/generate_docs.py @@ -44,6 +44,7 @@ from mliport.install.compatibility import ( # noqa: E402 BACKENDS, python_versions_for, + runtime_stack_text, ) from mliport.install.docs import ( # noqa: E402 LANGUAGES, @@ -100,9 +101,6 @@ def _sha_short(sha: object) -> str: def render_backend_environment(engine: str, lang: str) -> str: """Environment, Python range and runtime stack of one backend.""" backend = BACKENDS[engine] - frameworks = "/".join( - sorted({p.framework_version for p in backend.arch_profiles.values()}) - ) return "\n".join( [ _row(_ENVIRONMENT_HEADER[lang]), @@ -111,7 +109,7 @@ def render_backend_environment(engine: str, lang: str) -> str: [ f"`{backend.venv_name}`", ", ".join(python_versions_for(engine)), - f"`{backend.requirement}` ({backend.framework} {frameworks})", + runtime_stack_text(engine, lang), ] ), "", diff --git a/scripts/install_mliport.sh b/scripts/install_mliport.sh index 001b9c1d..97636038 100755 --- a/scripts/install_mliport.sh +++ b/scripts/install_mliport.sh @@ -65,7 +65,7 @@ for ARG in "$@"; do done bootstrap_help() { - echo "Usage: install_mliport.sh [--engines mace dpa grace uma] [--device auto|cuda|cpu]" + echo "Usage: install_mliport.sh [--engines mace dpa grace uma nep] [--device auto|cuda|cpu]" echo " [--source auto|official|china|china-aliyun|china-ustc|china-tencent|custom|offline]" echo " [--python 3.10|3.11|3.12] [--dry-run] [--clean] [--skip-doctor] [--non-interactive]" echo "UMA requires Python >=3.11. Help and dry-run never download Python." diff --git a/tests/mliport/mliport/test_install_nep_real.py b/tests/mliport/mliport/test_install_nep_real.py new file mode 100644 index 00000000..d5b17ad9 --- /dev/null +++ b/tests/mliport/mliport/test_install_nep_real.py @@ -0,0 +1,442 @@ +"""Real (non-mock) installation tests for the NEP backend (tracking #61). + +These tests execute ``scripts/install_mliport.sh`` for real: they create a +virtual environment, download wheels, install an editable mliport and run +``mliport doctor`` from the freshly created ``.venv-nep``. They need network +access, ``uv`` and minutes of time, so they only run when +``MLIPORT_REAL_INSTALL=1`` is set (the dedicated ``installer-real-nep`` CI job +does this). Plan-level behaviour is covered, without network, by +``test_install_plan.py``. + +Every installation happens in a throw-away copy of the repository so that the +real checkout never gets a ``.venv-nep`` and "no other environment was created" +can be asserted exactly. +""" + +from __future__ import annotations + +import os +import shutil +import stat +import subprocess +import sys +import urllib.request +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parents[3] +ENABLED = os.environ.get("MLIPORT_REAL_INSTALL") == "1" +LOG_DIR = Path(os.environ.get("MLIPORT_REAL_INSTALL_LOGDIR", "")).expanduser() +INSTALL_TIMEOUT = 1800 + +pytestmark = [ + pytest.mark.skipif( + not ENABLED, reason="real installs run only with MLIPORT_REAL_INSTALL=1" + ), + pytest.mark.skipif( + sys.platform == "win32" + or shutil.which("uv") is None + or shutil.which("bash") is None, + reason="needs POSIX bash and uv", + ), +] + +TUNA_PROBE = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple/calorine/" + + +# --------------------------------------------------------------------------- +# helpers +# --------------------------------------------------------------------------- + + +def _clean_env(extra: dict[str, str] | None = None) -> dict[str, str]: + """The user's shell, minus anything that could mask an installer defect. + + ``PYTHONPATH`` in particular is what let the old NEP install reach the + ``inventory`` stage without mliport in ``.venv-nep``. + """ + env = { + k: v + for k, v in os.environ.items() + if not k.startswith(("PIP_", "UV_", "MLIPORT_", "NEP_", "PYTHON")) + and k not in {"VIRTUAL_ENV", "CONDA_PREFIX"} + } + env.update(extra or {}) + return env + + +def _copy_workspace(destination: Path) -> Path: + """Copy what the installer needs (``mliport/`` and ``scripts/``).""" + destination.mkdir(parents=True) + ignore = shutil.ignore_patterns( + "__pycache__", + "*.egg-info", + "*.pyc", + ".venv*", + "build", + "dist", + ".pytest_cache", + ".ruff_cache", + ) + for name in ("mliport", "scripts"): + shutil.copytree(REPO / name, destination / name, ignore=ignore) + return destination + + +def _run( + argv: list[str], + cwd: Path, + *, + env: dict[str, str] | None = None, + log_name: str | None = None, + timeout: int = INSTALL_TIMEOUT, +) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + argv, + cwd=cwd, + env=env if env is not None else _clean_env(), + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + if log_name and str(LOG_DIR) not in {"", "."}: + LOG_DIR.mkdir(parents=True, exist_ok=True) + (LOG_DIR / f"{log_name}.log").write_text( + f"$ {' '.join(argv)}\n(cwd={cwd})\nexit={result.returncode}\n\n" + f"--- stdout ---\n{result.stdout}\n--- stderr ---\n{result.stderr}\n", + encoding="utf-8", + ) + return result + + +def _output(result: subprocess.CompletedProcess[str]) -> str: + return f"{result.stdout}\n{result.stderr}" + + +def _install( + workspace: Path, + *extra: str, + env: dict[str, str] | None = None, + log_name: str, +) -> subprocess.CompletedProcess[str]: + return _run( + [ + "bash", + "scripts/install_mliport.sh", + "--engines", + "nep", + "--non-interactive", + *extra, + ], + workspace, + env=env, + log_name=log_name, + ) + + +def _is_executable_file(path: Path) -> bool: + return ( + path.is_file() + and bool(path.stat().st_mode & stat.S_IXUSR) + and os.access(path, os.X_OK) + ) + + +def _doctor( + workspace: Path, + device: str, + env: dict[str, str] | None = None, + log_name: str | None = None, +): + return _run( + [ + str(workspace / ".venv-nep" / "bin" / "mliport"), + "doctor", + "--engine", + "nep", + "--device", + device, + ], + workspace, + env=env, + log_name=log_name, + timeout=300, + ) + + +def _no_gpumd_env() -> dict[str, str]: + env = _clean_env() + if shutil.which("gpumd", path=env.get("PATH")): + pytest.skip( + "a gpumd executable is on PATH; the missing-binary case cannot be shown" + ) + return env + + +# --------------------------------------------------------------------------- +# first install, official sources (shared by the tests below) +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def workspace(tmp_path_factory) -> Path: + path = _copy_workspace(tmp_path_factory.mktemp("nep-real") / "repo") + result = _install( + path, + "--device", + "cpu", + "--source", + "official", + log_name="01-install-official-cpu", + ) + assert result.returncode == 0, _output(result) + return path + + +def test_install_creates_executable_mliport_in_venv_nep(workspace: Path) -> None: + launcher = workspace / ".venv-nep" / "bin" / "mliport" + assert _is_executable_file(launcher), "the verify stage launches this file" + help_run = _run([str(launcher), "--help"], workspace, timeout=120) + assert help_run.returncode == 0, _output(help_run) + assert "doctor" in _output(help_run) + + +def test_install_writes_the_nep_launcher(workspace: Path) -> None: + assert _is_executable_file(workspace / "bin" / "mliport-nep") + + +def test_installed_mliport_is_the_editable_workspace_copy(workspace: Path) -> None: + python = workspace / ".venv-nep" / "bin" / "python" + probe = _run( + [ + str(python), + "-I", + "-c", + "import mliport, calorine, ase; from importlib.metadata import version; " + "print(mliport.__file__); print(version('calorine')); print(ase.__version__)", + ], + workspace, + timeout=120, + ) + assert probe.returncode == 0, _output(probe) + lines = probe.stdout.strip().splitlines() + assert Path(lines[0]).resolve().is_relative_to((workspace / "mliport").resolve()) + assert lines[1] == "4.1" + + +def test_doctor_cpu_passes_without_gpumd(workspace: Path) -> None: + result = _doctor( + workspace, "cpu", env=_no_gpumd_env(), log_name="02-doctor-cpu-no-gpumd" + ) + assert result.returncode == 0, _output(result) + + +def test_doctor_cuda_names_the_missing_gpumd_binary(workspace: Path) -> None: + result = _doctor( + workspace, "cuda", env=_no_gpumd_env(), log_name="03-doctor-cuda-no-gpumd" + ) + assert result.returncode != 0, "must fail closed, not pretend the GPU path works" + assert "gpumd" in _output(result).lower() + + +def test_doctor_cuda_accepts_gpumd_given_through_mliport_gpumd_command( + workspace: Path, tmp_path: Path +) -> None: + fake = tmp_path / "gpumd" + fake.write_text("#!/bin/sh\necho 'fake gpumd for doctor probe'\n", encoding="utf-8") + fake.chmod(0o755) + env = _no_gpumd_env() + env["MLIPORT_GPUMD_COMMAND"] = str(fake) + result = _doctor(workspace, "cuda", env=env, log_name="04-doctor-cuda-fake-gpumd") + assert result.returncode == 0, _output(result) + + # doctor does not read NEP_GPUMD_COMMAND; documented, and pinned here. + env = _no_gpumd_env() + env["NEP_GPUMD_COMMAND"] = str(fake) + other = _doctor(workspace, "cuda", env=env) + assert other.returncode != 0 + + +def test_nep_install_is_isolated_from_other_backends(workspace: Path) -> None: + created = sorted(p.name for p in workspace.iterdir() if p.name.startswith(".venv")) + assert created == [".venv-nep"] + python = workspace / ".venv-nep" / "bin" / "python" + listing = _run( + ["uv", "pip", "list", "--python", str(python)], workspace, timeout=120 + ) + assert listing.returncode == 0, _output(listing) + installed = { + line.split()[0].lower() + for line in listing.stdout.splitlines()[2:] + if line.strip() + } + for forbidden in ( + "torch", + "mace-torch", + "fairchem-core", + "deepmd-kit", + "tensorflow", + "tensorpotential", + ): + assert forbidden not in installed, forbidden + assert {"calorine", "ase", "numpy", "mliport"} <= installed + for other in ("mace", "dpa", "grace", "uma"): + assert not (workspace / "bin" / f"mliport-{other}").exists() + + +def test_repeat_install_keeps_working_and_keeps_the_environment( + workspace: Path, +) -> None: + marker = workspace / ".venv-nep" / "reinstall-marker" + marker.write_text("kept", encoding="utf-8") + result = _install( + workspace, "--device", "cpu", "--source", "official", log_name="05-reinstall" + ) + assert result.returncode == 0, _output(result) + assert marker.read_text(encoding="utf-8") == "kept", ( + "plain re-run must not delete the environment" + ) + assert _is_executable_file(workspace / ".venv-nep" / "bin" / "mliport") + assert _doctor(workspace, "cpu", env=_no_gpumd_env()).returncode == 0 + + +def test_clean_reinstall_recreates_only_venv_nep(workspace: Path) -> None: + marker = workspace / ".venv-nep" / "clean-marker" + marker.write_text("gone", encoding="utf-8") + bystander = workspace / ".venv-bystander" + bystander.mkdir() + (bystander / "keep").write_text("kept", encoding="utf-8") + try: + result = _install( + workspace, + "--device", + "cpu", + "--source", + "official", + "--clean", + log_name="06-clean-reinstall", + ) + assert result.returncode == 0, _output(result) + assert not marker.exists() + assert (bystander / "keep").read_text(encoding="utf-8") == "kept" + assert _is_executable_file(workspace / ".venv-nep" / "bin" / "mliport") + finally: + shutil.rmtree(bystander, ignore_errors=True) + + +def test_offline_reinstall_from_the_warm_cache(workspace: Path) -> None: + """The earlier installs filled uv's cache; ``--source offline`` must then + rebuild the environment without touching the network.""" + result = _install( + workspace, + "--device", + "cpu", + "--source", + "offline", + "--clean", + log_name="07-offline-reinstall", + ) + assert result.returncode == 0, _output(result) + assert _is_executable_file(workspace / ".venv-nep" / "bin" / "mliport") + assert _doctor(workspace, "cpu", env=_no_gpumd_env()).returncode == 0 + + +def test_gpu_host_install_without_gpumd_stops_at_doctor_and_names_gpumd( + tmp_path: Path, +) -> None: + """A GPU host (here: a V100 reported by an ``nvidia-smi`` shim, since CI has + no GPU) with ``--device auto`` asks for the CUDA path. Without gpumd the + closing doctor must fail naming it, the environment itself stays installed, + and the launcher is not advertised as installed.""" + shim_dir = tmp_path / "gpu-shim" + shim_dir.mkdir() + smi = shim_dir / "nvidia-smi" + smi.write_text( + "#!/bin/sh\necho 'Tesla V100-SXM2-16GB, 7.0, 580.173.02, 16384, 0, " + "GPU-00000000-0000-0000-0000-000000000000'\n", + encoding="utf-8", + ) + smi.chmod(0o755) + env = _no_gpumd_env() + env["PATH"] = f"{shim_dir}{os.pathsep}{env['PATH']}" + + path = _copy_workspace(tmp_path / "repo") + result = _install( + path, + "--device", + "auto", + "--source", + "official", + env=env, + log_name="08-install-gpu-host-no-gpumd", + ) + assert result.returncode != 0 + assert "gpumd" in _output(result).lower() + assert _is_executable_file(path / ".venv-nep" / "bin" / "mliport"), ( + "environment itself is installed" + ) + assert not (path / "bin" / "mliport-nep").exists() + + +# --------------------------------------------------------------------------- +# China mirror +# --------------------------------------------------------------------------- + + +def _tuna_reachable() -> bool: + try: + with urllib.request.urlopen(TUNA_PROBE, timeout=20) as response: # noqa: S310 + return response.status == 200 + except OSError: + return False + + +def test_china_source_really_uses_the_tuna_mirror(tmp_path: Path) -> None: + """A ``uv`` shim records every invocation, so the mirror's presence on the + NEP install step is observed in a real run, not read off the plan.""" + real_uv = shutil.which("uv") + assert real_uv + shim_dir = tmp_path / "shim" + shim_dir.mkdir() + uv_log = tmp_path / "uv-calls.log" + shim = shim_dir / "uv" + shim.write_text( + f'#!/bin/sh\nprintf \'%s\\n\' "$*" >> "{uv_log}"\nexec "{real_uv}" "$@"\n', + encoding="utf-8", + ) + shim.chmod(0o755) + env = _clean_env() + env["PATH"] = f"{shim_dir}{os.pathsep}{env['PATH']}" + + path = _copy_workspace(tmp_path / "repo") + reachable = _tuna_reachable() + result = _install( + path, + "--device", + "cpu", + "--source", + "china", + env=env, + log_name="09-install-china-cpu", + ) + + calls = uv_log.read_text(encoding="utf-8").splitlines() + nep_install = [ + c + for c in calls + if c.startswith("pip install") and "-e ./mliport[analysis-all]" in c + ] + assert nep_install, calls + assert all( + "--index-url https://mirrors.tuna.tsinghua.edu.cn/" in c for c in nep_install + ), nep_install + assert all("pypi.org" not in c for c in nep_install) + if reachable: + assert result.returncode == 0, _output(result) + assert _is_executable_file(path / ".venv-nep" / "bin" / "mliport") + assert _doctor(path, "cpu", env=_no_gpumd_env()).returncode == 0 + else: + # Fail closed: an unreachable mirror is an error, never a silent switch to PyPI. + assert result.returncode != 0 diff --git a/tests/mliport/mliport/test_install_plan.py b/tests/mliport/mliport/test_install_plan.py index a0962df2..18aab82c 100644 --- a/tests/mliport/mliport/test_install_plan.py +++ b/tests/mliport/mliport/test_install_plan.py @@ -318,3 +318,125 @@ def test_steps_are_argv_lists() -> None: for step in plan.steps: assert isinstance(step.argv, list) assert all(isinstance(a, str) for a in step.argv) + + +# --------------------------------------------------------------------------- +# NEP: mliport itself must be installed into .venv-nep (tracking #61) +# --------------------------------------------------------------------------- + +_NEP_PIP_ARGV_START = ("uv", "pip", "install") + + +def _nep_install(plan): + steps = [s for s in _pip_cmds(plan) if "-e" in s.argv] + assert len(steps) == 1, [s.argv for s in _pip_cmds(plan)] + return steps[0] + + +def test_nep_plan_installs_editable_mliport_with_pinned_calorine() -> None: + plan = generate_plan(None, ["nep"], device="cpu") + step = _nep_install(plan) + argv = step.argv + assert tuple(argv[:3]) == _NEP_PIP_ARGV_START + assert argv[argv.index("--python") + 1] == ".venv-nep/bin/python" + assert argv[argv.index("-e") + 1] == "./mliport[analysis-all]" + assert "calorine==4.1" in argv + # NumPy/ASE come from the resolver (mliport requires ase>=3.26), never from + # hand-written pins that could contradict it. + assert not any(a.startswith(("numpy", "ase")) for a in argv) + assert "--constraint" in argv + assert "calorine==4.1" in plan_constraints_text(plan) + + +def plan_constraints_text(plan) -> str: + step = next(s for s in plan.steps if s.stage == "constraints") + return step.argv[-1] + + +def test_nep_plan_stage_order_ends_with_launchable_verify() -> None: + plan = generate_plan(None, ["nep"], device="cpu") + stages = [s.stage for s in plan.steps] + assert stages == ["venv", "constraints", "pip", "check", "inventory", "verify"] + # mliport is installed before anything that executes .venv-nep/bin/mliport + verify = plan.steps[-1] + assert verify.argv[0] == ".venv-nep/bin/mliport" + assert verify.argv[1:4] == ["doctor", "--engine", "nep"] + first_mliport_install = stages.index("pip") + assert first_mliport_install < stages.index("verify") + + +def test_nep_plan_does_not_rely_on_ignored_pip_environment() -> None: + """The managed source profiles strip PIP_*/UV_* env, so mirror selection must + be argv-based; the old PIP_EXTRA_INDEX_URL env on the NEP step was dead.""" + for source in ("official", "china", "offline", "custom"): + plan = generate_plan(None, ["nep"], device="cpu", source=source) + for step in plan.steps: + assert not any(k.startswith(("PIP_", "UV_")) for k in step.env), source + + +def test_nep_plan_honours_china_mirror() -> None: + plan = generate_plan(None, ["nep"], device="cpu", source="china") + argv = _nep_install(plan).argv + assert "--index-url" in argv + assert argv[argv.index("--index-url") + 1].startswith("https://mirrors.tuna.") + assert "--offline" not in argv + + +def test_nep_plan_honours_offline_and_custom() -> None: + offline = generate_plan(None, ["nep"], device="cpu", source="offline") + assert "--offline" in _nep_install(offline).argv + for step in offline.steps: + joined = " ".join(step.argv) + assert "http://" not in joined and "https://" not in joined + + custom = generate_plan(None, ["nep"], device="cpu", source="custom") + joined = " ".join(_nep_install(custom).argv) + assert "--index-url" not in joined and "--extra-index-url" not in joined + + +def test_nep_official_plan_has_no_mirror_flags() -> None: + argv = _nep_install(generate_plan(None, ["nep"], device="cpu")).argv + assert "--index-url" not in argv and "--offline" not in argv + + +def test_nep_clean_removes_only_its_own_environment() -> None: + plan = generate_plan(None, ["nep"], device="cpu", clean=True) + clean = [s for s in plan.steps if s.stage == "clean"] + assert [s.argv for s in clean] == [["rm", "-rf", ".venv-nep"]] + + +def test_nep_is_opt_in_and_isolated_from_other_backends() -> None: + from mliport.install.cli import _parser + + assert "nep" not in _parser().parse_args([]).engines + assert _parser().parse_args(["--engines", "nep"]).engines == ["nep"] + + both = generate_plan(None, ["mace", "nep"], device="cpu", clean=True) + alone_mace = generate_plan(None, ["mace"], device="cpu", clean=True) + alone_nep = generate_plan(None, ["nep"], device="cpu", clean=True) + # Adding NEP appends its own steps; the MACE steps are byte-identical. + mace_in_both = [s.argv for s in both.steps if ".venv-mace" in " ".join(s.argv)] + assert mace_in_both == [s.argv for s in alone_mace.steps] + nep_in_both = [s.argv for s in both.steps if ".venv-nep" in " ".join(s.argv)] + assert nep_in_both == [s.argv for s in alone_nep.steps] + # no step touches another backend's environment + for step in both.steps: + joined = " ".join(step.argv) + assert not (".venv-mace" in joined and ".venv-nep" in joined) + + +@pytest.mark.parametrize("engine", ["mace", "dpa", "grace", "uma"]) +def test_other_backends_install_mliport_exactly_once_and_never_calorine(engine) -> None: + plan = generate_plan(None, [engine], device="cpu") + editable = [s for s in _pip_cmds(plan) if "-e" in s.argv] + assert len(editable) == 1 + assert "calorine" not in _all_argv(plan) + + +def test_nep_plan_still_warns_that_gpumd_is_not_built() -> None: + plan = generate_plan(None, ["nep"], device="cpu") + text = " ".join(plan.warnings) + assert "gpumd" in text + assert "does not build it" in text + assert "MLIPORT_GPUMD_COMMAND" in text + assert "NEP_GPUMD_COMMAND" in text diff --git a/tests/mliport/mliport/test_install_ux_contract.py b/tests/mliport/mliport/test_install_ux_contract.py index 5d86f95d..8e49739a 100644 --- a/tests/mliport/mliport/test_install_ux_contract.py +++ b/tests/mliport/mliport/test_install_ux_contract.py @@ -69,6 +69,51 @@ def test_readme_states_the_real_installation_model(readme): assert "bin/mliport-mace" in text +@pytest.mark.parametrize("readme", ["README.md", "README_CN.md"]) +def test_readme_documents_the_nep_backend(readme): + text = (REPO / readme).read_text(encoding="utf-8") + assert "--engines nep" in text # opt-in install command + assert ".venv-nep" in text and "bin/mliport-nep" in text # environment table + assert "--model-type nep" in text and "--nep-runtime" in text # example + assert "gpumd" in text and "nep.txt" in text # limits: own model, own build + assert "docs/backends/nep" in text + assert "NEP_GPUMD_COMMAND" not in text # doctor reads MLIPORT_GPUMD_COMMAND + + +def test_nep_runtime_stack_separates_numpy_from_cuda_toolkit(): + from mliport.install.compatibility import runtime_stack_text + + for lang in ("en", "zh"): + text = runtime_stack_text("nep", lang) + assert "calorine==4.1" in text + assert "12.9" in text and "CUDA Toolkit" in text + # the old wording read as "calorine 4.1 needs numpy 12.9" + assert "calorine==4.1` (12.9)" not in text + assert "numpy 12.9" not in text.lower() + # torch/tf backends keep their framework pins + assert "torch==" in runtime_stack_text("mace", "en") + + +@pytest.mark.parametrize("name", ["installation.md", "installation.zh.md"]) +def test_installation_docs_never_pair_calorine_with_the_cuda_version(name): + text = (REPO / "docs" / name).read_text(encoding="utf-8") + assert "calorine==4.1` (12.9)" not in text + assert "calorine==4.1 (numpy" not in text.lower() + assert "--engines nep" in text and ".venv-nep/bin/mliport" in text + + +@pytest.mark.parametrize("name", ["nep.md", "nep.zh.md"]) +def test_nep_backend_page_installs_mliport_and_names_the_doctor_variable(name): + text = (REPO / "docs" / "backends" / name).read_text(encoding="utf-8") + assert "-e './mliport[analysis-all]' calorine==4.1" in text + assert "MLIPORT_GPUMD_COMMAND" in text + + +def test_installer_help_lists_nep(): + text = (REPO / "scripts" / "install_mliport.sh").read_text(encoding="utf-8") + assert "[--engines mace dpa grace uma nep]" in text + + # --------------------------------------------------------------- DOC-03 def test_build_plan_fails_closed_for_backend_incompatible_python(): from mliport.install.plan import InstallPlanError, generate_plan From 67c23d2712d12ab0fe65e766c3bcb363ff32dd98 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 20:53:26 -0500 Subject: [PATCH 2/6] test(install): keep uv location vars in real-install env; temp ruff diff probe Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01GYQHUEKJ91XQC4Hzj11Q1X --- .github/workflows/test.yml | 2 ++ .../mliport/mliport/test_install_nep_real.py | 31 ++++++++++++++----- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7f54488e..f68547b2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -116,6 +116,8 @@ jobs: python-version: "3.12" enable-cache: true cache-dependency-glob: "mliport/pyproject.toml" + - name: TEMP diagnose ruff 0.5.1 format diff + run: uvx ruff@0.5.1 format --diff tests/mliport/mliport/test_install_nep_real.py || true - name: Install pytest only run: | uv venv .venv-ci --python 3.12 diff --git a/tests/mliport/mliport/test_install_nep_real.py b/tests/mliport/mliport/test_install_nep_real.py index d5b17ad9..cba29f40 100644 --- a/tests/mliport/mliport/test_install_nep_real.py +++ b/tests/mliport/mliport/test_install_nep_real.py @@ -50,18 +50,35 @@ # --------------------------------------------------------------------------- +# uv settings that say *where* things live (not *which index* to use). CI +# images set them so that uv finds its managed Python and cache; stripping them +# would make the bootstrap report "no local Python". +_UV_LOCATION_VARS = { + "UV_CACHE_DIR", + "UV_PYTHON_INSTALL_DIR", + "UV_PYTHON_BIN_DIR", + "UV_PYTHON_CACHE_DIR", + "UV_TOOL_DIR", + "UV_TOOL_BIN_DIR", +} + + def _clean_env(extra: dict[str, str] | None = None) -> dict[str, str]: """The user's shell, minus anything that could mask an installer defect. ``PYTHONPATH`` in particular is what let the old NEP install reach the - ``inventory`` stage without mliport in ``.venv-nep``. + ``inventory`` stage without mliport in ``.venv-nep``. Index/offline/python + selection variables are dropped; uv's location variables are kept. """ - env = { - k: v - for k, v in os.environ.items() - if not k.startswith(("PIP_", "UV_", "MLIPORT_", "NEP_", "PYTHON")) - and k not in {"VIRTUAL_ENV", "CONDA_PREFIX"} - } + env = {} + for key, value in os.environ.items(): + if key in {"VIRTUAL_ENV", "CONDA_PREFIX"}: + continue + if key.startswith(("PIP_", "MLIPORT_", "NEP_", "PYTHON")): + continue + if key.startswith("UV_") and key not in _UV_LOCATION_VARS: + continue + env[key] = value env.update(extra or {}) return env From 08ee846868563f43e21fb2e1a607d2dda3357b69 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 20:56:08 -0500 Subject: [PATCH 3/6] test(install): pin real doctor/repeat-install behaviour; document --clean; drop probe Real CI run showed: doctor --device cuda needs nvidia-smi before it reaches the gpumd check (use an nvidia-smi shim), and a plain re-run stops at venv creation for every backend (documented, test pinned to it). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01GYQHUEKJ91XQC4Hzj11Q1X --- .github/workflows/test.yml | 2 - docs/installation.md | 4 +- docs/installation.zh.md | 3 +- .../mliport/mliport/test_install_nep_real.py | 73 ++++++++++++------- 4 files changed, 50 insertions(+), 32 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f68547b2..7f54488e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -116,8 +116,6 @@ jobs: python-version: "3.12" enable-cache: true cache-dependency-glob: "mliport/pyproject.toml" - - name: TEMP diagnose ruff 0.5.1 format diff - run: uvx ruff@0.5.1 format --diff tests/mliport/mliport/test_install_nep_real.py || true - name: Install pytest only run: | uv venv .venv-ci --python 3.12 diff --git a/docs/installation.md b/docs/installation.md index 029f3b42..56ea2013 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -50,7 +50,9 @@ CUDA path and fails naming `gpumd` until one is on `PATH` or in build `gpumd` and re-run the installer, or install with `--device cpu` / `--skip-doctor` (and call `.venv-nep/bin/mliport` directly in the meantime). -Pass fewer engines if you only need some. `--dry-run` prints every step without +Re-running the installer while an environment already exists stops at its creation +step and leaves that environment untouched; add `--clean` to delete and rebuild +only the environments of the engines you name. Pass fewer engines if you only need some. `--dry-run` prints every step without executing anything. Unless you give `--skip-doctor`, the installer runs `mliport doctor` in each environment when it finishes. The analysis, transport and electrolyte features come with the same install; there is no second step. diff --git a/docs/installation.zh.md b/docs/installation.zh.md index 29157317..8039b0d8 100644 --- a/docs/installation.zh.md +++ b/docs/installation.zh.md @@ -43,7 +43,8 @@ mliport,所以 `.venv-nep/bin/mliport` 一定存在,`doctor` 才能运行。 `bin/mliport-nep` 启动器要等所有步骤都成功才会写出;可以编译好 `gpumd` 后重新运行安装脚本,也可以改用 `--device cpu` 或 `--skip-doctor` 安装(期间直接用 `.venv-nep/bin/mliport`)。 -只要部分后端时,`--engines` 后面少写几个就行。`--dry-run` 只打印要执行的步骤,不真正安装。 +环境已经存在时重新运行安装脚本,会在创建环境这一步停下,原有环境不会被改动;加 `--clean` 才会删除并重建 +你所指定后端的环境(只动这些)。只要部分后端时,`--engines` 后面少写几个就行。`--dry-run` 只打印要执行的步骤,不真正安装。 安装结束时脚本会在每个环境里运行一次 `mliport doctor`(加 `--skip-doctor` 可以跳过)。分析、 输运和电解质相关的依赖会一起装好,不用另外安装。 diff --git a/tests/mliport/mliport/test_install_nep_real.py b/tests/mliport/mliport/test_install_nep_real.py index cba29f40..fe2b9384 100644 --- a/tests/mliport/mliport/test_install_nep_real.py +++ b/tests/mliport/mliport/test_install_nep_real.py @@ -192,6 +192,26 @@ def _no_gpumd_env() -> dict[str, str]: return env +def _gpu_host_env(shim_dir: Path) -> dict[str, str]: + """A V100 host, as reported by an ``nvidia-smi`` shim (CI has no GPU). + + ``doctor --device cuda`` asks ``nvidia-smi`` for the device first and + crashes with a traceback when there is none, before it ever reaches the + gpumd check; the shim is therefore what makes the gpumd path observable. + """ + shim_dir.mkdir(parents=True, exist_ok=True) + smi = shim_dir / "nvidia-smi" + smi.write_text( + "#!/bin/sh\necho 'Tesla V100-SXM2-16GB, 7.0, 580.173.02, 16384, 0, " + "GPU-00000000-0000-0000-0000-000000000000'\n", + encoding="utf-8", + ) + smi.chmod(0o755) + env = _no_gpumd_env() + env["PATH"] = f"{shim_dir}{os.pathsep}{env['PATH']}" + return env + + # --------------------------------------------------------------------------- # first install, official sources (shared by the tests below) # --------------------------------------------------------------------------- @@ -250,11 +270,12 @@ def test_doctor_cpu_passes_without_gpumd(workspace: Path) -> None: assert result.returncode == 0, _output(result) -def test_doctor_cuda_names_the_missing_gpumd_binary(workspace: Path) -> None: - result = _doctor( - workspace, "cuda", env=_no_gpumd_env(), log_name="03-doctor-cuda-no-gpumd" - ) - assert result.returncode != 0, "must fail closed, not pretend the GPU path works" +def test_doctor_cuda_names_the_missing_gpumd_binary( + workspace: Path, tmp_path: Path +) -> None: + env = _gpu_host_env(tmp_path / "gpu-shim") + result = _doctor(workspace, "cuda", env=env, log_name="03-doctor-cuda-no-gpumd") + assert result.returncode != 0, "must fail closed, not pretend GPU NEP works" assert "gpumd" in _output(result).lower() @@ -262,18 +283,19 @@ def test_doctor_cuda_accepts_gpumd_given_through_mliport_gpumd_command( workspace: Path, tmp_path: Path ) -> None: fake = tmp_path / "gpumd" - fake.write_text("#!/bin/sh\necho 'fake gpumd for doctor probe'\n", encoding="utf-8") + fake.write_text("#!/bin/sh\necho 'fake gpumd for doctor'\n", encoding="utf-8") fake.chmod(0o755) - env = _no_gpumd_env() + env = _gpu_host_env(tmp_path / "gpu-shim") env["MLIPORT_GPUMD_COMMAND"] = str(fake) result = _doctor(workspace, "cuda", env=env, log_name="04-doctor-cuda-fake-gpumd") assert result.returncode == 0, _output(result) # doctor does not read NEP_GPUMD_COMMAND; documented, and pinned here. - env = _no_gpumd_env() + env = _gpu_host_env(tmp_path / "gpu-shim") env["NEP_GPUMD_COMMAND"] = str(fake) other = _doctor(workspace, "cuda", env=env) assert other.returncode != 0 + assert "gpumd" in _output(other).lower() def test_nep_install_is_isolated_from_other_backends(workspace: Path) -> None: @@ -303,18 +325,23 @@ def test_nep_install_is_isolated_from_other_backends(workspace: Path) -> None: assert not (workspace / "bin" / f"mliport-{other}").exists() -def test_repeat_install_keeps_working_and_keeps_the_environment( +def test_plain_repeat_install_fails_closed_and_leaves_the_environment_alone( workspace: Path, ) -> None: + """Without ``--clean`` the venv step stops because ``.venv-nep`` exists. + + This is how every backend's plan behaves (``uv venv`` without + ``--allow-existing``); NEP is pinned to the same behaviour and, above all, + the working environment must be left untouched. + """ marker = workspace / ".venv-nep" / "reinstall-marker" marker.write_text("kept", encoding="utf-8") result = _install( - workspace, "--device", "cpu", "--source", "official", log_name="05-reinstall" - ) - assert result.returncode == 0, _output(result) - assert marker.read_text(encoding="utf-8") == "kept", ( - "plain re-run must not delete the environment" + workspace, "--device", "cpu", "--source", "official", log_name="05-repeat" ) + assert result.returncode != 0 + assert "already exists" in _output(result) + assert marker.read_text(encoding="utf-8") == "kept" assert _is_executable_file(workspace / ".venv-nep" / "bin" / "mliport") assert _doctor(workspace, "cpu", env=_no_gpumd_env()).returncode == 0 @@ -367,17 +394,7 @@ def test_gpu_host_install_without_gpumd_stops_at_doctor_and_names_gpumd( no GPU) with ``--device auto`` asks for the CUDA path. Without gpumd the closing doctor must fail naming it, the environment itself stays installed, and the launcher is not advertised as installed.""" - shim_dir = tmp_path / "gpu-shim" - shim_dir.mkdir() - smi = shim_dir / "nvidia-smi" - smi.write_text( - "#!/bin/sh\necho 'Tesla V100-SXM2-16GB, 7.0, 580.173.02, 16384, 0, " - "GPU-00000000-0000-0000-0000-000000000000'\n", - encoding="utf-8", - ) - smi.chmod(0o755) - env = _no_gpumd_env() - env["PATH"] = f"{shim_dir}{os.pathsep}{env['PATH']}" + env = _gpu_host_env(tmp_path / "gpu-shim") path = _copy_workspace(tmp_path / "repo") result = _install( @@ -391,9 +408,9 @@ def test_gpu_host_install_without_gpumd_stops_at_doctor_and_names_gpumd( ) assert result.returncode != 0 assert "gpumd" in _output(result).lower() - assert _is_executable_file(path / ".venv-nep" / "bin" / "mliport"), ( - "environment itself is installed" - ) + assert _is_executable_file( + path / ".venv-nep" / "bin" / "mliport" + ), "environment itself is installed" assert not (path / "bin" / "mliport-nep").exists() From b8e79f490efa397ae1b434f006701b4738720bd9 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 21:00:50 -0500 Subject: [PATCH 4/6] test(install): nvidia-smi shim answers both mliport probes (index,uuid and full inventory) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01GYQHUEKJ91XQC4Hzj11Q1X --- tests/mliport/mliport/test_install_nep_real.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/mliport/mliport/test_install_nep_real.py b/tests/mliport/mliport/test_install_nep_real.py index fe2b9384..a43d1c41 100644 --- a/tests/mliport/mliport/test_install_nep_real.py +++ b/tests/mliport/mliport/test_install_nep_real.py @@ -202,8 +202,14 @@ def _gpu_host_env(shim_dir: Path) -> dict[str, str]: shim_dir.mkdir(parents=True, exist_ok=True) smi = shim_dir / "nvidia-smi" smi.write_text( - "#!/bin/sh\necho 'Tesla V100-SXM2-16GB, 7.0, 580.173.02, 16384, 0, " - "GPU-00000000-0000-0000-0000-000000000000'\n", + "#!/bin/sh\n" + 'case "$*" in\n' + # mliport.devices: index,uuid only + " *index,uuid*) echo '0, GPU-00000000-0000-0000-0000-000000000000' ;;\n" + # installer hardware probe: name,compute_cap,driver,memory,index,uuid + " *) echo 'Tesla V100-SXM2-16GB, 7.0, 580.173.02, 16384, 0, " + "GPU-00000000-0000-0000-0000-000000000000' ;;\n" + "esac\n", encoding="utf-8", ) smi.chmod(0o755) From 560219e9aff2a3c0d142b9eb34cf38342dc2ec9a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 21:00:59 -0500 Subject: [PATCH 5/6] test(install): fix nvidia-smi shim pattern (full inventory query also ends in index,uuid) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01GYQHUEKJ91XQC4Hzj11Q1X --- tests/mliport/mliport/test_install_nep_real.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mliport/mliport/test_install_nep_real.py b/tests/mliport/mliport/test_install_nep_real.py index a43d1c41..884c2918 100644 --- a/tests/mliport/mliport/test_install_nep_real.py +++ b/tests/mliport/mliport/test_install_nep_real.py @@ -205,7 +205,7 @@ def _gpu_host_env(shim_dir: Path) -> dict[str, str]: "#!/bin/sh\n" 'case "$*" in\n' # mliport.devices: index,uuid only - " *index,uuid*) echo '0, GPU-00000000-0000-0000-0000-000000000000' ;;\n" + " *--query-gpu=index,uuid*) echo '0, GPU-00000000-0000-0000-0000-000000000000' ;;\n" # installer hardware probe: name,compute_cap,driver,memory,index,uuid " *) echo 'Tesla V100-SXM2-16GB, 7.0, 580.173.02, 16384, 0, " "GPU-00000000-0000-0000-0000-000000000000' ;;\n" From fe93113624f116245fb2987e361380f24f50cf04 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 21:06:04 -0500 Subject: [PATCH 6/6] ci(installer-real-nep): also print the real install logs into the job log Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01GYQHUEKJ91XQC4Hzj11Q1X --- .github/workflows/test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7f54488e..67d83dbf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -126,6 +126,14 @@ jobs: MLIPORT_REAL_INSTALL_LOGDIR: ${{ runner.temp }}/nep-install-logs run: | .venv-ci/bin/python -m pytest tests/mliport/mliport/test_install_nep_real.py -v -rA -p no:cacheprovider + - name: Print install logs + if: always() + run: | + for f in "${{ runner.temp }}"/nep-install-logs/*.log; do + echo "::group::$(basename "$f")" + cat "$f" + echo "::endgroup::" + done - name: Upload install logs if: always() uses: actions/upload-artifact@v4