diff --git a/docs/workflows/neb.md b/docs/workflows/neb.md index 6bdc7b8c..88a18ad9 100644 --- a/docs/workflows/neb.md +++ b/docs/workflows/neb.md @@ -147,3 +147,9 @@ identity: model, task, head and dtype, cell and PBC, constraints, band setup and options. Step budgets are part of that identity in this beta, so a resume that changes `--max-steps` is rejected with `Resume fingerprint is incompatible`. The checkpoint is left untouched when that happens. + +A resume restores the band geometry recorded in the checkpoint but restarts +NEB optimisation from there (`geometry_resume_only`); it does not replay the +previous optimiser state. Convergence, the final saddle and the barrier can +differ from an uninterrupted run, so verify convergence and compare the +results before interpreting them. Every earlier checkpoint and export is kept. diff --git a/docs/workflows/neb.zh.md b/docs/workflows/neb.zh.md index 1524d85e..113addb5 100644 --- a/docs/workflows/neb.zh.md +++ b/docs/workflows/neb.zh.md @@ -116,3 +116,7 @@ ASE 的轨迹文件不保存自定义数组,所以检查点会把 `atom_id` / 续算在原来的目录里继续,并核对记录下来的信息:模型、task、head、精度,晶胞和 PBC,约束,能带 设置和各项参数。当前 beta 版本里步数上限也在核对范围内,续算时改 `--max-steps` 会报 `Resume fingerprint is incompatible` 并拒绝运行,检查点保持不变。 + +续算会恢复检查点里记录的能带几何结构,并从这里重新开始 NEB 优化(`geometry_resume_only`), +不会重放上一次的优化器状态。因此,续算不保证与一次跑完收敛到相同鞍点;是否收敛、最终鞍点和 +势垒都可能不同。必须核实收敛状态并比较结果,再解释势垒。之前每一次的检查点和导出文件都会保留。 diff --git a/tests/mliport/mliport/confsearch/test_resume_equivalence.py b/tests/mliport/mliport/confsearch/test_resume_equivalence.py new file mode 100644 index 00000000..2f7ec46a --- /dev/null +++ b/tests/mliport/mliport/confsearch/test_resume_equivalence.py @@ -0,0 +1,184 @@ +"""AUD-007: an interrupted confsearch run must resume into the same result. + +The tracker asks whether checkpoint resume truly preserves the structure, the +RNG streams and the model identity. Model identity is covered by +``test_resume_identity.py``; this module covers the other two by interrupting a +run after **every** durable unit boundary and asserting that the resumed run's +scientific result is identical to an uninterrupted run's. + +The engine checkpoints after each unit (``checkpoints/confsearch.json``) and +stores the per-replica ``md_rng``/``mc_rng`` bit-generator states, so the +resumed process must continue the exact same random streams. The synthetic +``EggCrateCalculator`` is deterministic, so any divergence in the recorded +ranking, minima or diagnostics is a resume defect, not model noise. +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import pytest +from ase.io import write + +from mliport.confsearch.engine import SearchEngine +from mliport.confsearch.workflow import run_confsearch_workflow + +if TYPE_CHECKING: + from pathlib import Path + + +class _Interrupted(RuntimeError): + """Test-only stand-in for a process that dies mid-run.""" + + +class _SyntheticWrapper: + """Backend-shaped wrapper around a synthetic calculator (no MLIP needed).""" + + task = "bulk" + + def __init__(self, calculator, model_path) -> None: + self._calculator = calculator + self._model_path = model_path + + def info(self) -> dict: + return { + "model_type": "mace", + "task": "bulk", + "backend_version": "test", + "framework_versions": {"torch": "test"}, + "default_dtype": "float64", + "model_path": str(self._model_path), + } + + def get_calculator(self): + return self._calculator + + +def _setup(run_dir: Path, *, cycles: int, frames: int): + from mliport.config import resolve_config + from tests.mliport.mliport.confsearch.calculators import EggCrateCalculator + from tests.mliport.mliport.confsearch.test_engine import _start + + run_dir.mkdir(parents=True, exist_ok=True) + model_file = run_dir / "model.pt" + model_file.write_bytes(b"identity model") + start_file = run_dir / "start.extxyz" + sites = [(0.5, 0.5), (0.25, 0.25), (0.75, 0.25)][:frames] + write(start_file, [_start(site) for site in sites], format="extxyz") + resolved = resolve_config( + calc_type="confsearch", + cli={ + "model_type": "mace", + "model_path": str(model_file), + "task": "bulk", + "cs_starts": str(start_file), + "cs_adsorbate_indices": "16-18", + "cs_cycles": cycles, + "cs_hold_ps": 0.05, + "cs_cool_ps": 0.05, + "cs_snapshot_interval_ps": 0.02, + "timestep": 0.5, + "cs_temperature_high": 800.0, + "cs_temperature_low": 300.0, + "cs_quench_fmax": 0.01, + "cs_quench_steps": 2000, + "cs_refine": False, + "cs_refine_fmax": 0.005, + "seed": 7, + }, + ) + return resolved, _SyntheticWrapper(EggCrateCalculator(), model_file) + + +def _run( + run_dir: Path, + *, + cycles: int, + frames: int, + stop_after_units: int | None = None, +) -> dict: + """Run one search, optionally killing it right after ``stop_after_units``.""" + + resolved, wrapper = _setup(run_dir, cycles=cycles, frames=frames) + output = run_dir / "run" + + if stop_after_units is None: + return run_confsearch_workflow(wrapper, resolved, output_dir=output).results + + original = SearchEngine._write_checkpoint + calls = {"n": 0} + + def interrupt_after_checkpoint(self) -> None: + calls["n"] += 1 + if calls["n"] > stop_after_units: + # The checkpoint of unit ``stop_after_units`` is already on disk. + raise _Interrupted(f"killed after {stop_after_units} unit(s)") + original(self) + + SearchEngine._write_checkpoint = interrupt_after_checkpoint + try: + with pytest.raises(_Interrupted): + run_confsearch_workflow(wrapper, resolved, output_dir=output) + finally: + SearchEngine._write_checkpoint = original + + checkpoint = json.loads( + (output / "checkpoints" / "confsearch.json").read_text(encoding="utf-8") + ) + assert checkpoint["unit_index"] == stop_after_units + + return run_confsearch_workflow( + wrapper, resolved, output_dir=output, resume=True + ).results + + +def _comparable(results: dict) -> dict: + """Drop the model *path*: where a model was stored is not identity.""" + + payload = json.loads(json.dumps(results, default=str)) + model = payload.get("model") + if isinstance(model, dict): + model.pop("model_path", None) + return payload + + +@pytest.mark.parametrize(("frames", "cycles"), [(1, 3), (2, 2)]) +def test_resume_reproduces_an_uninterrupted_run(tmp_path: Path, frames, cycles) -> None: + """Every durable boundary must resume into the identical scientific result.""" + + n_units = 1 + frames * cycles # one start unit, then one unit per replica-cycle + reference = _comparable(_run(tmp_path / "reference", cycles=cycles, frames=frames)) + + for stop_after in range(1, n_units): + resumed = _comparable( + _run( + tmp_path / f"stop_{stop_after}", + cycles=cycles, + frames=frames, + stop_after_units=stop_after, + ) + ) + assert resumed == reference, f"resume after unit {stop_after} diverged" + + +def test_resume_restores_the_recorded_rng_states(tmp_path: Path) -> None: + """The checkpoint must carry per-replica md/mc generator states.""" + + run_dir = tmp_path / "rng" + _run(run_dir, cycles=1, frames=2, stop_after_units=1) + + checkpoint = json.loads( + (run_dir / "run" / "checkpoints" / "confsearch.json").read_text( + encoding="utf-8" + ) + ) + replicas = checkpoint["rng_states"]["replicas"] + assert [entry["index"] for entry in replicas] == [0, 1] + for entry in replicas: + for stream in ("md", "mc"): + state = entry[stream] + assert state["bit_generator"] + assert isinstance(state["state"], dict) + # Different replicas must not share one generator state. + assert replicas[0]["md"] != replicas[1]["md"] diff --git a/tests/mliport/mliport/neb/test_neb_resume_equivalence.py b/tests/mliport/mliport/neb/test_neb_resume_equivalence.py new file mode 100644 index 00000000..92f63e1d --- /dev/null +++ b/tests/mliport/mliport/neb/test_neb_resume_equivalence.py @@ -0,0 +1,243 @@ +"""AUD-007: what a NEB geometry-resume actually preserves. + +The tracker asks whether resume truly keeps the structure, the random state and +the model identity. For NEB: + +* **structure** — every checkpoint stores the complete band as an ASE + trajectory; a resume must rebuild the *exact* band from it; +* **random state** — NEB has no stochastic optimizer. The only randomness is + the gradient pre-flight, which is seeded from a module constant, so it is + reproducible without checkpointing. There is no generator state to restore; +* **model identity** — enforced by the resume fingerprint (covered by + ``test_neb_workflow.py``). + +What is deliberately *not* preserved is the optimizer's internal state: the +checkpoint declares ``geometry_resume_only: True``, so a resume restarts the NEB +optimisation from the checkpointed geometry. These tests pin the exact part +(band restored verbatim, prior evidence kept) and the loose part (the resumed +run converges to the same saddle, but its path — and therefore its final +energy — may differ slightly from an uninterrupted run). +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, ClassVar + +import numpy as np +import pytest +from ase import Atoms +from ase.calculators.calculator import Calculator, all_changes +from ase.constraints import FixAtoms + +from mliport.neb.io import NEBCheckpointStore, load_checkpoint +from mliport.neb.workflow import run_neb_workflow + +if TYPE_CHECKING: + from pathlib import Path + +#: Six images and a curved minimum-energy path: the band genuinely has to move, +#: unlike a straight 1-D path where the projected force is already zero. +_CLI = {"n_intermediate_images": 6, "max_steps": 100, "checkpoint_interval": 5} + + +class _Interrupted(RuntimeError): + """Test-only stand-in for a process that dies mid-run.""" + + +class _CurvedMEP(Calculator): + """Curved 2-D well for the moving atom; stationary at ``x_rel = +-1``.""" + + implemented_properties: ClassVar[list[str]] = ["energy", "forces"] + arch = 0.6 + k = 5.0 + + def calculate( + self, + atoms=None, + properties=("energy", "forces"), + system_changes=all_changes, + ) -> None: + super().calculate(atoms, properties, system_changes) + positions = np.asarray(atoms.positions, dtype=float) + forces = np.zeros_like(positions) + x = positions[1, 0] - 2.0 + y = positions[1, 1] - 1.5 + z = positions[1, 2] - 1.5 + residual = y - self.arch * (1.0 - x * x) + energy = (x * x - 1.0) ** 2 + self.k * residual**2 + z * z + forces[1, 0] = -( + 4.0 * x * (x * x - 1.0) + 4.0 * self.k * self.arch * x * residual + ) + forces[1, 1] = -2.0 * self.k * residual + forces[1, 2] = -2.0 * z + self.results = {"energy": float(energy), "forces": forces} + + +def _endpoints() -> tuple[Atoms, Atoms]: + initial = Atoms( + "HHe", + positions=[[0.5, 0.5, 0.5], [1.0, 1.5, 1.5]], + cell=[4.0, 4.0, 4.0], + pbc=True, + ) + final = initial.copy() + final.positions[1, 0] = 3.0 + initial.set_constraint(FixAtoms(indices=[0])) + final.set_constraint(FixAtoms(indices=[0])) + return initial, final + + +def _wrapper(): + from tests.mliport.mliport.neb.test_neb_workflow import FakeMACEWrapper + + return FakeMACEWrapper(_CurvedMEP()) + + +def _resolved(model: Path): + from tests.mliport.mliport.neb.test_neb_workflow import _resolved as resolve + + return resolve(model, **_CLI) + + +def _output_for(tmp_path: Path) -> Path: + tmp_path.mkdir(parents=True, exist_ok=True) + model = tmp_path / "model.pt" + model.write_bytes(b"model identity") + return tmp_path / "run" + + +def _run(tmp_path: Path, *, interrupt_after_checkpoints: int | None = None) -> dict: + output = _output_for(tmp_path) + resolved = _resolved(tmp_path / "model.pt") + initial, final = _endpoints() + + if interrupt_after_checkpoints is None: + return run_neb_workflow( + _wrapper(), + resolved, + output_dir=output, + initial=initial, + final=final, + verbose=False, + ) + + original = NEBCheckpointStore.write + calls = {"n": 0} + + def interrupt_after_write(self, images, **kwargs): + calls["n"] += 1 + if calls["n"] > interrupt_after_checkpoints: + raise _Interrupted("simulated process death") + return original(self, images, **kwargs) + + NEBCheckpointStore.write = interrupt_after_write + try: + with pytest.raises(_Interrupted): + run_neb_workflow( + _wrapper(), + resolved, + output_dir=output, + initial=initial, + final=final, + verbose=False, + ) + finally: + NEBCheckpointStore.write = original + return {} + + +def _resume(tmp_path: Path) -> dict: + output = tmp_path / "run" + return run_neb_workflow( + _wrapper(), + _resolved(tmp_path / "model.pt"), + output_dir=output, + resume=output / "checkpoints" / "latest", + verbose=False, + ) + + +def _band_of(checkpoint: Path) -> list[list[list[float]]]: + _meta, images = load_checkpoint(checkpoint) + return [ + np.asarray(image.get_positions(), dtype=float).round(12).tolist() + for image in images + ] + + +def _checkpoints(output: Path) -> list[Path]: + return sorted((output / "checkpoints").glob("step_*")) + + +def test_resume_restores_the_checkpointed_band_exactly(tmp_path: Path) -> None: + """The resumed attempt must start from the checkpointed band, verbatim.""" + + _run(tmp_path, interrupt_after_checkpoints=3) + output = tmp_path / "run" + before = _checkpoints(output) + assert len(before) == 3 + interrupted_band = _band_of(output / "checkpoints" / "latest") + + _resume(tmp_path) + + after = _checkpoints(output) + assert len(after) > len(before), "resume must add checkpoints, not replace them" + # NEBCheckpointStore numbers checkpoints monotonically, so the first one + # after the interruption is the resumed attempt's own "prepared" snapshot. + first_of_resume = after[len(before)] + meta, _images = load_checkpoint(first_of_resume) + assert meta["stage"] == "prepared" + assert _band_of(first_of_resume) == interrupted_band + + +def test_resume_keeps_the_interrupted_attempts_evidence(tmp_path: Path) -> None: + """Checkpoints are immutable: the interrupted attempt's files survive.""" + + _run(tmp_path, interrupt_after_checkpoints=3) + output = tmp_path / "run" + names = [path.name for path in _checkpoints(output)] + + _resume(tmp_path) + + for name in names: + assert (output / "checkpoints" / name).is_dir() + exports = sorted(path.name for path in output.glob("vasp_path*")) + assert exports and all(name.startswith("vasp_path") for name in exports) + + +def test_resume_converges_to_the_same_saddle(tmp_path: Path) -> None: + """A geometry-resume re-optimises the band; the saddle must still match.""" + + reference = _run(tmp_path / "reference") + assert reference["status"] == "completed" + assert reference["converged"] is True + + _run(tmp_path / "resumed", interrupt_after_checkpoints=3) + resumed = _resume(tmp_path / "resumed") + assert resumed["status"] == "completed" + assert resumed["converged"] is True + + # Same reaction channel and same saddle, not the same path: the checkpoint + # is geometry-only, so the resumed optimiser restarts and lands on the same + # stationary point up to its own convergence tolerance. The observed + # difference for this fixture is ~2e-4 eV; 1e-3 eV is a loose guard against + # a gross divergence, not a precision claim. + assert resumed["climbing_image_index"] == reference["climbing_image_index"] + assert resumed["barrier_forward_sampled_eV"] == pytest.approx( + reference["barrier_forward_sampled_eV"], abs=1.0e-3 + ) + + +def test_geometry_resume_only_is_recorded_not_implicit(tmp_path: Path) -> None: + """The limitation is explicit in the checkpoint and the run context.""" + + _run(tmp_path, interrupt_after_checkpoints=2) + _resume(tmp_path) + + output = tmp_path / "run" + metadata, _images = load_checkpoint(output / "checkpoints" / "latest") + assert metadata["geometry_resume_only"] is True + + context = json.loads((output / "run_context.json").read_text(encoding="utf-8")) + assert context["geometry_resume_only"] is True