Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/workflows/neb.md
Original file line number Diff line number Diff line change
Expand Up @@ -147,3 +147,9 @@ identity: model, task, head and dtype, cell and PBC, constraints, band setup and
options. Step budgets are part of that identity in this beta, so a resume that
changes `--max-steps` is rejected with `Resume fingerprint is incompatible`. The
checkpoint is left untouched when that happens.

A resume restores the band geometry recorded in the checkpoint but restarts
NEB optimisation from there (`geometry_resume_only`); it does not replay the
previous optimiser state. Convergence, the final saddle and the barrier can
differ from an uninterrupted run, so verify convergence and compare the
results before interpreting them. Every earlier checkpoint and export is kept.
4 changes: 4 additions & 0 deletions docs/workflows/neb.zh.md
Original file line number Diff line number Diff line change
Expand Up @@ -116,3 +116,7 @@ ASE 的轨迹文件不保存自定义数组,所以检查点会把 `atom_id` /
续算在原来的目录里继续,并核对记录下来的信息:模型、task、head、精度,晶胞和 PBC,约束,能带
设置和各项参数。当前 beta 版本里步数上限也在核对范围内,续算时改 `--max-steps` 会报
`Resume fingerprint is incompatible` 并拒绝运行,检查点保持不变。

续算会恢复检查点里记录的能带几何结构,并从这里重新开始 NEB 优化(`geometry_resume_only`),
不会重放上一次的优化器状态。因此,续算不保证与一次跑完收敛到相同鞍点;是否收敛、最终鞍点和
势垒都可能不同。必须核实收敛状态并比较结果,再解释势垒。之前每一次的检查点和导出文件都会保留。
184 changes: 184 additions & 0 deletions tests/mliport/mliport/confsearch/test_resume_equivalence.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
"""AUD-007: an interrupted confsearch run must resume into the same result.

The tracker asks whether checkpoint resume truly preserves the structure, the
RNG streams and the model identity. Model identity is covered by
``test_resume_identity.py``; this module covers the other two by interrupting a
run after **every** durable unit boundary and asserting that the resumed run's
scientific result is identical to an uninterrupted run's.

The engine checkpoints after each unit (``checkpoints/confsearch.json``) and
stores the per-replica ``md_rng``/``mc_rng`` bit-generator states, so the
resumed process must continue the exact same random streams. The synthetic
``EggCrateCalculator`` is deterministic, so any divergence in the recorded
ranking, minima or diagnostics is a resume defect, not model noise.
"""

from __future__ import annotations

import json
from typing import TYPE_CHECKING

import pytest
from ase.io import write

from mliport.confsearch.engine import SearchEngine
from mliport.confsearch.workflow import run_confsearch_workflow

if TYPE_CHECKING:
from pathlib import Path


class _Interrupted(RuntimeError):
"""Test-only stand-in for a process that dies mid-run."""


class _SyntheticWrapper:
"""Backend-shaped wrapper around a synthetic calculator (no MLIP needed)."""

task = "bulk"

def __init__(self, calculator, model_path) -> None:
self._calculator = calculator
self._model_path = model_path

def info(self) -> dict:
return {
"model_type": "mace",
"task": "bulk",
"backend_version": "test",
"framework_versions": {"torch": "test"},
"default_dtype": "float64",
"model_path": str(self._model_path),
}

def get_calculator(self):
return self._calculator


def _setup(run_dir: Path, *, cycles: int, frames: int):
from mliport.config import resolve_config
from tests.mliport.mliport.confsearch.calculators import EggCrateCalculator
from tests.mliport.mliport.confsearch.test_engine import _start

run_dir.mkdir(parents=True, exist_ok=True)
model_file = run_dir / "model.pt"
model_file.write_bytes(b"identity model")
start_file = run_dir / "start.extxyz"
sites = [(0.5, 0.5), (0.25, 0.25), (0.75, 0.25)][:frames]
write(start_file, [_start(site) for site in sites], format="extxyz")
resolved = resolve_config(
calc_type="confsearch",
cli={
"model_type": "mace",
"model_path": str(model_file),
"task": "bulk",
"cs_starts": str(start_file),
"cs_adsorbate_indices": "16-18",
"cs_cycles": cycles,
"cs_hold_ps": 0.05,
"cs_cool_ps": 0.05,
"cs_snapshot_interval_ps": 0.02,
"timestep": 0.5,
"cs_temperature_high": 800.0,
"cs_temperature_low": 300.0,
"cs_quench_fmax": 0.01,
"cs_quench_steps": 2000,
"cs_refine": False,
"cs_refine_fmax": 0.005,
"seed": 7,
},
)
return resolved, _SyntheticWrapper(EggCrateCalculator(), model_file)


def _run(
run_dir: Path,
*,
cycles: int,
frames: int,
stop_after_units: int | None = None,
) -> dict:
"""Run one search, optionally killing it right after ``stop_after_units``."""

resolved, wrapper = _setup(run_dir, cycles=cycles, frames=frames)
output = run_dir / "run"

if stop_after_units is None:
return run_confsearch_workflow(wrapper, resolved, output_dir=output).results

original = SearchEngine._write_checkpoint
calls = {"n": 0}

def interrupt_after_checkpoint(self) -> None:
calls["n"] += 1
if calls["n"] > stop_after_units:
# The checkpoint of unit ``stop_after_units`` is already on disk.
raise _Interrupted(f"killed after {stop_after_units} unit(s)")
original(self)

SearchEngine._write_checkpoint = interrupt_after_checkpoint
try:
with pytest.raises(_Interrupted):
run_confsearch_workflow(wrapper, resolved, output_dir=output)
finally:
SearchEngine._write_checkpoint = original

checkpoint = json.loads(
(output / "checkpoints" / "confsearch.json").read_text(encoding="utf-8")
)
assert checkpoint["unit_index"] == stop_after_units

return run_confsearch_workflow(
wrapper, resolved, output_dir=output, resume=True
).results


def _comparable(results: dict) -> dict:
"""Drop the model *path*: where a model was stored is not identity."""

payload = json.loads(json.dumps(results, default=str))
model = payload.get("model")
if isinstance(model, dict):
model.pop("model_path", None)
return payload


@pytest.mark.parametrize(("frames", "cycles"), [(1, 3), (2, 2)])
def test_resume_reproduces_an_uninterrupted_run(tmp_path: Path, frames, cycles) -> None:
"""Every durable boundary must resume into the identical scientific result."""

n_units = 1 + frames * cycles # one start unit, then one unit per replica-cycle
reference = _comparable(_run(tmp_path / "reference", cycles=cycles, frames=frames))

for stop_after in range(1, n_units):
resumed = _comparable(
_run(
tmp_path / f"stop_{stop_after}",
cycles=cycles,
frames=frames,
stop_after_units=stop_after,
)
)
assert resumed == reference, f"resume after unit {stop_after} diverged"


def test_resume_restores_the_recorded_rng_states(tmp_path: Path) -> None:
"""The checkpoint must carry per-replica md/mc generator states."""

run_dir = tmp_path / "rng"
_run(run_dir, cycles=1, frames=2, stop_after_units=1)

checkpoint = json.loads(
(run_dir / "run" / "checkpoints" / "confsearch.json").read_text(
encoding="utf-8"
)
)
replicas = checkpoint["rng_states"]["replicas"]
assert [entry["index"] for entry in replicas] == [0, 1]
for entry in replicas:
for stream in ("md", "mc"):
state = entry[stream]
assert state["bit_generator"]
assert isinstance(state["state"], dict)
# Different replicas must not share one generator state.
assert replicas[0]["md"] != replicas[1]["md"]
Loading
Loading