diff --git a/mliport/mliport/confsearch/checkpoint.py b/mliport/mliport/confsearch/checkpoint.py new file mode 100644 index 00000000..287f76db --- /dev/null +++ b/mliport/mliport/confsearch/checkpoint.py @@ -0,0 +1,201 @@ +"""Atomic checkpoints, resume fingerprint and incomplete-attempt archiving. + +Reuses the NEB helpers (``atomic_write_json``, ``canonical_fingerprint``) so the +durability and identity contracts are the same ones the rest of mliport already +relies on. Spec section 11.2: a checkpoint is written after the start-quench +stage and after every (replica, cycle) unit, and resuming re-runs the +interrupted unit from the checkpointed RNG state. Raw files of an interrupted +attempt are moved to ``incomplete_attempt_N/`` and never deleted +(AGENTS.md rule 11). +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING + +import numpy as np + +from mliport.confsearch.errors import ConfsearchPreparationError +from mliport.confsearch.revisions import ( + CS_CHECKPOINT_SCHEMA_REVISION, + CS_SCIENTIFIC_REVISION, +) +from mliport.neb.io import atomic_write_json, canonical_fingerprint + +if TYPE_CHECKING: + from typing import Any + +CHECKPOINT_SCHEMA = "mliport.confsearch-checkpoint/1" +RESUME_FINGERPRINT_SCHEMA = "mliport.confsearch-resume-fingerprint/1" + + +@dataclass +class CheckpointData: + """Everything needed to resume a search exactly where it stopped.""" + + unit_index: int = 0 + stage: str = "prepared" + replicas: list[dict] = field(default_factory=list) + rng_states: dict[str, Any] = field(default_factory=dict) + registry: dict = field(default_factory=dict) + discovery: list[dict] = field(default_factory=list) + baseline: dict = field(default_factory=dict) + cumulative_md_ps: float = 0.0 + quench_index: int = 0 + + def to_dict(self) -> dict: + return { + "schema": CHECKPOINT_SCHEMA, + "checkpoint_schema_revision": CS_CHECKPOINT_SCHEMA_REVISION, + "scientific_revision": CS_SCIENTIFIC_REVISION, + "unit_index": int(self.unit_index), + "stage": self.stage, + "replicas": self.replicas, + "rng_states": self.rng_states, + "registry": self.registry, + "discovery": self.discovery, + "baseline": self.baseline, + "cumulative_md_ps": float(self.cumulative_md_ps), + "quench_index": int(self.quench_index), + } + + @classmethod + def from_dict(cls, payload: Mapping[str, Any]) -> CheckpointData: + return cls( + unit_index=int(payload.get("unit_index", 0)), + stage=str(payload.get("stage", "prepared")), + replicas=list(payload.get("replicas") or []), + rng_states=dict(payload.get("rng_states") or {}), + registry=dict(payload.get("registry") or {}), + discovery=list(payload.get("discovery") or []), + baseline=dict(payload.get("baseline") or {}), + cumulative_md_ps=float(payload.get("cumulative_md_ps", 0.0)), + quench_index=int(payload.get("quench_index", 0)), + ) + + +def build_resume_fingerprint( + *, + options, + model: Mapping[str, Any] | None, + input_hashes: Mapping[str, Any], + regions: Mapping[str, Any], + seed: int | None, + mliport_version: str, +) -> tuple[dict[str, Any], str]: + """Identity of a search: everything except pure output/verbosity settings. + + ``cs_cycles`` is deliberately included: ``extend`` is the only supported way + to change the budget, and it records the extension instead of pretending the + fingerprint is unchanged. + """ + import dataclasses # noqa: PLC0415 + + options_dict = { + item.name: _jsonable(getattr(options, item.name)) + for item in dataclasses.fields(options) + if not item.name.startswith("_") + } + value = { + "schema": RESUME_FINGERPRINT_SCHEMA, + "software": { + "mliport_version": mliport_version, + "cs_scientific_revision": CS_SCIENTIFIC_REVISION, + "cs_checkpoint_schema_revision": CS_CHECKPOINT_SCHEMA_REVISION, + }, + "model": dict(model or {}), + "input_hashes": dict(input_hashes), + "regions": dict(regions), + "seed": seed, + "options": options_dict, + } + return canonical_fingerprint(value) + + +def _jsonable(value: Any) -> Any: + if isinstance(value, (str, int, float, bool)) or value is None: + return value + if isinstance(value, (list, tuple)): + return [_jsonable(item) for item in value] + if isinstance(value, dict): + return {str(key): _jsonable(item) for key, item in value.items()} + if isinstance(value, np.ndarray): + return value.tolist() + return str(value) + + +def compare_fingerprints( + recorded: Mapping[str, Any], current: Mapping[str, Any] +) -> list[str]: + """Return the differing key paths between two resume fingerprints.""" + differences: list[str] = [] + + def walk(left: Any, right: Any, path: str) -> None: + if isinstance(left, Mapping) and isinstance(right, Mapping): + for key in sorted(set(left) | set(right)): + walk(left.get(key), right.get(key), f"{path}.{key}") + return + if left != right: + differences.append(f"{path}: recorded {left!r} != current {right!r}") + + walk(recorded, current, "") + return differences + + +class ConfsearchCheckpointStore: + """Write/read the single ``checkpoints/confsearch.json`` of one run.""" + + def __init__(self, run_dir: str | Path) -> None: + self.run_dir = Path(run_dir) + self.directory = self.run_dir / "checkpoints" + self.path = self.directory / "confsearch.json" + + def write(self, data: CheckpointData, fingerprint: Mapping[str, Any]) -> Path: + self.directory.mkdir(parents=True, exist_ok=True) + payload = data.to_dict() + payload["resume_fingerprint"] = dict(fingerprint) + payload["resume_fingerprint_sha256"] = canonical_fingerprint(dict(fingerprint))[ + 1 + ] + atomic_write_json(self.path, payload) + return self.path + + def load(self) -> tuple[CheckpointData, dict[str, Any]]: + if not self.path.is_file(): + raise ConfsearchPreparationError(f"No confsearch checkpoint at {self.path}") + payload = json.loads(self.path.read_text(encoding="utf-8")) + recorded = payload.get("resume_fingerprint") + if not isinstance(recorded, Mapping): + raise ConfsearchPreparationError("Checkpoint has no resume fingerprint") + expected = payload.get("resume_fingerprint_sha256") + actual = canonical_fingerprint(dict(recorded))[1] + if expected is not None and expected != actual: + raise ConfsearchPreparationError( + "Checkpoint resume fingerprint is corrupted (sha256 mismatch)" + ) + return CheckpointData.from_dict(payload), dict(recorded) + + def archive_incomplete(self, *, replica: int, cycle: int) -> Path | None: + """Move an interrupted cycle's raw directory aside; never delete it.""" + source = self.run_dir / "raw" / f"replica_{replica:03d}" / f"cycle_{cycle:03d}" + if not source.is_dir(): + return None + attempt = 1 + while (target := source.parent / f"incomplete_attempt_{attempt}").exists(): + attempt += 1 + source.rename(target) + return target + + +__all__ = [ + "CHECKPOINT_SCHEMA", + "RESUME_FINGERPRINT_SCHEMA", + "CheckpointData", + "ConfsearchCheckpointStore", + "build_resume_fingerprint", + "compare_fingerprints", +] diff --git a/mliport/mliport/confsearch/engine.py b/mliport/mliport/confsearch/engine.py new file mode 100644 index 00000000..cab4fac4 --- /dev/null +++ b/mliport/mliport/confsearch/engine.py @@ -0,0 +1,931 @@ +"""Confsearch search engine: cycles, MD segments, quench, Metropolis (spec 7). + +The engine is calculator-agnostic: it takes any ASE calculator with energy and +forces, which is what makes the synthetic-model end-to-end tests possible. The +model identity, gradient pre-flight and CLI/API wiring live one layer above +(phase P3). + +Flow (spec 7.1):: + + prepare starts -> stage "start_quench" (quench every start) + -> for cycle c = 1..cs_cycles: + for replica r = 0..n-1 (round robin, skip finished/failed): + run cycle (r, c); register its minima; checkpoint + -> consolidation -> refinement of top-K -> ranking, exports, diagnostics + +One model instance is used for the whole search. Every stochastic draw comes +from a per-replica ``SeedSequence`` stream (section 7.5), and every checkpoint +stores the bit-generator state before the next unit of work, so an interrupted +search resumes bitwise-identically. +""" + +from __future__ import annotations + +import json +import math +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import numpy as np +from ase import units +from ase.constraints import FixAtoms +from ase.md.langevin import Langevin +from ase.md.velocitydistribution import MaxwellBoltzmannDistribution + +from mliport.confsearch import status as cs_status +from mliport.confsearch.checkpoint import ( + CheckpointData, + ConfsearchCheckpointStore, +) +from mliport.confsearch.errors import ( + ConfsearchInternalError, + ConfsearchPreparationError, +) +from mliport.confsearch.geometry import wrap_positions +from mliport.confsearch.graphs import ( + ClassificationReference, + broken_reference_bonds, + classify_minimum, + covalent_bonds, +) +from mliport.confsearch.options import ConfsearchOptions +from mliport.confsearch.quench import QuenchResult, quench +from mliport.confsearch.regions import ( + Regions, + SurfaceGeometry, + mobile_indices, +) +from mliport.confsearch.registry import ( + MinimumOrigin, + MinimumRecord, + MinimumRegistry, +) +from mliport.confsearch.results import build_results +from mliport.runners.md_output import ( + AsyncMDOutputWriter, + MDFrameSnapshot, +) +from mliport.runners.safety import ( + evaluate_force_abort, + force_magnitudes, + force_views, + validate_safe_point, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Sequence + + from ase import Atoms + +#: Sampling distance for the "unique minima within 0.2 eV" discovery column. +DISCOVERY_WINDOW_EV = 0.2 + + +@dataclass(frozen=True, slots=True) +class PreparedStart: + """One validated start structure plus its regions and surface facts.""" + + index: int + atoms: Atoms + regions: Regions + surface: SurfaceGeometry + label: str + placement: Any = None + + def to_dict(self) -> dict: + return { + "index": self.index, + "label": self.label, + "n_atoms": len(self.atoms), + "regions": { + "adsorbate": self.regions.adsorbate.tolist(), + "slab": self.regions.slab.tolist(), + "fixed": self.regions.fixed.tolist(), + "relax": self.regions.relax.tolist(), + "fix_below_added": list(self.regions.fix_below_added), + "warnings": list(self.regions.warnings), + }, + "surface": { + "z_top_A": self.surface.z_top, + "z_min_A": self.surface.z_min, + "vacuum_gap_A": self.surface.vacuum_gap, + "adsorbate_centroid_z_A": self.surface.adsorbate_centroid_z, + }, + "placement": ( + self.placement.to_dict() if self.placement is not None else None + ), + } + + +@dataclass +class ReplicaState: + """One independent search chain.""" + + index: int + start: PreparedStart + md_rng: np.random.Generator + mc_rng: np.random.Generator + status: str = "active" + current: Atoms | None = None + current_energy: float | None = None + reference: ClassificationReference | None = None + start_minimum_energy: float | None = None + cycle_count: int = 0 + cycles_completed: list[int] = field(default_factory=list) + + def to_checkpoint(self) -> dict: + return { + "index": self.index, + "status": self.status, + "cycle_count": self.cycle_count, + "cycles_completed": list(self.cycles_completed), + "current_energy_eV": self.current_energy, + "start_minimum_energy_eV": self.start_minimum_energy, + "current": ( + None + if self.current is None + else { + "numbers": [int(z) for z in self.current.numbers], + "cell_A": np.asarray(self.current.cell, dtype=float).tolist(), + "pbc": [bool(value) for value in self.current.pbc], + "positions_A": np.asarray( + self.current.get_positions(), dtype=float + ).tolist(), + } + ), + "reference_start_positions_A": ( + None + if self.reference is None + else np.asarray(self.reference.start_positions, dtype=float).tolist() + ), + } + + +class _MDAbort(RuntimeError): + """Internal signal: the MD segment stopped with a recorded status.""" + + def __init__(self, status: str, message: str, frame: Atoms | None = None): + super().__init__(message) + self.status = status + self.frame = frame + + +class SearchEngine: + """Run one confsearch search (or resume one) with a single model instance.""" + + def __init__( + self, + *, + calculator, + options: ConfsearchOptions, + starts: Sequence[PreparedStart], + output_dir: str | Path, + seed: int, + model_record: dict | None = None, + gradient_preflight: dict | None = None, + resume: CheckpointData | None = None, + resume_fingerprint: dict | None = None, + log: Callable[[str], None] | None = None, + ) -> None: + if not starts: + raise ConfsearchPreparationError("confsearch needs at least one start") + self.calculator = calculator + self.options = options + self.starts = list(starts) + self.output_dir = Path(output_dir) + self.seed = int(seed) + self.model_record = dict(model_record or {}) + self.gradient_preflight = dict(gradient_preflight or {}) + self.resume = resume + self.resume_fingerprint = dict(resume_fingerprint or {}) + self._log_lines: list[str] = [] + self._log_callback = log + self.warnings: list[str] = [] + self.started = time.time() + + self.store = ConfsearchCheckpointStore(self.output_dir) + self.registry: MinimumRegistry + self.replicas: list[ReplicaState] + self.discovery: list[dict] = [] + self.baseline: dict = {} + self.cumulative_md_ps = 0.0 + self.quench_index = 0 + self.completed_units = 0 + self.stage = "prepared" + self._reference_graphs: tuple[Any, Any] | None = None + self._initialize_state() + + # -- logging ---------------------------------------------------------- + + def log(self, message: str) -> None: + self._log_lines.append(message) + if self._log_callback is not None: + self._log_callback(message) + + def log_lines(self) -> list[str]: + return list(self._log_lines) + + # -- state ------------------------------------------------------------ + + def _initialize_state(self) -> None: + adsorbate = np.asarray(self.starts[0].regions.adsorbate, dtype=int) + fixed = np.asarray(self.starts[0].regions.fixed, dtype=int) + self.registry = MinimumRegistry( + options=self.options, adsorbate=adsorbate, fixed=fixed + ) + seed_sequence = np.random.SeedSequence(self.seed) + placement_ss, *replica_sequences = seed_sequence.spawn(1 + len(self.starts)) + self.placement_seed_sequence = placement_ss + self.replicas = [] + for index, start in enumerate(self.starts): + md_ss, mc_ss = replica_sequences[index].spawn(2) + self.replicas.append( + ReplicaState( + index=index, + start=start, + md_rng=np.random.default_rng(md_ss), + mc_rng=np.random.default_rng(mc_ss), + ) + ) + if fixed.size == 0: + self.warnings.append( + "F is empty: the geometric identity rule G is disabled for this " + "search, so dedup relies on the fingerprint alone" + ) + + def _reference_identity(self) -> tuple[Any, Any]: + """Graphs shared by every start (section 8.1); fail closed on mismatch.""" + if self._reference_graphs is not None: + return self._reference_graphs + reference: tuple[Any, Any] | None = None + for start in self.starts: + regions = start.regions + graphs = ( + covalent_bonds( + start.atoms, + regions.adsorbate, + bond_scale=self.options.cs_bond_scale, + ionic_symbols=self.options.ionic_symbols, + ), + covalent_bonds( + start.atoms, + regions.slab, + bond_scale=self.options.cs_bond_scale, + ionic_symbols=self.options.ionic_symbols, + ), + ) + if reference is None: + reference = graphs + continue + if ( + graphs[0].bonds != reference[0].bonds + or graphs[1].bonds != reference[1].bonds + ): + raise ConfsearchPreparationError( + f"Start {start.index} has a different covalent graph than " + "start 0; all starts must describe the same molecule on the " + "same slab (section 8.1)" + ) + assert reference is not None + self._reference_graphs = reference + return reference + + # -- checkpoint ------------------------------------------------------- + + def _rng_states(self) -> dict: + return { + "replicas": [ + { + "index": replica.index, + "md": replica.md_rng.bit_generator.state, + "mc": replica.mc_rng.bit_generator.state, + } + for replica in self.replicas + ], + } + + def _write_checkpoint(self) -> None: + data = CheckpointData( + unit_index=self.completed_units, + stage=self.stage, + replicas=[replica.to_checkpoint() for replica in self.replicas], + rng_states=self._rng_states(), + registry=self.registry.to_dict(), + discovery=list(self.discovery), + baseline=dict(self.baseline), + cumulative_md_ps=self.cumulative_md_ps, + quench_index=self.quench_index, + ) + self.store.write(data, self.resume_fingerprint) + + def _restore(self) -> None: + assert self.resume is not None + self.completed_units = int(self.resume.unit_index) + self.stage = str(self.resume.stage) + self.discovery = list(self.resume.discovery) + self.baseline = dict(self.resume.baseline) + self.cumulative_md_ps = float(self.resume.cumulative_md_ps) + self.quench_index = int(self.resume.quench_index) + adsorbate = np.asarray(self.starts[0].regions.adsorbate, dtype=int) + fixed = np.asarray(self.starts[0].regions.fixed, dtype=int) + self.registry = MinimumRegistry.from_dict( + self.resume.registry, + options=self.options, + adsorbate=adsorbate, + fixed=fixed, + ) + states = { + int(entry["index"]): entry + for entry in (self.resume.rng_states or {}).get("replicas", []) + } + for entry in self.resume.replicas: + index = int(entry["index"]) + replica = self.replicas[index] + replica.status = str(entry.get("status", "active")) + replica.cycle_count = int(entry.get("cycle_count", 0)) + replica.cycles_completed = [ + int(value) for value in entry.get("cycles_completed", []) + ] + replica.current_energy = entry.get("current_energy_eV") + replica.start_minimum_energy = entry.get("start_minimum_energy_eV") + geometry = entry.get("current") + if geometry is not None: + from ase import Atoms # noqa: PLC0415 + + replica.current = Atoms( + numbers=np.asarray(geometry["numbers"], dtype=int), + positions=np.asarray(geometry["positions_A"], dtype=float), + cell=np.asarray(geometry["cell_A"], dtype=float), + pbc=np.asarray(geometry["pbc"], dtype=bool), + ) + reference_positions = entry.get("reference_start_positions_A") + if reference_positions is not None: + adsorbate_graph, slab_graph = self._reference_identity() + replica.reference = ClassificationReference( + adsorbate_graph=adsorbate_graph, + slab_graph=slab_graph, + start_positions=np.asarray(reference_positions, dtype=float), + ) + if index in states: + replica.md_rng.bit_generator.state = states[index]["md"] + replica.mc_rng.bit_generator.state = states[index]["mc"] + # Raw files of the interrupted attempt must survive (AGENTS.md rule 11). + for replica in self.replicas: + if replica.status != "active": + continue + cycle = replica.cycle_count + 1 + archived = self.store.archive_incomplete(replica=replica.index, cycle=cycle) + if archived is not None: + self.log(f"Archived interrupted attempt to {archived.name}") + + # -- plan ------------------------------------------------------------- + + def _plan(self) -> list[tuple[str, int, int]]: + units: list[tuple[str, int, int]] = [("start", 0, 0)] + for cycle in range(1, int(self.options.cs_cycles) + 1): + for replica in self.replicas: + units.append(("cycle", replica.index, cycle)) + return units + + # -- public entry point ----------------------------------------------- + + def run(self) -> dict: + self.output_dir.mkdir(parents=True, exist_ok=True) + if self.resume is not None: + self._restore() + self._reference_identity() + self._log_budget() + plan = self._plan() + for index, (kind, replica_index, cycle) in enumerate(plan): + if index < self.completed_units: + continue + if kind == "start": + self._run_start_stage() + self.stage = "start_quench_done" + else: + self._run_cycle_unit(replica_index, cycle) + self.stage = "cycles" + self.completed_units = index + 1 + self._write_checkpoint() + return self._finish() + + # -- budget / diagnostics --------------------------------------------- + + def _log_budget(self) -> None: + options = self.options + n_replicas = len(self.replicas) + md_steps = n_replicas * options.total_md_steps_per_cycle * options.cs_cycles + self.log( + "confsearch budget: " + f"{n_replicas} replicas x {options.cs_cycles} cycles, " + f"{md_steps} MD steps, <= {n_replicas * options.total_quench_upper_bound} " + "quenches" + ) + + def _discovery_row(self, record: MinimumRecord) -> dict: + valid = [ + unique + for unique in self.registry.uniques(cs_status.VALID) + if math.isfinite(unique.energy_eV) + ] + energies = [unique.energy_eV for unique in valid] + best = min(energies) if energies else None + within = ( + sum(1 for energy in energies if best is not None and energy - best <= 0.2) + if best is not None + else 0 + ) + energy = float(record.energy_eV) + return { + "cumulative_md_ps": float(self.cumulative_md_ps), + "quench_index": int(record.quench_index), + "unique_valid": len(valid), + "unique_valid_within_0p2eV": int(within), + "current_best_eV": best, + "energy_eV": energy if math.isfinite(energy) else None, + "status": record.status, + } + + # -- start stage ------------------------------------------------------ + + def _run_start_stage(self) -> None: + self.log("stage: quench every start") + adsorbate_graph, slab_graph = self._reference_identity() + for replica in self.replicas: + start = replica.start + result = quench( + start.atoms, + self.calculator, + self.options, + fixed_indices=start.regions.fixed, + ) + reference = ClassificationReference( + adsorbate_graph=adsorbate_graph, + slab_graph=slab_graph, + start_positions=np.asarray(result.atoms.get_positions(), dtype=float), + ) + record = self._register( + result, + regions=start.regions, + reference=reference, + origin=MinimumOrigin(replica=replica.index, cycle=0, source="start"), + md_ps=0.0, + ) + replica.reference = reference + replica.start_minimum_energy = record.energy_eV + if record.status != cs_status.VALID: + replica.status = cs_status.FAILED_START + replica.current = None + self.log( + f"replica {replica.index}: start quench is {record.status}; " + "marked failed_start (no cycles)" + ) + continue + replica.current = record.atoms.copy() + replica.current_energy = record.energy_eV + self.log( + f"replica {replica.index}: start minimum {record.energy_eV:.6f} eV" + ) + valid = [unique.energy_eV for unique in self.registry.uniques(cs_status.VALID)] + self.baseline = { + "stage": "start_quench", + "n_replicas": len(self.replicas), + "n_surviving": sum( + 1 for replica in self.replicas if replica.status == "active" + ), + "best_energy_eV": min(valid) if valid else None, + } + if not any(replica.status == "active" for replica in self.replicas): + raise ConfsearchPreparationError( + "No replica survived the start-quench stage; the search cannot " + "continue (section 7.1)" + ) + + # -- cycles ----------------------------------------------------------- + + def _run_cycle_unit(self, replica_index: int, cycle: int) -> None: + replica = self.replicas[replica_index] + if replica.status != "active" or replica.current is None: + return + self._run_cycle(replica, cycle) + replica.cycle_count = max(replica.cycle_count, cycle) + replica.cycles_completed.append(cycle) + + def _run_cycle(self, replica: ReplicaState, cycle: int) -> None: + options = self.options + regions = replica.start.regions + x0 = replica.current + assert x0 is not None + z_top = float(np.max(wrap_positions(x0)[regions.slab, 2])) + mobile = mobile_indices(x0, regions, options, z_top=z_top) + fixed_md = np.setdiff1d(np.arange(len(x0)), mobile) + + md_atoms = x0.copy() + md_atoms.set_constraint(FixAtoms(indices=fixed_md) if fixed_md.size else None) + md_atoms.calc = self.calculator + self._initialize_velocities(md_atoms, mobile, replica.md_rng) + + raw_dir = ( + self.output_dir + / "raw" + / f"replica_{replica.index:03d}" + / f"cycle_{cycle:03d}" + ) + raw_dir.mkdir(parents=True, exist_ok=True) + writer = AsyncMDOutputWriter( + trajectory_path=raw_dir / "trajectory.traj", + csv_path=raw_dir / "md.csv", + write_trajectory=True, + xdatcar_writer=None, + xdatcar_path=raw_dir / "XDATCAR", + outcar_writer=None, + ) + + dynamics = Langevin( + md_atoms, + timestep=float(options.timestep) * units.fs, + temperature_K=float(options.cs_temperature_high), + friction=float(options.friction) / units.fs, + rng=replica.md_rng, + fixcm=False, + ) + snapshots: list[Atoms] = [] + snapshot_steps = set(options.snapshot_steps) + abort_status: str | None = None + abort_message = "" + schedule: list[dict] = [] + + def observe() -> None: + nonlocal abort_status, abort_message + step = int(dynamics.nsteps) + phase = "hold" if step <= options.n_hold else "cool" + try: + energy = float(md_atoms.get_potential_energy()) + raw, applied = force_views( + md_atoms, energy, context=f"confsearch MD step {step}" + ) + kinetic, temperature, total = validate_safe_point( + md_atoms, + energy=energy, + forces_raw=raw, + forces_applied=applied, + temperature_fn=lambda atoms: float(atoms.get_temperature()), + context=f"confsearch MD step {step}", + ) + except (RuntimeError, ValueError) as exc: + abort_status = cs_status.MD_NONFINITE_ABORT + abort_message = str(exc) + raise _MDAbort(abort_status, abort_message, md_atoms.copy()) from exc + + gate = evaluate_force_abort(raw, threshold=options.fmax_abort) + frame = self._frame_summary( + step=step, + phase=phase, + energy=energy, + kinetic=kinetic, + total=total, + temperature=temperature, + volume=float(md_atoms.get_volume()), + raw=raw, + applied=applied, + gate_max=gate.max_force_raw, + ) + keep = ( + step % int(options.cs_save_interval) == 0 + or step in snapshot_steps + or step == options.total_md_steps_per_cycle + or gate.abort + ) + if keep: + # The raw trajectory keeps every requested frame, including the + # one that triggers an abort. + writer.submit( + MDFrameSnapshot( + atoms=md_atoms.copy(), summary=frame, outcar_forces=None + ) + ) + if step in snapshot_steps: + # Check the integrity *before* keeping the frame: a frame that + # already broke the adsorbate must not be quenched (spec 7.3 + # step 9 quenches only snapshots taken before the stop). + self._check_md_integrity(replica, md_atoms, step) + if gate.abort: + abort_status = cs_status.MD_FORCE_ABORT + abort_message = ( + f"raw force |F|={gate.max_force_raw:.6g} eV/A on atom " + f"{gate.atom_index} exceeds fmax_abort={options.fmax_abort}" + ) + raise _MDAbort(abort_status, abort_message, md_atoms.copy()) + if step in snapshot_steps: + snapshots.append(md_atoms.copy()) + + dynamics.attach(observe, interval=1) + steps_done = 0 + try: + dynamics.run(int(options.n_hold)) + steps_done += int(options.n_hold) + remaining = int(options.n_cool) + while remaining > 0: + block = min(int(options.cs_ramp_interval), remaining) + target = self._cool_temperature(steps_done, options) + dynamics.set_temperature(temperature_K=target) + schedule.append( + { + "step": int(dynamics.nsteps), + "target_temperature_K": float(target), + } + ) + dynamics.run(block) + steps_done += block + remaining -= block + # Spec 7.3 step 7: the end-of-cycle frame gets the same early + # integrity check as the hold-phase snapshots. + self._check_md_integrity(replica, md_atoms, steps_done) + except _MDAbort as exc: + abort_status = exc.status + abort_message = str(exc) + # The abort frame lives in the raw trajectory only; it is never + # quenched (spec 7.3 step 9). + finally: + try: + writer.close() + except Exception as exc: # pragma: no cover - writer failures are fatal + raise ConfsearchInternalError( + f"Raw MD output failed for replica {replica.index} cycle {cycle}: {exc}" + ) from exc + + self.cumulative_md_ps += steps_done * float(options.timestep) / 1000.0 + + # Quench the snapshots (chronological), then the end-of-cycle frame. + end_frame = md_atoms.copy() + quench_targets: list[tuple[str, int | None, Atoms]] = [ + ("snapshot", index, snapshot) for index, snapshot in enumerate(snapshots) + ] + if abort_status is None: + quench_targets.append(("end", None, end_frame)) + + end_record: MinimumRecord | None = None + for source, snapshot_index, frame in quench_targets: + result = quench( + frame, + self.calculator, + options, + fixed_indices=regions.fixed, + ) + record = self._register( + result, + regions=regions, + reference=replica.reference, + origin=MinimumOrigin( + replica=replica.index, + cycle=cycle, + source=source, + snapshot_index=snapshot_index, + ), + md_ps=self.cumulative_md_ps, + ) + if source == "end": + end_record = record + + cycle_payload = { + "replica": replica.index, + "cycle": cycle, + "status": abort_status or "completed", + "message": abort_message, + "steps_run": int(steps_done), + "n_hold": int(options.n_hold), + "n_cool": int(options.n_cool), + "snapshot_steps": list(options.snapshot_steps), + "temperature_schedule": schedule, + "md_time_ps": steps_done * float(options.timestep) / 1000.0, + "cumulative_md_ps": self.cumulative_md_ps, + "mobile_atoms": mobile.tolist(), + } + (raw_dir / "cycle.json").write_text( + json.dumps(cycle_payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + if abort_status is not None: + self.log( + f"replica {replica.index} cycle {cycle}: {abort_status} " + f"({abort_message}); current minimum unchanged" + ) + return + + self._metropolis(replica, end_record) + + def _cool_temperature(self, steps_done: int, options: ConfsearchOptions) -> float: + """Linear T(t) from cs_temperature_high to cs_temperature_low.""" + n_cool = max(1, options.n_cool) + fraction = min(1.0, max(0.0, (steps_done - options.n_hold) / n_cool)) + return float( + options.cs_temperature_high + + (options.cs_temperature_low - options.cs_temperature_high) * fraction + ) + + def _check_md_integrity( + self, replica: ReplicaState, atoms: Atoms, step: int + ) -> None: + reference = replica.reference + if reference is None: + return + broken = broken_reference_bonds( + atoms, + reference.adsorbate_graph, + break_scale=self.options.cs_bond_break_scale, + ionic_symbols=self.options.ionic_symbols, + ) + if broken: + raise _MDAbort( + cs_status.MD_INTEGRITY_ABORT, + f"adsorbate bond(s) {list(broken)} broke beyond " + f"{self.options.cs_bond_break_scale}x the covalent-radii sum at " + f"step {step}", + atoms.copy(), + ) + + def _initialize_velocities( + self, atoms: Atoms, mobile: np.ndarray, rng: np.random.Generator + ) -> None: + """Maxwell-Boltzmann over the mobile DOF; fixed atoms keep zero momenta.""" + MaxwellBoltzmannDistribution( + atoms, + temperature_K=float(self.options.cs_temperature_high), + force_temp=True, + rng=rng, + ) + fixed = np.setdiff1d(np.arange(len(atoms)), mobile) + momenta = np.asarray(atoms.get_momenta(), dtype=float) + if fixed.size: + momenta[fixed] = 0.0 + atoms.set_momenta(momenta) + ndof = int(atoms.get_number_of_degrees_of_freedom()) + kinetic = float(atoms.get_kinetic_energy()) + target = 0.5 * ndof * units.kB * float(self.options.cs_temperature_high) + if kinetic > 0.0: + atoms.set_momenta(momenta * math.sqrt(target / kinetic)) + + def _frame_summary( + self, + *, + step: int, + phase: str, + energy: float, + kinetic: float, + total: float, + temperature: float, + volume: float, + raw: np.ndarray, + applied: np.ndarray, + gate_max: float, + ) -> dict: + return { + "step": int(step), + "time_fs": float(step * float(self.options.timestep)), + "phase": phase, + "energy": float(energy), + "kinetic_energy": float(kinetic), + "total_energy": float(total), + "temperature": float(temperature), + "volume": float(volume), + "max_force_raw_eV_A": float(gate_max), + "max_force_applied_eV_A": float(np.max(force_magnitudes(applied))), + "configurational_stress": None, + "total_stress": None, + "configurational_pressure_gpa": None, + "total_pressure_gpa": None, + } + + # -- registration ----------------------------------------------------- + + def _register( + self, + result: QuenchResult, + *, + regions: Regions, + reference: ClassificationReference | None, + origin: MinimumOrigin, + md_ps: float, + ) -> MinimumRecord: + self.quench_index += 1 + if not result.usable: + classification = None + else: + assert reference is not None + classification = classify_minimum( + result.atoms, + regions=regions, + options=self.options, + reference=reference, + ) + if classification is None: + from mliport.confsearch.graphs import MinimumClassification # noqa: PLC0415 + + classification = MinimumClassification( + status=result.status, + flags=(), + adsorbate_slab_bonds=(), + reacted_bonds={}, + min_adsorbate_slab_distance=None, + min_self_image_distance=None, + min_vacuum_gap=None, + max_slab_displacement_ionic=None, + max_slab_displacement_nonionic=None, + ionic_coordination=(), + ) + record = MinimumRecord( + record_id=self.registry.new_record_id(), + energy_eV=float(result.energy_eV), + atoms=result.atoms, + classification=classification, + origin=origin, + quench_index=self.quench_index, + cumulative_md_ps=float(md_ps), + fmax_eV_A=float(result.fmax_eV_A), + steps=int(result.steps), + ) + self.registry.register(record) + self.discovery.append(self._discovery_row(record)) + return record + + def _metropolis( + self, replica: ReplicaState, end_record: MinimumRecord | None + ) -> None: + """Metropolis update on the end-of-cycle minimum (spec 7.3 step 11).""" + if end_record is None or replica.current_energy is None: + return + eligible = end_record.status == cs_status.VALID or ( + end_record.status == cs_status.REACTED + and self.options.cs_reaction_policy == "include" + ) + if not eligible: + return + delta = float(end_record.energy_eV) - float(replica.current_energy) + # The uniform draw happens every time an eligible proposal exists, so + # the stream stays aligned (section 7.3 step 11). + draw = float(replica.mc_rng.random()) + accept = delta <= 0.0 + if not accept: + exponent = -delta / (units.kB * float(self.options.cs_accept_temperature)) + accept = draw < math.exp(exponent) + if accept: + replica.current = end_record.atoms.copy() + replica.current_energy = float(end_record.energy_eV) + + # -- completion ------------------------------------------------------- + + def _finish(self) -> dict: + self.registry.consolidate() + references = [replica.reference for replica in self.replicas] + return build_results( + regions=self.starts[0].regions, + references=[item for item in references if item is not None], + calculator=self.calculator, + registry=self.registry, + options=self.options, + replicas=self.replicas, + discovery=self.discovery, + baseline=self.baseline, + warnings=self.warnings, + output_dir=self.output_dir, + model_record=self.model_record, + gradient_preflight=self.gradient_preflight, + cumulative_md_ps=self.cumulative_md_ps, + seed=self.seed, + starts=[start.to_dict() for start in self.starts], + ) + + +def run_confsearch( + calculator, + options: ConfsearchOptions, + *, + starts: Sequence[PreparedStart], + output_dir: str | Path, + seed: int, + model_record: dict | None = None, + gradient_preflight: dict | None = None, + resume: CheckpointData | None = None, + resume_fingerprint: dict | None = None, + log: Callable[[str], None] | None = None, +) -> dict: + """Run (or resume) one search and return its ``search_results.json`` content.""" + engine = SearchEngine( + calculator=calculator, + options=options, + starts=starts, + output_dir=output_dir, + seed=seed, + model_record=model_record, + gradient_preflight=gradient_preflight, + resume=resume, + resume_fingerprint=resume_fingerprint, + log=log, + ) + return engine.run() + + +__all__ = ["PreparedStart", "ReplicaState", "SearchEngine", "run_confsearch"] diff --git a/mliport/mliport/confsearch/graphs.py b/mliport/mliport/confsearch/graphs.py index 265b8eef..4b60930b 100644 --- a/mliport/mliport/confsearch/graphs.py +++ b/mliport/mliport/confsearch/graphs.py @@ -85,11 +85,13 @@ class MinimumClassification: flags: tuple[str, ...] adsorbate_slab_bonds: tuple[tuple[int, int, str, str, float, float], ...] reacted_bonds: dict[str, dict[str, list[list[int]]]] - min_adsorbate_slab_distance: float - min_self_image_distance: float - min_vacuum_gap: float - max_slab_displacement_ionic: float - max_slab_displacement_nonionic: float + #: ``None`` for a quench that never produced a converged geometry + #: (quench_failed / quench_not_converged): nothing was measured. + min_adsorbate_slab_distance: float | None + min_self_image_distance: float | None + min_vacuum_gap: float | None + max_slab_displacement_ionic: float | None + max_slab_displacement_nonionic: float | None ionic_coordination: tuple[tuple[int, int], ...] @property diff --git a/mliport/mliport/confsearch/quench.py b/mliport/mliport/confsearch/quench.py new file mode 100644 index 00000000..be2d146a --- /dev/null +++ b/mliport/mliport/confsearch/quench.py @@ -0,0 +1,179 @@ +"""Local minimization ("quench") and top-K refinement (spec sections 7.4, 10.3). + +Every quench is a fixed-cell ASE FIRE run on the relax set ``R`` with only the +input ``FixAtoms`` constraints active. The frame's velocities are dropped. +Non-finite values, a raw-force abort or a budget overrun are outcomes, not +exceptions: they are stored as ``quench_failed`` / ``quench_not_converged`` and +are never ranked and never become chain states. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import numpy as np +from ase.constraints import FixAtoms +from ase.optimize import FIRE + +from mliport.confsearch import status as cs_status +from mliport.confsearch.errors import ConfsearchInternalError +from mliport.runners.safety import ( + ForceSafetyAbort, + evaluate_force_abort, + force_views, +) + +if TYPE_CHECKING: + from ase import Atoms + + from mliport.confsearch.options import ConfsearchOptions + +#: FIRE defaults recorded with every quench (ASE's own default maxstep). +FIRE_MAXSTEP_A = 0.2 + + +@dataclass(frozen=True, slots=True) +class QuenchResult: + """Outcome of one fixed-cell quench.""" + + status: str + atoms: Atoms + energy_eV: float + fmax_eV_A: float + steps: int + maxstep_A: float = FIRE_MAXSTEP_A + + @property + def converged(self) -> bool: + return self.status == cs_status.VALID or self.status == cs_status.REACTED + + @property + def usable(self) -> bool: + return self.status not in { + cs_status.QUENCH_FAILED, + cs_status.QUENCH_NOT_CONVERGED, + } + + def to_dict(self) -> dict: + return { + "status": self.status, + "energy_eV": float(self.energy_eV), + "fmax_eV_A": float(self.fmax_eV_A), + "steps": int(self.steps), + "maxstep_A": float(self.maxstep_A), + } + + +def quench( + atoms: Atoms, + calculator, + options: ConfsearchOptions, + *, + fixed_indices: np.ndarray, + fmax: float | None = None, + steps: int | None = None, +) -> QuenchResult: + """Minimize ``atoms`` at fixed cell with FIRE on the relax set. + + ``fixed_indices`` becomes the only constraint; everything else moves. The + input frame is copied, so the caller's structure is never modified. + """ + work = atoms.copy() + work.calc = calculator + if "momenta" in work.arrays: + del work.arrays["momenta"] + fixed = np.asarray(fixed_indices, dtype=int) + work.set_constraint(FixAtoms(indices=fixed) if fixed.size else None) + + budget = int(options.cs_quench_steps if steps is None else steps) + target = float(options.cs_quench_fmax if fmax is None else fmax) + threshold = float(options.fmax_abort) + + abort: dict[str, object] = {} + + def observe() -> None: + energy = float(work.get_potential_energy()) + raw, applied = force_views(work, energy, context="confsearch quench") + gate = evaluate_force_abort(raw, threshold=threshold) + if gate.abort: + abort["reason"] = "force" + abort["max_force"] = gate.max_force_raw + abort["atom_index"] = gate.atom_index + raise ForceSafetyAbort( + step=int(optimizer.nsteps), + max_force=gate.max_force_raw, + atom_index=gate.atom_index, + threshold=threshold, + ) + del applied + + optimizer = FIRE(work, logfile=None, maxstep=FIRE_MAXSTEP_A) + optimizer.attach(observe, interval=1) + status = cs_status.QUENCH_NOT_CONVERGED + try: + optimizer.run(fmax=target, steps=budget) + except ForceSafetyAbort: + status = cs_status.QUENCH_FAILED + except (RuntimeError, ValueError, FloatingPointError): + status = cs_status.QUENCH_FAILED + else: + if optimizer.converged(): + status = cs_status.VALID + + try: + energy = float(work.get_potential_energy()) + except Exception: + # A calculator that raises on a bad geometry is a quench outcome, not a + # search crash. + energy = float("nan") + if not np.isfinite(energy): + status = cs_status.QUENCH_FAILED + try: + raw, _applied = force_views(work, energy, context="confsearch quench final") + except (RuntimeError, ValueError): + status = cs_status.QUENCH_FAILED + fmax_value = float("nan") + else: + magnitudes = np.linalg.norm(raw, axis=1) + free = np.setdiff1d(np.arange(len(work)), fixed) + fmax_value = float(np.max(magnitudes[free])) if free.size else 0.0 + + # A converged optimizer that left the fixed atoms moved is an internal bug. + if fixed.size and status == cs_status.VALID: + deltas = np.asarray(work.get_positions()) - np.asarray(atoms.get_positions()) + if np.any(deltas[fixed] != 0.0): + raise ConfsearchInternalError( + "A fixed atom moved during a confsearch quench" + ) + + return QuenchResult( + status=status, + atoms=work, + energy_eV=energy, + fmax_eV_A=fmax_value, + steps=int(optimizer.nsteps), + ) + + +def refine( + atoms: Atoms, + calculator, + options: ConfsearchOptions, + *, + fixed_indices: np.ndarray, +) -> QuenchResult: + """Re-quench a representative at the tighter refinement criterion.""" + if not options.cs_refine: + raise ConfsearchInternalError("refine() called with cs_refine disabled") + return quench( + atoms, + calculator, + options, + fixed_indices=fixed_indices, + fmax=options.cs_refine_fmax, + steps=options.cs_refine_steps, + ) + + +__all__ = ["FIRE_MAXSTEP_A", "QuenchResult", "quench", "refine"] diff --git a/mliport/mliport/confsearch/registry.py b/mliport/mliport/confsearch/registry.py index a00cd06a..1b8fed1b 100644 --- a/mliport/mliport/confsearch/registry.py +++ b/mliport/mliport/confsearch/registry.py @@ -25,6 +25,8 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING +import math + import numpy as np from mliport.confsearch.fingerprint import ( @@ -36,6 +38,9 @@ from mliport.confsearch.geometry import mic_vectors if TYPE_CHECKING: + from collections.abc import Mapping + from typing import Any + from ase import Atoms from mliport.confsearch.graphs import MinimumClassification @@ -145,7 +150,7 @@ def to_dict(self) -> dict: return { "id": self.unique_id, "status": self.status, - "energy_eV": float(representative.energy_eV), + "energy_eV": _json_float(representative.energy_eV), "flags": list(self.flags), "hit_count": self.hit_count, "replicas": sorted(self.replicas), @@ -153,8 +158,8 @@ def to_dict(self) -> dict: "first_found_quench_index": int(self.first_found_quench_index), "representative": { "record_id": representative.record_id, - "energy_eV": float(representative.energy_eV), - "fmax_eV_A": float(representative.fmax_eV_A), + "energy_eV": _json_float(representative.energy_eV), + "fmax_eV_A": _json_float(representative.fmax_eV_A), "steps": int(representative.steps), "origin": representative.origin.to_dict(), "numbers": [int(z) for z in representative.atoms.numbers], @@ -167,19 +172,71 @@ def to_dict(self) -> dict: "members": [ { "record_id": member.record_id, - "energy_eV": float(member.energy_eV), - "fmax_eV_A": float(member.fmax_eV_A), + "energy_eV": _json_float(member.energy_eV), + "fmax_eV_A": _json_float(member.fmax_eV_A), "steps": int(member.steps), "status": member.status, "flags": list(member.flags), "origin": member.origin.to_dict(), - "cumulative_md_ps": float(member.cumulative_md_ps), + "cumulative_md_ps": _json_float(member.cumulative_md_ps), + "quench_index": int(member.quench_index), } for member in self.members ], } +def _optional_float(value: Any) -> float | None: + return None if value is None else float(value) + + +def _json_float(value: Any) -> float | None: + """JSON has no NaN/Inf: a non-finite measurement is recorded as null.""" + if value is None: + return None + number = float(value) + return number if math.isfinite(number) else None + + +def rebuild_classification(payload: Mapping[str, Any]) -> MinimumClassification: + """Rebuild a :class:`MinimumClassification` from its serialized form.""" + from mliport.confsearch.graphs import MinimumClassification # noqa: PLC0415 + + bonds = tuple( + ( + int(item["adsorbate_index"]), + int(item["slab_index"]), + str(item["adsorbate_element"]), + str(item["slab_element"]), + float(item["distance_A"]), + float(item["q"]), + ) + for item in payload.get("adsorbate_slab_bonds", []) + ) + coordination = tuple( + (int(item["adsorbate_index"]), int(item["neighbours"])) + for item in payload.get("ionic_adsorbate_coordination", []) + ) + return MinimumClassification( + status=str(payload["status"]), + flags=tuple(payload.get("flags", [])), + adsorbate_slab_bonds=bonds, + reacted_bonds=dict(payload.get("reacted_bonds") or {}), + min_adsorbate_slab_distance=float( + payload.get("min_adsorbate_slab_distance_A", 0.0) + ), + min_self_image_distance=float(payload.get("min_self_image_distance_A", 0.0)), + min_vacuum_gap=float(payload.get("min_vacuum_gap_A", 0.0)), + max_slab_displacement_ionic=float( + payload.get("max_slab_displacement_ionic_A", 0.0) + ), + max_slab_displacement_nonionic=float( + payload.get("max_slab_displacement_nonionic_A", 0.0) + ), + ionic_coordination=coordination, + ) + + class MinimumRegistry: """Deduplicating registry of quench results, per status class.""" @@ -355,6 +412,10 @@ def to_dict(self) -> dict: return { "schema": "mliport.confsearch-registry/1", "n_records": self.total_records(), + "counters": { + "next_unique_id": self._next_unique_id, + "next_record_id": self._next_record_id, + }, "classes": { status: [ unique.to_dict() @@ -367,6 +428,79 @@ def to_dict(self) -> dict: "merges": list(self.merges), } + @classmethod + def from_dict( + cls, + payload: Mapping[str, Any], + *, + options: ConfsearchOptions, + adsorbate: np.ndarray, + fixed: np.ndarray | None = None, + ) -> MinimumRegistry: + """Rebuild a registry from a checkpoint payload (fingerprints recomputed).""" + from ase import Atoms # noqa: PLC0415 + + registry = cls( + options=options, adsorbate=np.asarray(adsorbate, dtype=int), fixed=fixed + ) + for status, entries in (payload.get("classes") or {}).items(): + for entry in entries: + representative = entry["representative"] + geometry = Atoms( + numbers=np.asarray(representative["numbers"], dtype=int), + positions=np.asarray(representative["positions_A"], dtype=float), + cell=np.asarray(representative["cell_A"], dtype=float), + pbc=np.asarray(representative["pbc"], dtype=bool), + ) + classification = rebuild_classification( + representative["classification"] + ) + unique = UniqueMinimum( + unique_id=int(entry["id"]), + status=str(entry.get("status", status)), + replicas={int(value) for value in entry.get("replicas", [])}, + first_found_md_ps=float(entry.get("first_found_md_ps", 0.0)), + first_found_quench_index=int( + entry.get("first_found_quench_index", 0) + ), + ) + for member in entry.get("members", []): + origin = member["origin"] + record = MinimumRecord( + record_id=int(member["record_id"]), + energy_eV=float(member["energy_eV"]), + atoms=geometry.copy(), + classification=classification, + origin=MinimumOrigin( + replica=int(origin["replica"]), + cycle=int(origin["cycle"]), + source=str(origin["source"]), + snapshot_index=origin.get("snapshot_index"), + ), + quench_index=int(member.get("quench_index", 0)), + cumulative_md_ps=float(member.get("cumulative_md_ps", 0.0)), + fmax_eV_A=float(member.get("fmax_eV_A", 0.0) or 0.0), + steps=int(member.get("steps", 0)), + ) + unique.add(record) + # Fingerprints are recomputed from the representative geometry + # so a resumed registry can keep matching (section 9.4). + representative_record = unique.representative + representative_record.fingerprint = registry.fingerprint( + representative_record.atoms + ) + registry._classes.setdefault(status, []).append(unique) + registry._unique_by_id[unique.unique_id] = unique + counters = payload.get("counters") or {} + registry._next_unique_id = int( + counters.get("next_unique_id", registry._next_unique_id) + ) + registry._next_record_id = int( + counters.get("next_record_id", registry._next_record_id) + ) + registry.merges = list(payload.get("merges") or []) + return registry + def displacement_between( self, first: MinimumRecord, second: MinimumRecord ) -> float: diff --git a/mliport/mliport/confsearch/results.py b/mliport/mliport/confsearch/results.py new file mode 100644 index 00000000..4f6105aa --- /dev/null +++ b/mliport/mliport/confsearch/results.py @@ -0,0 +1,1012 @@ +"""Ranking, saturation diagnostics, refinement and DFT exports (spec section 10). + +Everything here is a *screening* product: the outputs say "lowest found", never +"global minimum", and every number carries the provenance needed to re-rank the +candidates with DFT (spec sections 10.2, 15). + +Ranking rules enforced here (spec 10.1/10.3 revision 4, review of PR #64): + +* a refinement replaces the quench result only when it converged **and** its + classification is still rankable; an unconverged refinement keeps the quench + result and is flagged, and a refinement that changed to a non-rankable status + leaves the ranking with a record saying why; +* the refined set is re-deduplicated with the section 9.3 rule, and the merged + entry keeps the summed hit counts of its sources; +* refined and unrefined energies are never interleaved: each table has its own + best energy and its own export window, and the exports come from the refined + table whenever it exists. +""" + +from __future__ import annotations + +import csv +import json +import math +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import numpy as np +from ase import Atoms +from ase.constraints import FixAtoms +from ase.io import write as ase_write + +from mliport.confsearch import status as cs_status +from mliport.confsearch.graphs import covalent_bonds, classify_minimum, ionic_mask +from mliport.confsearch.quench import refine +from mliport.confsearch.registry import ( + MinimumRecord, + MinimumRegistry, + UniqueMinimum, +) +from mliport.writers.contcar import ContcarWriter + +if TYPE_CHECKING: + from collections.abc import Sequence + + from mliport.confsearch.graphs import ClassificationReference + from mliport.confsearch.options import ConfsearchOptions + from mliport.confsearch.regions import Regions + +RESULTS_SCHEMA = "mliport.confsearch-results/1" +#: Saturation window for the "new low minima in the last quarter" diagnostic. +NEW_LOW_WINDOW_EV = 0.2 + + +@dataclass(frozen=True, slots=True) +class RefinementOutcome: + """Refinement result for one representative.""" + + unique_id: int + status: str + energy_eV: float + fmax_eV_A: float + steps: int + atoms: Atoms + classification: Any + converged: bool + + def to_dict(self) -> dict: + return { + "status": self.status, + "energy_eV": float(self.energy_eV), + "fmax_eV_A": float(self.fmax_eV_A), + "steps": int(self.steps), + "converged": bool(self.converged), + "classification": self.classification.to_dict(), + } + + +def _index_ranges(indices: Sequence[int]) -> str: + """Compress 1-based indices into CP2K ``a..b`` ranges.""" + ordered = sorted(set(int(index) for index in indices)) + if not ordered: + return "" + parts: list[str] = [] + start = previous = ordered[0] + for index in ordered[1:]: + if index == previous + 1: + previous = index + continue + parts.append(str(start) if start == previous else f"{start}..{previous}") + start = previous = index + parts.append(str(start) if start == previous else f"{start}..{previous}") + return " ".join(parts) + + +def make_whole_adsorbate( + atoms: Atoms, regions: Regions, options: ConfsearchOptions +) -> Atoms: + """Wrap the slab and make the adsorbate whole (spec section 10.5). + + Slab atoms are wrapped into the cell. The non-ionic adsorbate is rebuilt by + a breadth-first walk over the covalent graph under MIC, each new atom placed + at the image closest to its parent; ionic adsorbate atoms are then placed at + the image closest to their nearest non-ionic adsorbate atom. + """ + work = atoms.copy() + work.wrap() + cell = np.asarray(work.cell, dtype=float) + inverse = np.linalg.inv(cell) + positions = np.asarray(work.get_positions(), dtype=float) + adsorbate = np.asarray(regions.adsorbate, dtype=int) + ionic = ionic_mask(work, options) + non_ionic = [int(index) for index in adsorbate if not ionic[index]] + + if non_ionic: + graph = covalent_bonds( + work, + np.asarray(non_ionic, dtype=int), + bond_scale=options.cs_bond_scale, + ionic_symbols=options.ionic_symbols, + ) + adjacency: dict[int, set[int]] = {index: set() for index in non_ionic} + for first, second in graph.bonds: + adjacency[int(first)].add(int(second)) + adjacency[int(second)].add(int(first)) + start = non_ionic[0] + visited = {start} + queue = [start] + while queue: + current = queue.pop(0) + for neighbour in sorted(adjacency[current]): + if neighbour in visited: + continue + delta = positions[neighbour] - positions[current] + fractional = delta @ inverse + fractional -= np.round(fractional) + positions[neighbour] = positions[current] + fractional @ cell + visited.add(neighbour) + queue.append(neighbour) + reference = np.asarray(non_ionic, dtype=int) + else: + reference = adsorbate + + for index in adsorbate: + if not ionic[index]: + continue + deltas = positions[reference] - positions[index] + fractional = deltas @ inverse + fractional -= np.round(fractional) + candidates = positions[index] + fractional @ cell + nearest = int(np.argmin(np.linalg.norm(candidates - positions[index], axis=1))) + positions[index] = candidates[nearest] + work.positions = positions + return work + + +# --------------------------------------------------------------------------- +# ranking entries +# --------------------------------------------------------------------------- + + +def _refined_entry( + group: dict, *, best_energy: float, n_atoms: int, total_quenches: int +) -> dict: + """Ranking entry for a re-deduplicated refined minimum.""" + energy = float(group["energy_eV"]) + return { + "id": int(group["ids"][0]), + "merged_ids": [int(value) for value in group["ids"]], + "status": str(group["status"]), + "energy_eV": energy, + "delta_energy_eV": float(energy - best_energy), + "delta_energy_per_atom_eV": float((energy - best_energy) / max(1, n_atoms)), + "hit_count": int(group["hit_count"]), + "replicas": sorted(int(value) for value in group["replicas"]), + "n_replicas": len(group["replicas"]), + "first_found_md_ps": float(group["first_found_md_ps"]), + "first_found_fraction": float( + group["first_found_quench_index"] / total_quenches + if total_quenches + else 0.0 + ), + "first_found_quench_index": int(group["first_found_quench_index"]), + "flags": list(group["flags"]), + "chemisorbed": bool(group["classification"].chemisorbed), + "refined": True, + "refinement_status": str(group["status"]), + "refinement_failed": False, + "quench_energy_eV": float(group["quench_energy_eV"]), + "fmax_eV_A": float(group["fmax_eV_A"]), + "classification": group["classification"].to_dict(), + "origin": group["origin"].to_dict(), + } + + +def _unrefined_entry( + unique: UniqueMinimum, + *, + best_energy: float, + n_atoms: int, + total_quenches: int, + refinement: RefinementOutcome | None = None, +) -> dict: + """Ranking entry for a quench result (no usable refinement).""" + representative = unique.representative + energy = float(unique.energy_eV) + return { + "id": int(unique.unique_id), + "merged_ids": [int(unique.unique_id)], + "status": unique.status, + "energy_eV": energy, + "delta_energy_eV": float(energy - best_energy), + "delta_energy_per_atom_eV": float((energy - best_energy) / max(1, n_atoms)), + "hit_count": int(unique.hit_count), + "replicas": sorted(int(value) for value in unique.replicas), + "n_replicas": len(unique.replicas), + "first_found_md_ps": float(unique.first_found_md_ps), + "first_found_fraction": float( + unique.first_found_quench_index / total_quenches if total_quenches else 0.0 + ), + "first_found_quench_index": int(unique.first_found_quench_index), + "flags": list(unique.flags), + "chemisorbed": representative.classification.chemisorbed, + "refined": False, + "refinement_status": None if refinement is None else refinement.status, + "refinement_failed": bool(refinement is not None and not refinement.converged), + "quench_energy_eV": energy, + "fmax_eV_A": float(representative.fmax_eV_A), + "classification": representative.classification.to_dict(), + "origin": representative.origin.to_dict(), + } + + +def _dedup_refined( + accepted: dict[int, RefinementOutcome], + *, + quench_rankable: list[tuple[UniqueMinimum, float]], + options: ConfsearchOptions, + regions: Regions, +) -> list[dict]: + """Re-deduplicate the refined set with the section 9.3 rule (spec 10.3). + + A temporary registry applies the same ``dE`` AND (``G`` OR ``D``) matching to + the refined representatives in ascending refined energy; the merged group + keeps the summed hit counts, the union of replicas and the earliest + first-found record of its sources. + """ + by_id = {unique.unique_id: unique for unique, _energy in quench_rankable} + scratch = MinimumRegistry( + options=options, adsorbate=regions.adsorbate, fixed=regions.fixed + ) + groups: dict[int, dict] = {} + ordered = sorted(accepted.items(), key=lambda item: (item[1].energy_eV, item[0])) + for unique_id, outcome in ordered: + source = by_id[unique_id] + record = MinimumRecord( + record_id=scratch.new_record_id(), + energy_eV=float(outcome.energy_eV), + atoms=outcome.atoms, + classification=outcome.classification, + origin=source.representative.origin, + quench_index=int(source.first_found_quench_index), + cumulative_md_ps=float(source.first_found_md_ps), + fmax_eV_A=float(outcome.fmax_eV_A), + steps=int(outcome.steps), + ) + registered = scratch.register(record) + group = groups.get(registered.unique_id) + if group is None: + groups[registered.unique_id] = { + "energy_eV": float(outcome.energy_eV), + "quench_energy_eV": float(source.energy_eV), + "atoms": outcome.atoms, + "classification": outcome.classification, + "fmax_eV_A": float(outcome.fmax_eV_A), + "steps": int(outcome.steps), + "ids": [int(unique_id)], + "hit_count": int(source.hit_count), + "replicas": {int(value) for value in source.replicas}, + "first_found_md_ps": float(source.first_found_md_ps), + "first_found_quench_index": int(source.first_found_quench_index), + "status": str(outcome.status), + # Spec 10.3 revision 6: a refined entry's flags come from the + # refined classification (chemisorbed/physisorbed and the + # far-field slab flag can all change during refinement). + "flags": tuple(outcome.classification.flags), + "origin": source.representative.origin, + } + continue + group["ids"].append(int(unique_id)) + group["hit_count"] += int(source.hit_count) + group["replicas"] |= {int(value) for value in source.replicas} + group["first_found_md_ps"] = min( + group["first_found_md_ps"], float(source.first_found_md_ps) + ) + group["first_found_quench_index"] = min( + group["first_found_quench_index"], int(source.first_found_quench_index) + ) + return [groups[key] for key in sorted(groups)] + + +def _refined_payload(group: dict, energy: float) -> dict: + """``minimum.json`` body for a refined (possibly merged) entry.""" + atoms = group["atoms"] + return { + "id": int(group["ids"][0]), + "merged_ids": [int(value) for value in group["ids"]], + "status": str(group["status"]), + "energy_eV": float(energy), + "flags": list(group["flags"]), + "hit_count": int(group["hit_count"]), + "replicas": sorted(int(value) for value in group["replicas"]), + "first_found_md_ps": float(group["first_found_md_ps"]), + "first_found_quench_index": int(group["first_found_quench_index"]), + "representative": { + "record_id": None, + "energy_eV": float(energy), + "fmax_eV_A": float(group["fmax_eV_A"]), + "steps": int(group["steps"]), + "origin": group["origin"].to_dict(), + "numbers": [int(z) for z in atoms.numbers], + "symbols": atoms.get_chemical_symbols(), + "cell_A": np.asarray(atoms.cell, dtype=float).tolist(), + "pbc": [bool(value) for value in atoms.pbc], + "positions_A": np.asarray(atoms.get_positions(), dtype=float).tolist(), + "classification": group["classification"].to_dict(), + }, + "members": [], + } + + +# --------------------------------------------------------------------------- +# exports +# --------------------------------------------------------------------------- + + +def _export_candidate( + directory: Path, + *, + unique_id: int, + payload: dict, + atoms: Atoms, + options: ConfsearchOptions, + regions: Regions, + entry: dict, + energy: float, +) -> dict: + """Write the DFT-facing files for one exported rank (spec section 10.5).""" + directory.mkdir(parents=True, exist_ok=True) + whole = make_whole_adsorbate(atoms, regions, options) + + constrained = whole.copy() + fixed = np.asarray(regions.fixed, dtype=int) + constrained.set_constraint(FixAtoms(indices=fixed) if fixed.size else None) + ContcarWriter().write( + constrained, + directory / "POSCAR", + comment=f"confsearch rank {unique_id} E={energy:.6f} eV", + direct=True, + ) + + export_atoms = whole.copy() + export_atoms.info["confsearch_id"] = int(unique_id) + export_atoms.info["confsearch_status"] = entry["status"] + export_atoms.info["energy_eV"] = float(energy) + export_atoms.info["flags"] = list(entry["flags"]) + ase_write(directory / "structure.extxyz", export_atoms, format="extxyz") + ase_write(directory / "cp2k_coord.xyz", export_atoms, format="xyz") + + cell = np.asarray(whole.cell, dtype=float) + (directory / "cp2k_cell.inc").write_text( + "".join( + f"{name} {vector[0]:.12f} {vector[1]:.12f} {vector[2]:.12f}\n" + for name, vector in zip("ABC", cell, strict=True) + ), + encoding="utf-8", + ) + (directory / "cp2k_fixed_atoms.inc").write_text( + "# CP2K &MOTION/&CONSTRAINT/&FIXED_ATOMS list, 1-based indices\n" + f"LIST {_index_ranges([int(index) + 1 for index in fixed])}\n".lstrip() + if fixed.size + else "# CP2K &MOTION/&CONSTRAINT/&FIXED_ATOMS list: no fixed atoms\n", + encoding="utf-8", + ) + + body = dict(payload) + body["export"] = { + "rank_energy_eV": float(energy), + "ranking": entry, + "coordinate_convention": ( + "slab wrapped into the cell; adsorbate made whole under MIC" + ), + } + (directory / "minimum.json").write_text( + json.dumps(body, indent=2, sort_keys=True, default=_json_default) + "\n", + encoding="utf-8", + ) + files = [ + "POSCAR", + "structure.extxyz", + "cp2k_coord.xyz", + "cp2k_cell.inc", + "cp2k_fixed_atoms.inc", + "minimum.json", + ] + return { + "rank": int(unique_id), + "path": directory.name, + "energy_eV": float(energy), + "refined": bool(entry.get("refined")), + "refinement_failed": bool(entry.get("refinement_failed")), + "files": files, + } + + +# --------------------------------------------------------------------------- +# diagnostics +# --------------------------------------------------------------------------- + + +def _saturation(entries: list[dict], *, total_quenches: int) -> dict: + if not entries: + return { + "best_hit_count": 0, + "best_replicas": [], + "best_first_found_fraction": None, + "best_repeatedly_found": False, + "new_low_minima_last_quarter": 0, + "lowest_found_language": "lowest found (no global-minimum claim)", + } + best = entries[0] + best_energy = float(best["energy_eV"]) + fraction = ( + float(best["first_found_quench_index"]) / total_quenches + if total_quenches + else 0.0 + ) + cutoff = int(math.ceil(0.75 * total_quenches)) + recent = sum( + 1 + for entry in entries + if float(entry["energy_eV"]) - best_energy <= NEW_LOW_WINDOW_EV + and int(entry["first_found_quench_index"]) > cutoff + ) + return { + "best_id": int(best["id"]), + "best_energy_eV": best_energy, + "best_hit_count": int(best["hit_count"]), + "best_replicas": list(best["replicas"]), + "best_first_found_md_ps": float(best["first_found_md_ps"]), + "best_first_found_fraction": fraction, + "best_repeatedly_found": bool( + int(best["hit_count"]) >= 3 and len(best["replicas"]) >= 2 + ), + "new_low_minima_last_quarter": int(recent), + "lowest_found_language": "lowest found (no global-minimum claim)", + } + + +def _status_warnings( + registry: MinimumRegistry, options: ConfsearchOptions, ranked: list[dict] +) -> list[str]: + warnings: list[str] = [] + all_records = registry.total_records() + if not all_records: + return warnings + classes = registry.status_classes() + reacted = sum(unique.hit_count for unique in classes.get(cs_status.REACTED, [])) + if reacted / all_records > 0.10: + warnings.append( + f"{100 * reacted / all_records:.1f}% of quenches are reacted; the " + "temperature may be too high or the model unreliable for this chemistry" + ) + unusable = sum( + unique.hit_count + for status in (cs_status.DESORBED, cs_status.IMAGE_CONTACT) + for unique in classes.get(status, []) + ) + if unusable / all_records > 0.10: + warnings.append( + f"{100 * unusable / all_records:.1f}% of quenches are desorbed or in " + "image contact; check the cell vacuum and the placement scales" + ) + not_converged = sum( + unique.hit_count for unique in classes.get(cs_status.QUENCH_NOT_CONVERGED, []) + ) + if not_converged / all_records > 0.05: + warnings.append( + f"{100 * not_converged / all_records:.1f}% of quenches did not converge " + "within cs_quench_steps" + ) + for entry in ranked[: int(options.cs_top_k)]: + if "far_field_slab_changed" in entry["flags"]: + warnings.append( + f"top-{options.cs_top_k} minimum {entry['id']} has " + "far_field_slab_changed: its energy difference may reflect slab " + "changes rather than adsorption" + ) + return warnings + + +def _baseline_report( + baseline: dict, + top_entries: list[dict], + *, + uniques_by_id: dict[int, UniqueMinimum], + options: ConfsearchOptions, +) -> dict: + """Baseline comparison (spec 10.4). + + ``top_k_never_reached_by_start_quench`` lists the top-K minima that have **no + member** whose origin is a start quench -- not merely a representative that + did not come from a start. + """ + report = dict(baseline) + best_baseline = baseline.get("best_energy_eV") + if best_baseline is None or not top_entries: + report["top_k_never_reached_by_start_quench"] = None + return report + never: list[int] = [] + for entry in top_entries[: int(options.cs_top_k)]: + ids = entry.get("merged_ids") or [entry["id"]] + reached = False + for unique_id in ids: + unique = uniques_by_id.get(int(unique_id)) + if unique is None: + continue + if any(member.origin.source == "start" for member in unique.members): + reached = True + break + if not reached: + never.append(int(entry["id"])) + report["top_k_never_reached_by_start_quench"] = never + return report + + +# --------------------------------------------------------------------------- +# results assembly +# --------------------------------------------------------------------------- + + +def build_results( + *, + registry: MinimumRegistry, + options: ConfsearchOptions, + replicas: Sequence[Any], + discovery: list[dict], + baseline: dict, + warnings: list[str], + output_dir: Path, + model_record: dict, + gradient_preflight: dict, + cumulative_md_ps: float, + seed: int, + starts: list[dict], + regions: Regions | None = None, + references: Sequence[ClassificationReference] | None = None, + calculator=None, +) -> dict: + """Write every result artifact and return the ``search_results.json`` body.""" + output_dir = Path(output_dir) + (output_dir / "diagnostics").mkdir(parents=True, exist_ok=True) + _write_discovery_csv(output_dir / "diagnostics" / "discovery.csv", discovery) + _write_minima(output_dir, registry) + + rankable_statuses = cs_status.rankable_statuses(options.cs_reaction_policy) + quench_rankable = sorted( + ( + (unique, unique.energy_eV) + for status in rankable_statuses + for unique in registry.uniques(status) + ), + key=lambda item: (item[1], item[0].unique_id), + ) + total_quenches = registry.total_records() + n_atoms = len(quench_rankable[0][0].representative.atoms) if quench_rankable else 0 + uniques_by_id = {unique.unique_id: unique for unique, _ in quench_rankable} + + refinements: dict[int, RefinementOutcome] = {} + if ( + options.cs_refine + and quench_rankable + and calculator is not None + and regions is not None + ): + refinements = _refine_top_k( + quench_rankable, + options=options, + regions=regions, + references=references or [], + replicas=replicas, + calculator=calculator, + ) + + # Spec 10.1/10.3: only a converged refinement whose status is still rankable + # replaces the quench result. Everything else is recorded with its reason. + accepted: dict[int, RefinementOutcome] = {} + removed: set[int] = set() + refinement_notes: list[dict] = [] + for unique, _quench_energy in quench_rankable: + outcome = refinements.get(unique.unique_id) + if outcome is None: + continue + if not outcome.converged: + refinement_notes.append( + { + "id": unique.unique_id, + "reason": "refinement_not_converged", + "refinement_status": outcome.status, + "kept": "quench_result", + } + ) + continue + if outcome.status not in rankable_statuses: + # The refinement is the tighter classification: an entry that is no + # longer rankable leaves the ranking entirely (spec 10.1). + removed.add(unique.unique_id) + refinement_notes.append( + { + "id": unique.unique_id, + "reason": "refinement_changed_status", + "refinement_status": outcome.status, + "kept": "none (no longer rankable)", + "energy_eV": float(outcome.energy_eV), + } + ) + continue + accepted[unique.unique_id] = outcome + + refined_groups = ( + _dedup_refined( + accepted, + quench_rankable=quench_rankable, + options=options, + regions=regions, + ) + if accepted and regions is not None + else [] + ) + + refined_best = min((group["energy_eV"] for group in refined_groups), default=None) + unrefined_candidates = [ + (unique, energy) + for unique, energy in quench_rankable + if unique.unique_id not in accepted and unique.unique_id not in removed + ] + unrefined_best = min( + (energy for _unique, energy in unrefined_candidates), default=None + ) + + refined_entries = [ + _refined_entry( + group, + best_energy=float(refined_best), + n_atoms=n_atoms, + total_quenches=total_quenches, + ) + for group in sorted( + refined_groups, key=lambda group: (group["energy_eV"], group["ids"][0]) + ) + ] + unrefined_entries = [ + _unrefined_entry( + unique, + best_energy=float(unrefined_best), + n_atoms=n_atoms, + total_quenches=total_quenches, + refinement=refinements.get(unique.unique_id), + ) + for unique, _energy in sorted( + unrefined_candidates, key=lambda item: (item[1], item[0].unique_id) + ) + ] + + # Never interleave energies at different fmax: exports come from the refined + # table when it exists, and the window is relative to that table's best. + export_groups: dict[int, dict] = {} + if refined_entries: + export_source = refined_entries + export_groups = {int(group["ids"][0]): group for group in refined_groups} + best_export: float | None = float(refined_best) + else: + export_source = unrefined_entries + best_export = float(unrefined_best) if unrefined_best is not None else None + + exports: list[dict] = [] + if regions is not None and export_source and best_export is not None: + window = [ + entry + for entry in export_source + if float(entry["energy_eV"]) - best_export <= options.cs_export_window + ][: int(options.cs_top_k)] + for position, entry in enumerate(window, start=1): + unique_id = int(entry["id"]) + if unique_id in export_groups: + group = export_groups[unique_id] + atoms = group["atoms"] + payload = _refined_payload(group, float(entry["energy_eV"])) + else: + unique = uniques_by_id[unique_id] + atoms = unique.representative.atoms + payload = unique.to_dict() + exports.append( + _export_candidate( + output_dir / "top" / f"rank_{position:03d}", + unique_id=unique_id, + payload=payload, + atoms=atoms, + options=options, + regions=regions, + entry=entry, + energy=float(entry["energy_eV"]), + ) + ) + + # Spec 10.3 revision 6: a candidate in the refinement window whose + # refinement did not converge must still reach the DFT deliverable. It is + # exported with its quench geometry and energy as top/unrefined_NNN/ and a + # warning names it -- a failed refinement must never silently remove a top + # candidate. + unrefined_by_id = {int(entry["id"]): entry for entry in unrefined_entries} + unconverged_ids = sorted( + ( + int(unique_id) + for unique_id, outcome in refinements.items() + if not outcome.converged and int(unique_id) in unrefined_by_id + ), + key=lambda unique_id: ( + float(unrefined_by_id[unique_id]["energy_eV"]), + unique_id, + ), + ) + if regions is not None: + for position, unique_id in enumerate(unconverged_ids, start=1): + unique = uniques_by_id[unique_id] + entry = unrefined_by_id[unique_id] + exports.append( + _export_candidate( + output_dir / "top" / f"unrefined_{position:03d}", + unique_id=unique_id, + payload=unique.to_dict(), + atoms=unique.representative.atoms, + options=options, + regions=regions, + entry=entry, + energy=float(entry["energy_eV"]), + ) + ) + + ranked_for_diagnostics = refined_entries or unrefined_entries + saturation = _saturation(ranked_for_diagnostics, total_quenches=total_quenches) + baseline_report = _baseline_report( + baseline, + ranked_for_diagnostics, + uniques_by_id=uniques_by_id, + options=options, + ) + warnings = list(warnings) + if unconverged_ids: + warnings.append( + "refinement did not converge for candidate(s) " + + ", ".join(str(unique_id) for unique_id in unconverged_ids) + + "; they are exported as top/unrefined_NNN/ with their quench " + "geometry and energy (spec 10.3)" + ) + warnings += _status_warnings(registry, options, ranked_for_diagnostics) + warnings = list(dict.fromkeys(warnings)) + + status = "completed" + if replicas and not any( + getattr(replica, "status", None) == "active" for replica in replicas + ): + status = "failed" + + results = { + "schema": RESULTS_SCHEMA, + "status": status, + "scientific_revision": _scientific_revision(), + "seed": int(seed), + "settings": { + "cs_mode": options.cs_mode, + "cs_reaction_policy": options.cs_reaction_policy, + "cs_refine": bool(options.cs_refine), + "cs_dedup_energy_tol": float(options.cs_dedup_energy_tol), + "cs_dedup_geom_tol": float(options.cs_dedup_geom_tol), + "cs_dedup_fp_tol": float(options.cs_dedup_fp_tol), + "cs_fp_sigma": float(options.cs_fp_sigma), + }, + "ranking": { + "best_energy_eV": ( + float(refined_best) + if refined_entries + else (float(unrefined_best) if unrefined_best is not None else None) + ), + "best_energy_source": "refined" if refined_entries else "quench", + "n_rankable": len(refined_entries) + len(unrefined_entries), + "refined": refined_entries, + "unrefined": unrefined_entries, + }, + "diagnostics": { + "discovery_curve": "diagnostics/discovery.csv", + "n_quenches": total_quenches, + "cumulative_md_ps": float(cumulative_md_ps), + "saturation": saturation, + "registry": registry.to_dict(), + }, + "baseline": baseline_report, + "exports": exports, + "warnings": warnings, + "starts": starts, + "model": model_record, + "gradient_preflight": gradient_preflight, + "refinement": { + "enabled": bool(options.cs_refine), + "fmax_eV_A": float(options.cs_refine_fmax), + "steps": int(options.cs_refine_steps), + "n_refined": len(accepted), + "n_merged_into_existing": int( + sum(len(group["ids"]) - 1 for group in refined_groups) + ), + "notes": refinement_notes, + }, + } + (output_dir / "search_results.json").write_text( + json.dumps(results, indent=2, sort_keys=True, default=_json_default) + "\n", + encoding="utf-8", + ) + _write_artifacts(output_dir, results) + return results + + +def _scientific_revision() -> int: + from mliport.confsearch.revisions import CS_SCIENTIFIC_REVISION # noqa: PLC0415 + + return CS_SCIENTIFIC_REVISION + + +def _json_default(value: Any) -> Any: + if isinstance(value, np.ndarray): + return value.tolist() + if isinstance(value, (np.floating, np.integer)): + return value.item() + return str(value) + + +def _finite_info(info: dict) -> dict: + """Drop non-finite floats: extxyz cannot carry NaN/Inf.""" + cleaned: dict = {} + for key, value in info.items(): + if isinstance(value, float) and not math.isfinite(value): + continue + if isinstance(value, (list, tuple)): + cleaned[key] = list(value) + continue + cleaned[key] = value + return cleaned + + +def _write_minima(output_dir: Path, registry: MinimumRegistry) -> None: + """Write ``minima/all_minima.extxyz`` and ``minima/registry.json`` (spec 11.1). + + Every quench result is kept, whatever its status, so rejected, failed and + ``reacted`` geometries can be inspected after the run. + """ + directory = output_dir / "minima" + directory.mkdir(parents=True, exist_ok=True) + frames: list[Atoms] = [] + for status in sorted(registry.status_classes()): + for unique in sorted(registry.uniques(status), key=lambda item: item.unique_id): + for member in sorted(unique.members, key=lambda item: item.record_id): + atoms = member.atoms.copy() + atoms.info.update( + _finite_info( + { + "confsearch_unique_id": int(unique.unique_id), + "confsearch_record_id": int(member.record_id), + "confsearch_status": member.status, + "confsearch_flags": list(member.flags), + "energy_eV": float(member.energy_eV), + "fmax_eV_A": float(member.fmax_eV_A), + "quench_steps": int(member.steps), + "quench_index": int(member.quench_index), + "cumulative_md_ps": float(member.cumulative_md_ps), + "origin_replica": int(member.origin.replica), + "origin_cycle": int(member.origin.cycle), + "origin_source": member.origin.source, + } + ) + ) + frames.append(atoms) + if frames: + ase_write(directory / "all_minima.extxyz", frames, format="extxyz") + (directory / "registry.json").write_text( + json.dumps(registry.to_dict(), indent=2, sort_keys=True, default=_json_default) + + "\n", + encoding="utf-8", + ) + + +def _write_discovery_csv(path: Path, rows: list[dict]) -> None: + fieldnames = [ + "cumulative_md_ps", + "quench_index", + "unique_valid", + "unique_valid_within_0p2eV", + "current_best_eV", + "energy_eV", + "status", + ] + with path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + for row in rows: + writer.writerow({name: row.get(name, "") for name in fieldnames}) + + +def _refine_top_k( + rankable: list[tuple[UniqueMinimum, float]], + *, + options: ConfsearchOptions, + regions: Regions, + references: Sequence[ClassificationReference], + replicas: Sequence[Any], + calculator, +) -> dict[int, RefinementOutcome]: + best = rankable[0][1] + window = [ + (unique, energy) + for unique, energy in rankable + if energy - best <= options.cs_export_window + ][: int(options.cs_top_k)] + outcomes: dict[int, RefinementOutcome] = {} + for unique, _energy in window: + representative = unique.representative + result = refine( + representative.atoms, + calculator, + options, + fixed_indices=regions.fixed, + ) + reference = _reference_for_record(representative, references, replicas) + classification = classify_minimum( + result.atoms, regions=regions, options=options, reference=reference + ) + outcomes[unique.unique_id] = RefinementOutcome( + unique_id=unique.unique_id, + status=classification.status if result.usable else result.status, + energy_eV=float(result.energy_eV), + fmax_eV_A=float(result.fmax_eV_A), + steps=int(result.steps), + atoms=result.atoms, + classification=classification, + converged=bool(result.converged), + ) + return outcomes + + +def _reference_for_record( + record: MinimumRecord, + references: Sequence[ClassificationReference], + replicas: Sequence[Any], +) -> ClassificationReference: + """Pick the reference of the replica the record came from.""" + if references and len(references) == len(replicas): + index = int(record.origin.replica) + if 0 <= index < len(references): + return references[index] + if references: + return references[0] + raise ValueError("No classification reference available for refinement") + + +def _write_artifacts(output_dir: Path, results: dict) -> None: + manifest: dict[str, dict] = {} + for path in sorted(output_dir.rglob("*")): + if not path.is_file() or path.name == "artifacts.json": + continue + relative = path.relative_to(output_dir).as_posix() + manifest[relative] = { + "bytes": path.stat().st_size, + "format": _format_for(path), + "units": "eV, Angstrom, fs, K", + } + payload = { + "schema": "mliport.confsearch-artifacts/1", + "status": results["status"], + "files": manifest, + } + (output_dir / "artifacts.json").write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + +def _format_for(path: Path) -> str: + return { + ".json": "json", + ".csv": "csv", + ".extxyz": "extxyz", + ".xyz": "xyz", + ".traj": "ase-traj", + ".inc": "cp2k-include", + ".log": "text", + }.get(path.suffix, "text") + + +__all__ = [ + "RESULTS_SCHEMA", + "RefinementOutcome", + "build_results", + "make_whole_adsorbate", +] diff --git a/tests/mliport/mliport/confsearch/calculators.py b/tests/mliport/mliport/confsearch/calculators.py new file mode 100644 index 00000000..e84aed0a --- /dev/null +++ b/tests/mliport/mliport/confsearch/calculators.py @@ -0,0 +1,242 @@ +"""Test-only synthetic calculators (spec section 13.1). + +No backend imports: every potential here is analytic, CPU-only and +deterministic, so the whole search engine is testable in CI. + +* :class:`EggCrateCalculator` -- a rigid periodic slab plus a 3-atom bonded + molecule whose anchor atom feels a piecewise-harmonic site term. The minima + sit exactly at known sites with exactly known energies. +* :class:`MorseBreakCalculator` -- the same slab/site term with a shallow Morse + bond so that hot MD breaks the molecule reproducibly. +* :class:`DirectForceCalculator` -- advertises ``direct_forces=True``. +* :class:`NonFiniteCalculator` -- returns NaN for the energy. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, ClassVar + +import numpy as np +from ase.calculators.calculator import Calculator, all_changes + +if TYPE_CHECKING: + from collections.abc import Sequence + +#: Anchor-atom index inside the adsorbate block. +ANCHOR_LOCAL_INDEX = 0 + + +def _mic(delta: np.ndarray, cell: np.ndarray) -> np.ndarray: + fractional = np.asarray(delta, dtype=float) @ np.linalg.inv(cell) + fractional -= np.round(fractional) + return fractional @ cell + + +class EggCrateCalculator(Calculator): + """Rigid slab + 3-atom molecule with a piecewise-harmonic site term. + + Sites (fractional, in units of the in-plane cell) and their exact minimum + energies (eV): + + ``hollow`` (1/2, 1/2) -> -0.60, ``bridge`` (1/2, 0) -> -0.35, + ``top`` (0, 0) -> -0.20. + + The internal terms vanish at their minimum geometry (bonds ``bond_A``, + angle ``angle_deg``), so a relaxed molecule sitting on a site has exactly + the site energy. + """ + + implemented_properties: ClassVar[list[str]] = ["energy", "forces"] + + def __init__( + self, + *, + sites: Sequence[tuple[float, float, float]] | None = None, + bond_A: float = 1.5, + angle_deg: float = 109.47, + bond_k: float = 20.0, + angle_k: float = 5.0, + site_k: float = 4.0, + n_slab: int = 4, + spacing_A: float = 3.0, + ) -> None: + super().__init__() + self.sites = list( + sites + if sites is not None + else [ + (0.5, 0.5, -0.60), # hollow + (0.5, 0.0, -0.35), # bridge + (0.0, 0.0, -0.20), # top + ] + ) + self.bond_A = float(bond_A) + self.angle = np.deg2rad(angle_deg) + self.bond_k = float(bond_k) + self.angle_k = float(angle_k) + self.site_k = float(site_k) + self.n_slab = int(n_slab) + self.spacing_A = float(spacing_A) + + # -- helpers --------------------------------------------------------- + @property + def cell(self) -> np.ndarray: + return np.diag( + [self.n_slab * self.spacing_A, self.n_slab * self.spacing_A, 20.0] + ) + + def _site_energy_forces(self, position: np.ndarray) -> tuple[float, np.ndarray]: + """Nearest-site piecewise harmonic term (exact minima at the sites).""" + cell = self.cell + in_plane = cell[:2, :2] + best: tuple[float, np.ndarray] | None = None + for u, v, depth in self.sites: + target = np.array([u * cell[0, 0], v * cell[1, 1], position[2]]) + delta = _mic(position - target, cell) + delta[2] = 0.0 + value = float(depth) + 0.5 * self.site_k * float(np.dot(delta, delta)) + if best is None or value < best[0]: + best = (value, delta) + assert best is not None + return best[0], -self.site_k * best[1] + + def _internal(self, positions: np.ndarray) -> tuple[float, np.ndarray]: + """Harmonic bonds 0-1, 1-2 and a harmonic angle 0-1-2.""" + energy = 0.0 + forces = np.zeros_like(positions) + for first, second in ((0, 1), (1, 2)): + delta = positions[second] - positions[first] + distance = float(np.linalg.norm(delta)) + stretch = distance - self.bond_A + energy += 0.5 * self.bond_k * stretch**2 + # Force ON the second atom; the first feels the opposite one. + force_on_second = -self.bond_k * stretch * delta / distance + forces[second] += force_on_second + forces[first] -= force_on_second + first, middle, last = positions[0], positions[1], positions[2] + left = first - middle + right = last - middle + cos_theta = float( + np.dot(left, right) / (np.linalg.norm(left) * np.linalg.norm(right)) + ) + cos_theta = float(np.clip(cos_theta, -1.0, 1.0)) + theta = float(np.arccos(cos_theta)) + energy += 0.5 * self.angle_k * (theta - self.angle) ** 2 + # d(theta)/dr for a three-body angle; standard chain-rule form. + sin_theta = float(np.sin(theta)) or 1e-12 + gradient_first = ( + cos_theta * left / np.dot(left, left) + - right / (np.linalg.norm(left) * np.linalg.norm(right)) + ) / sin_theta + gradient_last = ( + cos_theta * right / np.dot(right, right) + - left / (np.linalg.norm(left) * np.linalg.norm(right)) + ) / sin_theta + dtheta = self.angle_k * (theta - self.angle) + forces[0] -= dtheta * gradient_first + forces[2] -= dtheta * gradient_last + forces[1] -= -dtheta * (gradient_first + gradient_last) + return energy, forces + + def calculate( + self, + atoms=None, + properties=("energy", "forces"), + system_changes=all_changes, + ) -> None: + super().calculate(atoms, properties, system_changes) + positions = np.asarray(atoms.get_positions(), dtype=float) + n_slab = len(atoms) - 3 + energy = 0.0 + forces = np.zeros_like(positions) + molecule = positions[n_slab:] + site_energy, site_force = self._site_energy_forces(molecule[ANCHOR_LOCAL_INDEX]) + energy += site_energy + forces[n_slab + ANCHOR_LOCAL_INDEX] += site_force + internal_energy, internal_forces = self._internal(molecule) + energy += internal_energy + forces[n_slab:] += internal_forces + self.results = {"energy": float(energy), "forces": forces} + + +class MorseBreakCalculator(EggCrateCalculator): + """Egg-crate model whose first bond is a shallow Morse potential. + + ``D_e`` is small enough that a 1000 K Langevin segment breaks the bond + reproducibly with a fixed seed. + """ + + def __init__(self, *, D_e: float = 0.04, alpha: float = 1.0, **kwargs) -> None: + kwargs.setdefault("bond_k", 20.0) + super().__init__(**kwargs) + self.D_e = float(D_e) + self.alpha = float(alpha) + + def _internal(self, positions: np.ndarray) -> tuple[float, np.ndarray]: + energy, forces = super()._internal(positions) + # Replace the harmonic 0-1 bond by a Morse bond (subtract the harmonic + # term, add the Morse term). + delta = positions[1] - positions[0] + distance = float(np.linalg.norm(delta)) + stretch = distance - self.bond_A + # Remove the harmonic 0-1 term (energy and both forces). + energy -= 0.5 * self.bond_k * stretch**2 + harmonic_on_second = -self.bond_k * stretch * delta / distance + forces[1] -= harmonic_on_second + forces[0] += harmonic_on_second + # Add the Morse term. + exponential = np.exp(-self.alpha * stretch) + morse = self.D_e * (1.0 - exponential) ** 2 + d_morse = 2.0 * self.D_e * self.alpha * exponential * (1.0 - exponential) + energy += morse + morse_on_second = -d_morse * delta / distance + forces[1] += morse_on_second + forces[0] -= morse_on_second + return energy, forces + + +class DirectForceCalculator(EggCrateCalculator): + """Advertises ``direct_forces`` so the identity gate can be tested.""" + + def info(self) -> dict: + info = dict(super().info()) + info["direct_forces"] = True + return info + + +class NonFiniteCalculator(EggCrateCalculator): + """Well-behaved for ``fail_after`` evaluations, then NaN energy/forces. + + The start quench must succeed, so the NaN has to appear during the MD + segment; the counter makes that deterministic. + """ + + def __init__(self, *, fail_after: int = 8, **kwargs) -> None: + super().__init__(**kwargs) + self.fail_after = int(fail_after) + self.calls = 0 + + def calculate( + self, + atoms=None, + properties=("energy", "forces"), + system_changes=all_changes, + ) -> None: + self.calls += 1 + if self.calls <= self.fail_after: + super().calculate(atoms, properties, system_changes) + return + super().calculate(atoms, properties, system_changes) + self.results = { + "energy": float("nan"), + "forces": np.full_like(atoms.positions, float("nan")), + } + + +__all__ = [ + "ANCHOR_LOCAL_INDEX", + "DirectForceCalculator", + "EggCrateCalculator", + "MorseBreakCalculator", + "NonFiniteCalculator", +] diff --git a/tests/mliport/mliport/confsearch/test_engine.py b/tests/mliport/mliport/confsearch/test_engine.py new file mode 100644 index 00000000..836f630b --- /dev/null +++ b/tests/mliport/mliport/confsearch/test_engine.py @@ -0,0 +1,445 @@ +"""End-to-end search-engine tests on the synthetic egg-crate model (spec 13.2). + +The model has three known sites with exactly known energies: + +* hollow (1/2, 1/2) -> -0.60 eV +* bridge (1/2, 0) -> -0.35 eV +* top (0, 0) -> -0.20 eV + +The internal terms vanish at their minimum geometry, so a relaxed molecule on a +site has exactly the site energy. The slab is fixed, which enables the +geometric identity rule G of spec section 9.3 revision 4. +""" + +from __future__ import annotations + +import json +from dataclasses import replace + +import numpy as np +import pytest +from ase import Atoms +from ase.constraints import FixAtoms +from ase.io import read + +from mliport.confsearch.checkpoint import ConfsearchCheckpointStore +from mliport.confsearch.engine import PreparedStart, run_confsearch +from mliport.confsearch.options import ConfsearchOptions +from mliport.confsearch.regions import resolve_regions, validate_all +from tests.mliport.mliport.confsearch.calculators import ( + EggCrateCalculator, + MorseBreakCalculator, + NonFiniteCalculator, +) + +SLAB_ATOMS = 16 +MOLECULE_ATOMS = 3 +SITE_ENERGIES = {(0.5, 0.5): -0.60, (0.5, 0.0): -0.35, (0.0, 0.0): -0.20} + + +def _molecule(angle_deg: float = 109.47) -> Atoms: + angle = np.deg2rad(angle_deg) + return Atoms( + "CSO", + positions=[ + [0.0, 0.0, 0.0], + [1.5, 0.0, 0.0], + [1.5 - 1.5 * np.cos(angle), 1.5 * np.sin(angle), 0.0], + ], + pbc=False, + ) + + +def _slab() -> Atoms: + spacing = 3.0 + return Atoms( + "S" * SLAB_ATOMS, + positions=[ + [0.5 + spacing * i, 0.5 + spacing * j, 1.0] + for i in range(4) + for j in range(4) + ], + cell=[12.0, 12.0, 20.0], + pbc=True, + ) + + +def _start(site: tuple[float, float], *, fixed: bool = True) -> Atoms: + slab = _slab() + molecule = _molecule() + shift = np.array([site[0] * 12.0, site[1] * 12.0, 2.0]) + atoms = Atoms( + numbers=np.concatenate([slab.numbers, molecule.numbers]), + positions=np.concatenate( + [slab.get_positions(), molecule.get_positions() + shift] + ), + cell=slab.cell, + pbc=slab.pbc, + ) + if fixed: + atoms.set_constraint(FixAtoms(indices=list(range(SLAB_ATOMS)))) + return atoms + + +def _options(**overrides) -> ConfsearchOptions: + base = { + "cs_starts": ("starts.xyz",), + "cs_adsorbate_indices": "16-18", + "cs_cycles": 2, + "cs_hold_ps": 0.05, + "cs_cool_ps": 0.05, + "cs_snapshot_interval_ps": 0.02, + "timestep": 0.5, + "cs_temperature_high": 800.0, + "cs_temperature_low": 300.0, + "cs_quench_fmax": 0.01, + "cs_quench_steps": 2000, + "cs_refine": True, + "cs_refine_fmax": 0.005, + "cs_refine_steps": 2000, + } + base.update(overrides) + return ConfsearchOptions(**base) + + +def _prepared(sites, *, fixed: bool = True, options=None): + options = options or _options() + starts = [] + for index, site in enumerate(sites): + atoms = _start(site, fixed=fixed) + regions = resolve_regions( + atoms, options, adsorbate_indices=np.arange(SLAB_ATOMS, len(atoms)) + ) + surface = validate_all(atoms, regions, options) + starts.append( + PreparedStart( + index=index, + atoms=atoms, + regions=regions, + surface=surface, + label=f"start_{index:03d}", + ) + ) + return starts + + +def _run(tmp_path, sites, *, seed=7, calculator=None, options=None, name="run"): + options = options or _options() + starts = _prepared(sites, options=options) + output = tmp_path / name + results = run_confsearch( + calculator or EggCrateCalculator(), + options, + starts=starts, + output_dir=output, + seed=seed, + model_record={"model_type": "synthetic", "task": "bulk"}, + gradient_preflight={"status": "passed"}, + ) + return output, results + + +# --------------------------------------------------------------------------- +# end to end +# --------------------------------------------------------------------------- + + +def test_search_finds_every_known_minimum_with_its_exact_energy(tmp_path): + sites = list(SITE_ENERGIES) + output, results = _run(tmp_path, sites) + assert results["status"] == "completed" + entries = results["ranking"]["refined"] + results["ranking"]["unrefined"] + energies = sorted(entry["energy_eV"] for entry in entries) + for expected in sorted(SITE_ENERGIES.values()): + assert any(abs(energy - expected) < 1e-6 for energy in energies), ( + expected, + energies, + ) + # Every known minimum was reached, and the baseline (start quench only) + # already reports the deepest one. + assert results["baseline"]["best_energy_eV"] == pytest.approx(-0.60, abs=1e-6) + assert results["diagnostics"]["n_quenches"] >= len(sites) + assert (output / "search_results.json").is_file() + payload = json.loads((output / "search_results.json").read_text()) + assert payload["schema"] == "mliport.confsearch-results/1" + + +def test_same_basin_quenches_merge_through_the_geometry_rule(tmp_path): + # Three replicas start at the same site: every quench must merge into one + # unique minimum with hit_count >= 3 (G is enabled by the fixed slab). + sites = [(0.5, 0.5)] * 3 + _output, results = _run(tmp_path, sites) + entries = results["ranking"]["refined"] + results["ranking"]["unrefined"] + deepest = [entry for entry in entries if abs(entry["energy_eV"] + 0.60) < 1e-6] + assert deepest + assert max(entry["hit_count"] for entry in deepest) >= 3 + assert max(entry["n_replicas"] for entry in deepest) >= 2 + + +def test_fixed_atoms_never_move_and_slab_is_reported(tmp_path): + output, results = _run(tmp_path, [(0.5, 0.5)], name="fixed") + assert results["starts"][0]["regions"]["fixed"] == list(range(SLAB_ATOMS)) + registry = results["diagnostics"]["registry"] + representative = registry["classes"]["valid"][0]["representative"] + reference = read(output / "top" / "rank_001" / "structure.extxyz") + slab_positions = np.asarray(reference.get_positions())[:SLAB_ATOMS] + assert np.allclose(slab_positions[:, 2], 1.0) + assert len(representative["positions_A"]) == SLAB_ATOMS + MOLECULE_ATOMS + + +def test_exports_re_read_equal_to_the_minimum_up_to_lattice_translations(tmp_path): + output, results = _run(tmp_path, [(0.5, 0.5)], name="exports") + assert results["exports"] + export_dir = output / "top" / "rank_001" + for name in ( + "POSCAR", + "structure.extxyz", + "cp2k_coord.xyz", + "cp2k_cell.inc", + "cp2k_fixed_atoms.inc", + "minimum.json", + ): + assert (export_dir / name).is_file(), name + + payload = json.loads((export_dir / "minimum.json").read_text()) + representative = payload["representative"] + reference = Atoms( + numbers=np.asarray(representative["numbers"], dtype=int), + positions=np.asarray(representative["positions_A"], dtype=float), + cell=np.asarray(representative["cell_A"], dtype=float), + pbc=np.asarray(representative["pbc"], dtype=bool), + ) + exported = read(export_dir / "structure.extxyz") + cell = np.asarray(exported.cell, dtype=float) + fractional = ( + np.asarray(exported.get_positions(), dtype=float) + - np.asarray(reference.get_positions(), dtype=float) + ) @ np.linalg.inv(cell) + residual = fractional - np.round(fractional) + assert np.all(np.abs(residual) < 1e-8), np.abs(residual).max() + assert exported.get_pbc().all() + + fixed_inc = (export_dir / "cp2k_fixed_atoms.inc").read_text() + assert fixed_inc.startswith("# CP2K") + assert "LIST 1..16" in fixed_inc + cell_inc = (export_dir / "cp2k_cell.inc").read_text().splitlines() + assert len(cell_inc) == 3 + assert all(line.startswith(("A ", "B ", "C ")) for line in cell_inc) + + poscar = (export_dir / "POSCAR").read_text() + assert "Selective dynamics" in poscar + flag_lines = [ + line.split()[-3:] for line in poscar.splitlines() if len(line.split()) >= 6 + ] + assert sum(1 for flags in flag_lines if flags == ["F", "F", "F"]) >= SLAB_ATOMS + + +def test_search_is_bitwise_reproducible_for_the_same_seed(tmp_path): + sites = list(SITE_ENERGIES) + first_dir, first = _run(tmp_path, sites, seed=11, name="first") + second_dir, second = _run(tmp_path, sites, seed=11, name="second") + assert first["ranking"] == second["ranking"] + assert first["diagnostics"]["n_quenches"] == second["diagnostics"]["n_quenches"] + raw_first = sorted( + (path.relative_to(first_dir).as_posix(), path.read_bytes()) + for path in first_dir.rglob("*") + if path.is_file() and "checkpoints" not in path.parts + ) + raw_second = sorted( + (path.relative_to(second_dir).as_posix(), path.read_bytes()) + for path in second_dir.rglob("*") + if path.is_file() and "checkpoints" not in path.parts + ) + assert [name for name, _ in raw_first] == [name for name, _ in raw_second] + + +def test_temperature_schedule_is_linear_and_recorded(tmp_path): + options = _options(cs_ramp_interval=10) + output, _results = _run(tmp_path, [(0.5, 0.5)], options=options, name="ramp") + cycle = json.loads( + (output / "raw" / "replica_000" / "cycle_001" / "cycle.json").read_text() + ) + schedule = cycle["temperature_schedule"] + assert schedule + assert schedule[0]["target_temperature_K"] == pytest.approx(800.0) + n_cool = options.n_cool + for entry in schedule: + fraction = (entry["step"] - options.n_hold) / n_cool + expected = 800.0 + (300.0 - 800.0) * fraction + assert entry["target_temperature_K"] == pytest.approx(expected) + assert cycle["status"] == "completed" + assert cycle["snapshot_steps"] == [40, 80] + + +def test_raw_md_output_uses_the_shared_csv_header(tmp_path): + output, _results = _run(tmp_path, [(0.5, 0.5)], name="raw") + csv_path = output / "raw" / "replica_000" / "cycle_001" / "md.csv" + header = csv_path.read_text().splitlines()[0] + from mliport.runners.md_output import MD_CSV_HEADER + + assert header == ",".join(MD_CSV_HEADER) + assert (output / "raw" / "replica_000" / "cycle_001" / "trajectory.traj").is_file() + + +def test_force_abort_is_recorded_and_the_search_continues(tmp_path): + options = _options(cs_temperature_high=1200.0, fmax_abort=2.0) + output, results = _run(tmp_path, [(0.5, 0.5)], options=options, name="abort") + cycle = json.loads( + (output / "raw" / "replica_000" / "cycle_001" / "cycle.json").read_text() + ) + assert cycle["status"] == "md_force_abort" + assert "fmax_abort" in cycle["message"] + assert results["status"] == "completed" + + +def test_non_finite_energy_is_recorded_as_an_abort(tmp_path): + options = _options(cs_temperature_high=1200.0) + output, _results = _run( + tmp_path, + [(0.5, 0.5)], + calculator=NonFiniteCalculator(fail_after=6), + options=options, + name="nonfinite", + ) + cycle = json.loads( + (output / "raw" / "replica_000" / "cycle_001" / "cycle.json").read_text() + ) + assert cycle["status"] == "md_nonfinite_abort" + + +def test_morse_model_reports_an_integrity_abort_or_reacted_minimum(tmp_path): + options = _options( + cs_temperature_high=1500.0, + cs_temperature_low=300.0, + cs_hold_ps=0.1, + cs_cool_ps=0.05, + cs_snapshot_interval_ps=0.02, + ) + output, results = _run( + tmp_path, + [(0.5, 0.5)], + calculator=MorseBreakCalculator(D_e=0.03), + options=options, + name="morse", + ) + statuses = [ + json.loads( + ( + output / "raw" / "replica_000" / f"cycle_{cycle:03d}" / "cycle.json" + ).read_text() + )["status"] + for cycle in (1, 2) + ] + reacted = results["diagnostics"]["registry"]["classes"].get("reacted", []) + assert "md_integrity_abort" in statuses or reacted + + +# --------------------------------------------------------------------------- +# checkpoint / resume +# --------------------------------------------------------------------------- + + +def test_resume_reproduces_the_uninterrupted_search(tmp_path, monkeypatch): + from mliport.confsearch.engine import SearchEngine + + sites = list(SITE_ENERGIES) + reference_dir, reference = _run(tmp_path, sites, seed=23, name="reference") + + # Simulate a crash right after the second checkpointed unit. + options = _options() + starts = _prepared(sites, options=options) + interrupted_dir = tmp_path / "interrupted" + original = SearchEngine._write_checkpoint + counter = {"n": 0} + + def crash_after_two(self): + original(self) + counter["n"] += 1 + if counter["n"] >= 2: + raise KeyboardInterrupt("simulated crash") + + monkeypatch.setattr(SearchEngine, "_write_checkpoint", crash_after_two) + with pytest.raises(KeyboardInterrupt): + run_confsearch( + EggCrateCalculator(), + options, + starts=starts, + output_dir=interrupted_dir, + seed=23, + model_record={"model_type": "synthetic", "task": "bulk"}, + gradient_preflight={"status": "passed"}, + ) + monkeypatch.undo() + + store = ConfsearchCheckpointStore(interrupted_dir) + data, fingerprint = store.load() + assert 0 < data.unit_index < 1 + len(sites) * options.cs_cycles + + resumed = run_confsearch( + EggCrateCalculator(), + options, + starts=starts, + output_dir=interrupted_dir, + seed=23, + model_record={"model_type": "synthetic", "task": "bulk"}, + gradient_preflight={"status": "passed"}, + resume=data, + resume_fingerprint=fingerprint, + ) + assert resumed["ranking"] == reference["ranking"] + assert ( + resumed["diagnostics"]["n_quenches"] == reference["diagnostics"]["n_quenches"] + ) + + +def test_resume_fingerprint_refuses_a_changed_option(tmp_path): + from mliport.confsearch.checkpoint import ( + build_resume_fingerprint, + compare_fingerprints, + ) + + options = _options() + first, _sha = build_resume_fingerprint( + options=options, + model={"model_type": "synthetic"}, + input_hashes={"starts": "abc"}, + regions={"adsorbate": [16, 17, 18]}, + seed=1, + mliport_version="2.0.0b4", + ) + second, _sha = build_resume_fingerprint( + options=replace(options, cs_temperature_high=900.0), + model={"model_type": "synthetic"}, + input_hashes={"starts": "abc"}, + regions={"adsorbate": [16, 17, 18]}, + seed=1, + mliport_version="2.0.0b4", + ) + differences = compare_fingerprints(first, second) + assert differences + assert any("cs_temperature_high" in item for item in differences) + + +def test_checkpoint_round_trip_restores_the_registry(tmp_path): + output, results = _run(tmp_path, [(0.5, 0.5)], name="roundtrip") + store = ConfsearchCheckpointStore(output) + data, _fingerprint = store.load() + assert data.registry["classes"] + assert data.rng_states["replicas"] + assert data.unit_index == 1 + 1 * 2 # start stage + two cycles + assert data.registry["schema"] == "mliport.confsearch-registry/1" + assert ( + results["diagnostics"]["registry"]["schema"] == "mliport.confsearch-registry/1" + ) + + +def test_interrupted_raw_directories_are_archived_not_deleted(tmp_path): + store = ConfsearchCheckpointStore(tmp_path / "run") + source = tmp_path / "run" / "raw" / "replica_000" / "cycle_001" + source.mkdir(parents=True) + (source / "md.csv").write_text("partial\n") + archived = store.archive_incomplete(replica=0, cycle=1) + assert archived is not None + assert archived.name == "incomplete_attempt_1" + assert (archived / "md.csv").read_text() == "partial\n" + assert not source.exists() diff --git a/tests/mliport/mliport/confsearch/test_refinement.py b/tests/mliport/mliport/confsearch/test_refinement.py new file mode 100644 index 00000000..b034c6dd --- /dev/null +++ b/tests/mliport/mliport/confsearch/test_refinement.py @@ -0,0 +1,305 @@ +"""Refinement, ranking and diagnostics contracts (spec 10.1/10.3/10.4, PR #64 review). + +The refinement engine is stubbed so the ranking rules can be tested directly: + +* a refinement that lowers a candidate's order must move it to the top and into + ``top/rank_001``; +* two refined minima that deduplicate into one entry must merge (summed hit + counts), not be exported twice; +* an unconverged refinement keeps the quench result and is flagged; +* a refinement that changes to a non-rankable status leaves the ranking with a + recorded reason; +* ``minima/all_minima.extxyz`` keeps every quench result of every status; +* the baseline and saturation diagnostics count *members*, not representatives. +""" + +from __future__ import annotations + +import json +from types import SimpleNamespace + +import numpy as np +import pytest +from ase.io import read + +from mliport.confsearch import results as results_module +from mliport.confsearch.engine import MinimumOrigin, MinimumRecord, MinimumRegistry +from mliport.confsearch.regions import resolve_regions +from tests.mliport.mliport.confsearch.structures import fake_classification +from tests.mliport.mliport.confsearch.test_engine import SLAB_ATOMS, _options, _start + + +def _setup(): + options = _options() + atoms = _start((0.5, 0.5)) + regions = resolve_regions( + atoms, options, adsorbate_indices=np.arange(SLAB_ATOMS, len(atoms)) + ) + registry = MinimumRegistry( + options=options, adsorbate=regions.adsorbate, fixed=regions.fixed + ) + return options, regions, registry, atoms + + +def _register( + registry, atoms, *, energy, replica, source="end", status="valid", cycle=1, quench=1 +): + record = MinimumRecord( + record_id=registry.new_record_id(), + energy_eV=float(energy), + atoms=atoms, + classification=fake_classification(status=status), + origin=MinimumOrigin(replica=replica, cycle=cycle, source=source), + quench_index=int(quench), + cumulative_md_ps=float(quench), + fmax_eV_A=0.04, + steps=10, + ) + return registry.register(record) + + +def _outcome( + unique_id, *, energy, status="valid", converged=True, atoms=None, fmax=0.02 +): + return results_module.RefinementOutcome( + unique_id=int(unique_id), + status=status, + energy_eV=float(energy), + fmax_eV_A=float(fmax), + steps=5, + atoms=atoms, + classification=fake_classification(status=status), + converged=bool(converged), + ) + + +def _build(options, regions, registry, refinements, atoms, *, tmp_path): + monkeypatch_target = results_module._refine_top_k + results_module._refine_top_k = lambda *a, **k: refinements + try: + return results_module.build_results( + registry=registry, + options=options, + replicas=[SimpleNamespace(status="active")], + discovery=[], + baseline={"stage": "start_quench", "best_energy_eV": -0.60}, + warnings=[], + output_dir=tmp_path, + model_record={"model_type": "synthetic"}, + gradient_preflight={"status": "passed"}, + cumulative_md_ps=1.0, + seed=7, + starts=[], + regions=regions, + references=[None], + calculator=object(), + ) + finally: + results_module._refine_top_k = monkeypatch_target + + +def test_refinement_reorders_the_ranking_and_the_exports(tmp_path): + """A candidate whose refined energy is lower must become rank_001.""" + options, regions, registry, atoms = _setup() + high = _register(registry, atoms, energy=-0.50, replica=0) + low = _register(registry, atoms, energy=-0.45, replica=1, quench=2) + refined_high = atoms.copy() + refined_high.positions[SLAB_ATOMS] += 0.02 + refined_low = atoms.copy() + refined_low.positions[SLAB_ATOMS] += 0.05 + refinements = { + high.unique_id: _outcome(high.unique_id, energy=-0.20, atoms=refined_high), + low.unique_id: _outcome(low.unique_id, energy=-0.90, atoms=refined_low), + } + results = _build(options, regions, registry, refinements, atoms, tmp_path=tmp_path) + + refined = results["ranking"]["refined"] + assert [entry["id"] for entry in refined] == [low.unique_id, high.unique_id] + assert refined[0]["delta_energy_eV"] == pytest.approx(0.0) + assert results["ranking"]["best_energy_source"] == "refined" + assert results["ranking"]["unrefined"] == [] + exported = read(tmp_path / "top" / "rank_001" / "structure.extxyz") + # rank_001 is the refined geometry of the *lowest refined* candidate. + assert exported.info["confsearch_id"] == low.unique_id + + +def test_refined_duplicates_merge_into_one_entry(tmp_path): + """Spec 10.3: re-deduplicate the refined set, keeping the hit counts.""" + options, regions, registry, atoms = _setup() + first = _register(registry, atoms, energy=-0.50, replica=0) + _register(registry, atoms, energy=-0.50, replica=1, quench=2) # same basin + assert first.hit_count == 2 + second = _register(registry, atoms, energy=-0.30, replica=2, quench=3) + refinements = { + first.unique_id: _outcome(first.unique_id, energy=-0.80, atoms=atoms), + second.unique_id: _outcome(second.unique_id, energy=-0.80, atoms=atoms.copy()), + } + results = _build(options, regions, registry, refinements, atoms, tmp_path=tmp_path) + + refined = results["ranking"]["refined"] + assert len(refined) == 1 + assert refined[0]["hit_count"] == 3 # 2 + 1, summed over the merged sources + assert sorted(refined[0]["merged_ids"]) == sorted( + [first.unique_id, second.unique_id] + ) + assert results["refinement"]["n_merged_into_existing"] == 1 + assert len(results["exports"]) == 1 + + +def test_unconverged_refinement_keeps_the_quench_result_and_flags_it(tmp_path): + options, regions, registry, atoms = _setup() + unique = _register(registry, atoms, energy=-0.50, replica=0) + refinements = { + unique.unique_id: _outcome( + unique.unique_id, + energy=-0.40, + status="quench_not_converged", + converged=False, + ) + } + results = _build(options, regions, registry, refinements, atoms, tmp_path=tmp_path) + + assert results["ranking"]["refined"] == [] + unrefined = results["ranking"]["unrefined"] + assert len(unrefined) == 1 + assert unrefined[0]["energy_eV"] == pytest.approx(-0.50) # quench energy kept + assert unrefined[0]["refinement_failed"] is True + notes = results["refinement"]["notes"] + assert notes and notes[0]["reason"] == "refinement_not_converged" + assert notes[0]["kept"] == "quench_result" + + +def test_unconverged_refinement_candidate_is_still_exported(tmp_path): + """Spec 10.3 revision 6: a failed refinement must not delete a candidate. + + The candidate keeps its quench geometry/energy, is exported as + `top/unrefined_NNN/` next to the refined ranks, and a warning names it. + """ + options, regions, registry, atoms = _setup() + failing = _register(registry, atoms, energy=-0.60, replica=0) + good = _register(registry, atoms, energy=-0.50, replica=1, quench=2) + refinements = { + failing.unique_id: _outcome( + failing.unique_id, + energy=float("nan"), + status="quench_failed", + converged=False, + ), + good.unique_id: _outcome(good.unique_id, energy=-0.70, atoms=atoms.copy()), + } + results = _build(options, regions, registry, refinements, atoms, tmp_path=tmp_path) + + assert [entry["id"] for entry in results["ranking"]["refined"]] == [good.unique_id] + exported = {item["path"]: item for item in results["exports"]} + assert "rank_001" in exported and exported["rank_001"]["refined"] is True + assert "unrefined_001" in exported + assert exported["unrefined_001"]["refined"] is False + assert exported["unrefined_001"]["refinement_failed"] is True + # The unrefined export keeps the *quench* geometry and energy. + assert exported["unrefined_001"]["energy_eV"] == pytest.approx(-0.60) + unrefined = (tmp_path / "top" / "unrefined_001").resolve() + payload = json.loads((unrefined / "minimum.json").read_text()) + assert payload["export"]["ranking"]["energy_eV"] == pytest.approx(-0.60) + assert (unrefined / "POSCAR").is_file() + assert any("did not converge" in warning for warning in results["warnings"]) + + +def test_refined_entry_flags_come_from_the_refined_classification(tmp_path): + """Spec 10.3 revision 6: flags follow the refined classification.""" + options, regions, registry, atoms = _setup() + unique = _register(registry, atoms, energy=-0.50, replica=0) + outcome = _outcome(unique.unique_id, energy=-0.60, atoms=atoms.copy()) + # The refined classification says physisorbed + far-field slab change. + outcome = results_module.RefinementOutcome( + unique_id=outcome.unique_id, + status=outcome.status, + energy_eV=outcome.energy_eV, + fmax_eV_A=outcome.fmax_eV_A, + steps=outcome.steps, + atoms=outcome.atoms, + classification=fake_classification( + status="valid", flags=("far_field_slab_changed",) + ), + converged=True, + ) + results = _build( + options, + regions, + registry, + {unique.unique_id: outcome}, + atoms, + tmp_path=tmp_path, + ) + entry = results["ranking"]["refined"][0] + # The quench representative was flagged "physisorbed"; the refined entry + # must carry the refined classification's flags instead. + assert entry["flags"] == ["far_field_slab_changed"] + + +def test_refinement_status_change_removes_the_entry_with_a_reason(tmp_path): + options, regions, registry, atoms = _setup() + unique = _register(registry, atoms, energy=-0.50, replica=0) + refinements = { + unique.unique_id: _outcome(unique.unique_id, energy=-0.40, status="desorbed") + } + results = _build(options, regions, registry, refinements, atoms, tmp_path=tmp_path) + + assert results["ranking"]["refined"] == [] + assert results["ranking"]["unrefined"] == [] + notes = results["refinement"]["notes"] + assert notes and notes[0]["reason"] == "refinement_changed_status" + assert notes[0]["refinement_status"] == "desorbed" + + +def test_minima_directory_keeps_every_quench_result(tmp_path): + options, regions, registry, atoms = _setup() + _register(registry, atoms, energy=-0.50, replica=0, source="start", quench=1) + _register(registry, atoms, energy=-0.40, replica=0, quench=2) + _register(registry, atoms, energy=-0.20, replica=1, status="reacted", quench=3) + _register( + registry, + atoms, + energy=float("nan"), + replica=1, + status="quench_failed", + quench=4, + ) + + results = _build(options, regions, registry, {}, atoms, tmp_path=tmp_path) + + frames = read(tmp_path / "minima" / "all_minima.extxyz", index=":") + assert len(frames) == 4 + statuses = {frame.info["confsearch_status"] for frame in frames} + assert {"valid", "reacted", "quench_failed"} <= statuses + # A failed quench has a non-finite energy: the frame is kept without it. + failed = [f for f in frames if f.info["confsearch_status"] == "quench_failed"] + assert "energy_eV" not in failed[0].info + payload = json.loads((tmp_path / "minima" / "registry.json").read_text()) + assert payload["schema"] == "mliport.confsearch-registry/1" + assert results["diagnostics"]["registry"] == payload + + +def test_baseline_and_saturation_use_members_and_first_found_index(tmp_path): + options, regions, registry, atoms = _setup() + # One unique reached by a start quench (member origin) and one only by MD. + reached = _register( + registry, atoms, energy=-0.60, replica=0, source="start", quench=1 + ) + _register(registry, atoms, energy=-0.60, replica=1, quench=4) # same basin, MD + never = _register(registry, atoms, energy=-0.30, replica=0, quench=2) + assert reached.hit_count == 2 + + results = _build(options, regions, registry, {}, atoms, tmp_path=tmp_path) + + baseline = results["baseline"] + assert baseline["top_k_never_reached_by_start_quench"] == [never.unique_id] + best = results["ranking"]["unrefined"][0] + # first_found_fraction must use the *first-found* quench index (1), not the + # representative's own quench index. + assert best["first_found_quench_index"] == 1 + assert best["first_found_fraction"] == pytest.approx(1 / registry.total_records()) + saturation = results["diagnostics"]["saturation"] + assert saturation["best_hit_count"] == 2 + assert saturation["best_first_found_fraction"] == pytest.approx( + 1 / registry.total_records() + )