Update finalize to meet standard

docs/function_guides/ask_tell_generator.rst

@@ -8,7 +8,7 @@ These generators, implementations, methods, and subclasses are in BETA, and
 may change in future releases.
 
 The Generator interface is expected to roughly correspond with CAMPA's standard:
-https://github.com/campa-consortium/generator_standard
+https://github.com/campa-consortium/gest-api
 
 libEnsemble is in the process of supporting generator objects that implement the following interface:
 

libensemble/gen_classes/aposmm.py

@@ -2,7 +2,7 @@
 from typing import List
 
 import numpy as np
-from generator_standard.vocs import VOCS
+from gest_api.vocs import VOCS
 from numpy import typing as npt
 
 from libensemble.generators import PersistentGenInterfacer
@@ -12,6 +12,21 @@
 class APOSMM(PersistentGenInterfacer):
     """
     Standalone object-oriented APOSMM generator
+
+    VOCS variables must include both regular and *_on_cube versions. E.g.,:
+    vars_std = {
+        "var1": [-10.0, 10.0],
+        "var2": [0.0, 100.0],
+        "var3": [1.0, 50.0],
+        "var1_on_cube": [0, 1.0],
+        "var2_on_cube": [0, 1.0],
+        "var3_on_cube": [0, 1.0]
+    }
+    variables_mapping = {
+        "x": ["var1", "var2", "var3"],
+        "x_on_cube": ["var1_on_cube", "var2_on_cube", "var3_on_cube"],
+    }
+    gen = APOSMM(vocs, variables_mapping=variables_mapping, ...)
     """
 
     def __init__(
@@ -26,25 +41,36 @@ def __init__(
         from libensemble.gen_funcs.persistent_aposmm import aposmm
 
         self.VOCS = vocs
-
         gen_specs["gen_f"] = aposmm
-        self.n = len(list(self.VOCS.variables.keys()))
-
         gen_specs["user"] = {}
-        gen_specs["user"]["lb"] = np.array([vocs.variables[i].domain[0] for i in vocs.variables])
-        gen_specs["user"]["ub"] = np.array([vocs.variables[i].domain[1] for i in vocs.variables])
+        super().__init__(vocs, History, persis_info, gen_specs, libE_info, **kwargs)
+
+        # Set bounds using the correct x mapping
+        x_mapping = self.variables_mapping["x"]
+        self.gen_specs["user"]["lb"] = np.array([vocs.variables[var].domain[0] for var in x_mapping])
+        self.gen_specs["user"]["ub"] = np.array([vocs.variables[var].domain[1] for var in x_mapping])
+
+        if not gen_specs.get("out"):
+            x_size = len(self.variables_mapping.get("x", []))
+            x_on_cube_size = len(self.variables_mapping.get("x_on_cube", []))
+            assert x_size > 0 and x_on_cube_size > 0, "Both x and x_on_cube must be specified in variables_mapping"
+            assert (
+                x_size == x_on_cube_size
+            ), f"x and x_on_cube must have same length but got {x_size} and {x_on_cube_size}"
 
-        if not gen_specs.get("out"):  # gen_specs never especially changes for aposmm even as the problem varies
             gen_specs["out"] = [
-                ("x", float, self.n),
-                ("x_on_cube", float, self.n),
+                ("x", float, x_size),
+                ("x_on_cube", float, x_on_cube_size),
                 ("sim_id", int),
                 ("local_min", bool),
                 ("local_pt", bool),
             ]
-            gen_specs["persis_in"] = ["x", "f", "local_pt", "sim_id", "sim_ended", "x_on_cube", "local_min"]
-        super().__init__(vocs, History, persis_info, gen_specs, libE_info, **kwargs)
 
+            gen_specs["persis_in"] = ["sim_id", "x", "x_on_cube", "f", "sim_ended"]
+            if "components" in kwargs or "components" in gen_specs.get("user", {}):
+                gen_specs["persis_in"].append("fvec")
+
+        # SH - Need to know if this is gen_on_manager or not.
         if not self.persis_info.get("nworkers"):
             self.persis_info["nworkers"] = kwargs.get("nworkers", gen_specs["user"].get("max_active_runs", 4))
         self.all_local_minima = []

libensemble/gen_classes/gpCAM.py

@@ -4,7 +4,7 @@
 from typing import List
 
 import numpy as np
-from generator_standard.vocs import VOCS
+from gest_api.vocs import VOCS
 from gpcam import GPOptimizer as GP
 from numpy import typing as npt
 
@@ -55,9 +55,9 @@ def __init__(self, VOCS: VOCS, ask_max_iter: int = 10, random_seed: int = 1, *ar
         self.noise = 1e-8  # 1e-12
         self.ask_max_iter = ask_max_iter
 
-    def _validate_vocs(self, VOCS):
-        assert len(self.VOCS.variables), "VOCS must contain variables."
-        assert len(self.VOCS.objectives), "VOCS must contain at least one objective."
+    def _validate_vocs(self, vocs):
+        assert len(vocs.variables), "VOCS must contain variables."
+        assert len(vocs.objectives), "VOCS must contain at least one objective."
 
     def suggest_numpy(self, n_trials: int) -> npt.NDArray:
         if self.all_x.shape[0] == 0:

libensemble/gen_classes/sampling.py

@@ -1,7 +1,7 @@
 """Generator classes providing points using sampling"""
 
 import numpy as np
-from generator_standard.vocs import VOCS
+from gest_api.vocs import VOCS
 
 from libensemble.generators import LibensembleGenerator
 

libensemble/gen_funcs/aposmm_localopt_support.py

@@ -17,6 +17,7 @@
 
 import numpy as np
 import psutil
+import traceback
 
 import libensemble.gen_funcs
 from libensemble.message_numbers import EVAL_GEN_TAG, STOP_TAG  # Only used to simulate receiving from manager
@@ -585,8 +586,8 @@ def run_local_tao(user_specs, comm_queue, x0, f0, child_can_read, parent_can_rea
 def opt_runner(run_local_opt, user_specs, comm_queue, x0, f0, child_can_read, parent_can_read):
     try:
         run_local_opt(user_specs, comm_queue, x0, f0, child_can_read, parent_can_read)
-    except Exception as e:
-        comm_queue.put(ErrorMsg(e))
+    except Exception:
+        comm_queue.put(ErrorMsg(traceback.format_exc()))
         parent_can_read.set()
 
 

libensemble/generators.py

@@ -2,15 +2,15 @@
 from typing import List, Optional
 
 import numpy as np
-from generator_standard import Generator
-from generator_standard.vocs import VOCS
+from gest_api import Generator
+from gest_api.vocs import VOCS
 from numpy import typing as npt
 
 from libensemble.comms.comms import QCommProcess  # , QCommThread
 from libensemble.executors import Executor
 from libensemble.message_numbers import EVAL_GEN_TAG, PERSIS_STOP
 from libensemble.tools.tools import add_unique_random_streams
-from libensemble.utils.misc import list_dicts_to_np, np_to_list_dicts
+from libensemble.utils.misc import list_dicts_to_np, np_to_list_dicts, unmap_numpy_array
 
 
 class GeneratorNotStartedException(Exception):
@@ -55,12 +55,18 @@ def __init__(
 
         self.variables_mapping = variables_mapping
         if not self.variables_mapping:
+            self.variables_mapping = {}
+        # Map variables to x if not already mapped
+        if "x" not in self.variables_mapping:
+            # SH TODO - is this check needed?
             if len(list(self.VOCS.variables.keys())) > 1 or list(self.VOCS.variables.keys())[0] != "x":
-                self.variables_mapping["x"] = list(self.VOCS.variables.keys())
+                self.variables_mapping["x"] = self._get_unmapped_keys(self.VOCS.variables, "x")
+        # Map objectives to f if not already mapped
+        if "f" not in self.variables_mapping:
             if (
                 len(list(self.VOCS.objectives.keys())) > 1 or list(self.VOCS.objectives.keys())[0] != "f"
             ):  # e.g. {"f": ["f"]} doesn't need mapping
-                self.variables_mapping["f"] = list(self.VOCS.objectives.keys())
+                self.variables_mapping["f"] = self._get_unmapped_keys(self.VOCS.objectives, "f")
 
         if len(kwargs) > 0:  # so user can specify gen-specific parameters as kwargs to constructor
             if not self.gen_specs.get("user"):
@@ -74,6 +80,15 @@ def __init__(
     def _validate_vocs(self, vocs) -> None:
         pass
 
+    def _get_unmapped_keys(self, vocs_dict, default_key):
+        """Get keys from vocs_dict that aren't already mapped to other keys in variables_mapping."""
+        # Get all variables that aren't already mapped to other keys
+        mapped_vars = []
+        for mapped_list in self.variables_mapping.values():
+            mapped_vars.extend(mapped_list)
+        unmapped_vars = [v for v in list(vocs_dict.keys()) if v not in mapped_vars]
+        return unmapped_vars
+
     @abstractmethod
     def suggest_numpy(self, num_points: Optional[int] = 0) -> npt.NDArray:
         """Request the next set of points to evaluate, as a NumPy array."""
@@ -119,6 +134,7 @@ def __init__(
         self.History = History
         self.libE_info = libE_info
         self.running_gen_f = None
+        self.gen_result = None
 
     def setup(self) -> None:
         """Must be called once before calling suggest/ingest. Initializes the background thread."""
@@ -139,16 +155,24 @@ def setup(self) -> None:
             user_function=True,
         )
 
-        # this is okay since the object isnt started until the first suggest
+        # This can be set here since the object isnt started until the first suggest
         self.libE_info["comm"] = self.running_gen_f.comm
 
-    def _set_sim_ended(self, results: npt.NDArray) -> npt.NDArray:
-        new_results = np.zeros(len(results), dtype=self.gen_specs["out"] + [("sim_ended", bool), ("f", float)])
-        for field in results.dtype.names:
+    def _prep_fields(self, results: npt.NDArray) -> npt.NDArray:
+        """Filter out fields that are not in persis_in and add sim_ended to the dtype"""
+        filtered_dtype = [
+            (name, results.dtype[name]) for name in results.dtype.names if name in self.gen_specs["persis_in"]
+        ]
+
+        new_dtype = filtered_dtype + [("sim_ended", bool)]
+        new_results = np.zeros(len(results), dtype=new_dtype)
+
+        for field in new_results.dtype.names:
             try:
                 new_results[field] = results[field]
-            except ValueError:  # lets not slot in data that the gen doesnt need?
+            except ValueError:
                 continue
+
         new_results["sim_ended"] = True
         return new_results
 
@@ -167,7 +191,7 @@ def suggest_numpy(self, num_points: int = 0) -> npt.NDArray:
     def ingest_numpy(self, results: npt.NDArray, tag: int = EVAL_GEN_TAG) -> None:
         """Send the results of evaluations to the generator, as a NumPy array."""
         if results is not None:
-            results = self._set_sim_ended(results)
+            results = self._prep_fields(results)
             Work = {"libE_info": {"H_rows": np.copy(results["sim_id"]), "persistent": True, "executor": None}}
             self.running_gen_f.send(tag, Work)
             self.running_gen_f.send(
@@ -176,7 +200,40 @@ def ingest_numpy(self, results: npt.NDArray, tag: int = EVAL_GEN_TAG) -> None:
         else:
             self.running_gen_f.send(tag, None)
 
-    def finalize(self, results: npt.NDArray = None) -> (npt.NDArray, dict, int):
-        """Send any last results to the generator, and it to close down."""
-        self.ingest_numpy(results, PERSIS_STOP)  # conversion happens in ingest
-        return self.running_gen_f.result()
+    def finalize(self) -> None:
+        """Stop the generator process and store the returned data."""
+        self.ingest_numpy(None, PERSIS_STOP)  # conversion happens in ingest
+        self.gen_result = self.running_gen_f.result()
+
+    def export(
+        self, user_fields: bool = False, as_dicts: bool = False
+    ) -> tuple[npt.NDArray | list | None, dict | None, int | None]:
+        """Return the generator's results
+        Parameters
+        ----------
+        user_fields : bool, optional
+            If True, return local_H with variables unmapped from arrays back to individual fields.
+            Default is False.
+        as_dicts : bool, optional
+            If True, return local_H as list of dictionaries instead of numpy array.
+            Default is False.
+        Returns
+        -------
+        local_H : npt.NDArray | list
+            Generator history array (unmapped if user_fields=True, as dicts if as_dicts=True).
+        persis_info : dict
+            Persistent information.
+        tag : int
+            Status flag (e.g., FINISHED_PERSISTENT_GEN_TAG).
+        """
+        if not self.gen_result:
+            return (None, None, None)
+        local_H, persis_info, tag = self.gen_result
+        if user_fields and local_H is not None and self.variables_mapping:
+            local_H = unmap_numpy_array(local_H, self.variables_mapping)
+        if as_dicts and local_H is not None:
+            if user_fields and self.variables_mapping:
+                local_H = np_to_list_dicts(local_H, self.variables_mapping, allow_arrays=True)
+            else:
+                local_H = np_to_list_dicts(local_H, allow_arrays=True)
+        return (local_H, persis_info, tag)

libensemble/tests/functionality_tests/check_libE_stats.py

@@ -1,4 +1,4 @@
-""" Script to check format of libE_stats.txt
+"""Script to check format of libE_stats.txt
 
 Checks matching start and end times existing for calculation and tasks if
 required. Checks that dates/times are in a valid format.

libensemble/tests/functionality_tests/test_asktell_sampling.py

@@ -14,8 +14,8 @@
 # TESTSUITE_NPROCS: 2 4
 
 import numpy as np
-from generator_standard import Generator
-from generator_standard.vocs import VOCS
+from gest_api import Generator
+from gest_api.vocs import VOCS
 
 # Import libEnsemble items for this test
 from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f

libensemble/tests/functionality_tests/test_persistent_uniform_gen_decides_stop.py

@@ -82,9 +82,7 @@
             assert (
                 sum(counts == init_batch_size) >= ngens
             ), "The initial batch of each gen should be common among initial_batch_size number of points"
-            assert (
-                len(counts) > 1
-            ), "All gen_ended_times are the same; they should be different for the async case"
+            assert len(counts) > 1, "All gen_ended_times are the same; they should be different for the async case"
 
             gen_workers = np.unique(H["gen_worker"])
             print("Generators that issued points", gen_workers)

libensemble/tests/regression_tests/test_asktell_aposmm_nlopt.py

@@ -28,7 +28,7 @@
 libensemble.gen_funcs.rc.aposmm_optimizers = "nlopt"
 from time import time
 
-from generator_standard.vocs import VOCS
+from gest_api.vocs import VOCS
 
 from libensemble import Ensemble
 from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f
@@ -53,12 +53,13 @@
     workflow.exit_criteria = ExitCriteria(sim_max=2000)
 
     vocs = VOCS(
-        variables={"core": [-3, 3], "edge": [-2, 2]},
+        variables={"core": [-3, 3], "edge": [-2, 2], "core_on_cube": [-3, 3], "edge_on_cube": [-2, 2]},
         objectives={"energy": "MINIMIZE"},
     )
 
     aposmm = APOSMM(
         vocs,
+        variables_mapping={"x": ["core", "edge"], "x_on_cube": ["core_on_cube", "edge_on_cube"], "f": ["energy"]},
         initial_sample_size=100,
         sample_points=minima,
         localopt_method="LN_BOBYQA",
@@ -68,6 +69,7 @@
         max_active_runs=workflow.nworkers,  # should this match nworkers always? practically?
     )
 
+    # SH TODO - dont want this stuff duplicated - pass with vocs instead
     workflow.gen_specs = GenSpecs(
         persis_in=["x", "x_on_cube", "sim_id", "local_min", "local_pt", "f"],
         generator=aposmm,

libensemble/tests/regression_tests/test_asktell_gpCAM.py

@@ -22,7 +22,7 @@
 import warnings
 
 import numpy as np
-from generator_standard.vocs import VOCS
+from gest_api.vocs import VOCS
 
 from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f
 from libensemble.gen_classes.gpCAM import GP_CAM, GP_CAM_Covar

libensemble/tests/regression_tests/test_persistent_gp_multitask_ax.py

@@ -50,7 +50,7 @@ def run_simulation(H, persis_info, sim_specs, libE_info):
         z = 8
     elif task == "cheap_model":
         z = 1
-    print('in sim', task)
+    print("in sim", task)
 
     libE_output = np.zeros(1, dtype=sim_specs["out"])
     calc_status = WORKER_DONE