diff --git a/docs/function_guides/ask_tell_generator.rst b/docs/function_guides/ask_tell_generator.rst index 6212b24f5d..73f97124c3 100644 --- a/docs/function_guides/ask_tell_generator.rst +++ b/docs/function_guides/ask_tell_generator.rst @@ -8,7 +8,7 @@ These generators, implementations, methods, and subclasses are in BETA, and may change in future releases. The Generator interface is expected to roughly correspond with CAMPA's standard: -https://github.com/campa-consortium/generator_standard +https://github.com/campa-consortium/gest-api libEnsemble is in the process of supporting generator objects that implement the following interface: diff --git a/libensemble/gen_classes/aposmm.py b/libensemble/gen_classes/aposmm.py index 45a522279c..cd9a9c257b 100644 --- a/libensemble/gen_classes/aposmm.py +++ b/libensemble/gen_classes/aposmm.py @@ -2,7 +2,7 @@ from typing import List import numpy as np -from generator_standard.vocs import VOCS +from gest_api.vocs import VOCS from numpy import typing as npt from libensemble.generators import PersistentGenInterfacer @@ -12,6 +12,21 @@ class APOSMM(PersistentGenInterfacer): """ Standalone object-oriented APOSMM generator + + VOCS variables must include both regular and *_on_cube versions. E.g.,: + vars_std = { + "var1": [-10.0, 10.0], + "var2": [0.0, 100.0], + "var3": [1.0, 50.0], + "var1_on_cube": [0, 1.0], + "var2_on_cube": [0, 1.0], + "var3_on_cube": [0, 1.0] + } + variables_mapping = { + "x": ["var1", "var2", "var3"], + "x_on_cube": ["var1_on_cube", "var2_on_cube", "var3_on_cube"], + } + gen = APOSMM(vocs, variables_mapping=variables_mapping, ...) """ def __init__( @@ -26,25 +41,36 @@ def __init__( from libensemble.gen_funcs.persistent_aposmm import aposmm self.VOCS = vocs - gen_specs["gen_f"] = aposmm - self.n = len(list(self.VOCS.variables.keys())) - gen_specs["user"] = {} - gen_specs["user"]["lb"] = np.array([vocs.variables[i].domain[0] for i in vocs.variables]) - gen_specs["user"]["ub"] = np.array([vocs.variables[i].domain[1] for i in vocs.variables]) + super().__init__(vocs, History, persis_info, gen_specs, libE_info, **kwargs) + + # Set bounds using the correct x mapping + x_mapping = self.variables_mapping["x"] + self.gen_specs["user"]["lb"] = np.array([vocs.variables[var].domain[0] for var in x_mapping]) + self.gen_specs["user"]["ub"] = np.array([vocs.variables[var].domain[1] for var in x_mapping]) + + if not gen_specs.get("out"): + x_size = len(self.variables_mapping.get("x", [])) + x_on_cube_size = len(self.variables_mapping.get("x_on_cube", [])) + assert x_size > 0 and x_on_cube_size > 0, "Both x and x_on_cube must be specified in variables_mapping" + assert ( + x_size == x_on_cube_size + ), f"x and x_on_cube must have same length but got {x_size} and {x_on_cube_size}" - if not gen_specs.get("out"): # gen_specs never especially changes for aposmm even as the problem varies gen_specs["out"] = [ - ("x", float, self.n), - ("x_on_cube", float, self.n), + ("x", float, x_size), + ("x_on_cube", float, x_on_cube_size), ("sim_id", int), ("local_min", bool), ("local_pt", bool), ] - gen_specs["persis_in"] = ["x", "f", "local_pt", "sim_id", "sim_ended", "x_on_cube", "local_min"] - super().__init__(vocs, History, persis_info, gen_specs, libE_info, **kwargs) + gen_specs["persis_in"] = ["sim_id", "x", "x_on_cube", "f", "sim_ended"] + if "components" in kwargs or "components" in gen_specs.get("user", {}): + gen_specs["persis_in"].append("fvec") + + # SH - Need to know if this is gen_on_manager or not. if not self.persis_info.get("nworkers"): self.persis_info["nworkers"] = kwargs.get("nworkers", gen_specs["user"].get("max_active_runs", 4)) self.all_local_minima = [] diff --git a/libensemble/gen_classes/gpCAM.py b/libensemble/gen_classes/gpCAM.py index 585fe46967..5118ffdbca 100644 --- a/libensemble/gen_classes/gpCAM.py +++ b/libensemble/gen_classes/gpCAM.py @@ -4,7 +4,7 @@ from typing import List import numpy as np -from generator_standard.vocs import VOCS +from gest_api.vocs import VOCS from gpcam import GPOptimizer as GP from numpy import typing as npt @@ -55,9 +55,9 @@ def __init__(self, VOCS: VOCS, ask_max_iter: int = 10, random_seed: int = 1, *ar self.noise = 1e-8 # 1e-12 self.ask_max_iter = ask_max_iter - def _validate_vocs(self, VOCS): - assert len(self.VOCS.variables), "VOCS must contain variables." - assert len(self.VOCS.objectives), "VOCS must contain at least one objective." + def _validate_vocs(self, vocs): + assert len(vocs.variables), "VOCS must contain variables." + assert len(vocs.objectives), "VOCS must contain at least one objective." def suggest_numpy(self, n_trials: int) -> npt.NDArray: if self.all_x.shape[0] == 0: diff --git a/libensemble/gen_classes/sampling.py b/libensemble/gen_classes/sampling.py index 72263750e1..5e8102c22b 100644 --- a/libensemble/gen_classes/sampling.py +++ b/libensemble/gen_classes/sampling.py @@ -1,7 +1,7 @@ """Generator classes providing points using sampling""" import numpy as np -from generator_standard.vocs import VOCS +from gest_api.vocs import VOCS from libensemble.generators import LibensembleGenerator diff --git a/libensemble/gen_funcs/aposmm_localopt_support.py b/libensemble/gen_funcs/aposmm_localopt_support.py index 190e02dadd..901162783f 100644 --- a/libensemble/gen_funcs/aposmm_localopt_support.py +++ b/libensemble/gen_funcs/aposmm_localopt_support.py @@ -17,6 +17,7 @@ import numpy as np import psutil +import traceback import libensemble.gen_funcs from libensemble.message_numbers import EVAL_GEN_TAG, STOP_TAG # Only used to simulate receiving from manager @@ -585,8 +586,8 @@ def run_local_tao(user_specs, comm_queue, x0, f0, child_can_read, parent_can_rea def opt_runner(run_local_opt, user_specs, comm_queue, x0, f0, child_can_read, parent_can_read): try: run_local_opt(user_specs, comm_queue, x0, f0, child_can_read, parent_can_read) - except Exception as e: - comm_queue.put(ErrorMsg(e)) + except Exception: + comm_queue.put(ErrorMsg(traceback.format_exc())) parent_can_read.set() diff --git a/libensemble/generators.py b/libensemble/generators.py index fa91ec4079..7c7c5b933f 100644 --- a/libensemble/generators.py +++ b/libensemble/generators.py @@ -2,15 +2,15 @@ from typing import List, Optional import numpy as np -from generator_standard import Generator -from generator_standard.vocs import VOCS +from gest_api import Generator +from gest_api.vocs import VOCS from numpy import typing as npt from libensemble.comms.comms import QCommProcess # , QCommThread from libensemble.executors import Executor from libensemble.message_numbers import EVAL_GEN_TAG, PERSIS_STOP from libensemble.tools.tools import add_unique_random_streams -from libensemble.utils.misc import list_dicts_to_np, np_to_list_dicts +from libensemble.utils.misc import list_dicts_to_np, np_to_list_dicts, unmap_numpy_array class GeneratorNotStartedException(Exception): @@ -55,12 +55,18 @@ def __init__( self.variables_mapping = variables_mapping if not self.variables_mapping: + self.variables_mapping = {} + # Map variables to x if not already mapped + if "x" not in self.variables_mapping: + # SH TODO - is this check needed? if len(list(self.VOCS.variables.keys())) > 1 or list(self.VOCS.variables.keys())[0] != "x": - self.variables_mapping["x"] = list(self.VOCS.variables.keys()) + self.variables_mapping["x"] = self._get_unmapped_keys(self.VOCS.variables, "x") + # Map objectives to f if not already mapped + if "f" not in self.variables_mapping: if ( len(list(self.VOCS.objectives.keys())) > 1 or list(self.VOCS.objectives.keys())[0] != "f" ): # e.g. {"f": ["f"]} doesn't need mapping - self.variables_mapping["f"] = list(self.VOCS.objectives.keys()) + self.variables_mapping["f"] = self._get_unmapped_keys(self.VOCS.objectives, "f") if len(kwargs) > 0: # so user can specify gen-specific parameters as kwargs to constructor if not self.gen_specs.get("user"): @@ -74,6 +80,15 @@ def __init__( def _validate_vocs(self, vocs) -> None: pass + def _get_unmapped_keys(self, vocs_dict, default_key): + """Get keys from vocs_dict that aren't already mapped to other keys in variables_mapping.""" + # Get all variables that aren't already mapped to other keys + mapped_vars = [] + for mapped_list in self.variables_mapping.values(): + mapped_vars.extend(mapped_list) + unmapped_vars = [v for v in list(vocs_dict.keys()) if v not in mapped_vars] + return unmapped_vars + @abstractmethod def suggest_numpy(self, num_points: Optional[int] = 0) -> npt.NDArray: """Request the next set of points to evaluate, as a NumPy array.""" @@ -119,6 +134,7 @@ def __init__( self.History = History self.libE_info = libE_info self.running_gen_f = None + self.gen_result = None def setup(self) -> None: """Must be called once before calling suggest/ingest. Initializes the background thread.""" @@ -139,16 +155,24 @@ def setup(self) -> None: user_function=True, ) - # this is okay since the object isnt started until the first suggest + # This can be set here since the object isnt started until the first suggest self.libE_info["comm"] = self.running_gen_f.comm - def _set_sim_ended(self, results: npt.NDArray) -> npt.NDArray: - new_results = np.zeros(len(results), dtype=self.gen_specs["out"] + [("sim_ended", bool), ("f", float)]) - for field in results.dtype.names: + def _prep_fields(self, results: npt.NDArray) -> npt.NDArray: + """Filter out fields that are not in persis_in and add sim_ended to the dtype""" + filtered_dtype = [ + (name, results.dtype[name]) for name in results.dtype.names if name in self.gen_specs["persis_in"] + ] + + new_dtype = filtered_dtype + [("sim_ended", bool)] + new_results = np.zeros(len(results), dtype=new_dtype) + + for field in new_results.dtype.names: try: new_results[field] = results[field] - except ValueError: # lets not slot in data that the gen doesnt need? + except ValueError: continue + new_results["sim_ended"] = True return new_results @@ -167,7 +191,7 @@ def suggest_numpy(self, num_points: int = 0) -> npt.NDArray: def ingest_numpy(self, results: npt.NDArray, tag: int = EVAL_GEN_TAG) -> None: """Send the results of evaluations to the generator, as a NumPy array.""" if results is not None: - results = self._set_sim_ended(results) + results = self._prep_fields(results) Work = {"libE_info": {"H_rows": np.copy(results["sim_id"]), "persistent": True, "executor": None}} self.running_gen_f.send(tag, Work) self.running_gen_f.send( @@ -176,7 +200,40 @@ def ingest_numpy(self, results: npt.NDArray, tag: int = EVAL_GEN_TAG) -> None: else: self.running_gen_f.send(tag, None) - def finalize(self, results: npt.NDArray = None) -> (npt.NDArray, dict, int): - """Send any last results to the generator, and it to close down.""" - self.ingest_numpy(results, PERSIS_STOP) # conversion happens in ingest - return self.running_gen_f.result() + def finalize(self) -> None: + """Stop the generator process and store the returned data.""" + self.ingest_numpy(None, PERSIS_STOP) # conversion happens in ingest + self.gen_result = self.running_gen_f.result() + + def export( + self, user_fields: bool = False, as_dicts: bool = False + ) -> tuple[npt.NDArray | list | None, dict | None, int | None]: + """Return the generator's results + Parameters + ---------- + user_fields : bool, optional + If True, return local_H with variables unmapped from arrays back to individual fields. + Default is False. + as_dicts : bool, optional + If True, return local_H as list of dictionaries instead of numpy array. + Default is False. + Returns + ------- + local_H : npt.NDArray | list + Generator history array (unmapped if user_fields=True, as dicts if as_dicts=True). + persis_info : dict + Persistent information. + tag : int + Status flag (e.g., FINISHED_PERSISTENT_GEN_TAG). + """ + if not self.gen_result: + return (None, None, None) + local_H, persis_info, tag = self.gen_result + if user_fields and local_H is not None and self.variables_mapping: + local_H = unmap_numpy_array(local_H, self.variables_mapping) + if as_dicts and local_H is not None: + if user_fields and self.variables_mapping: + local_H = np_to_list_dicts(local_H, self.variables_mapping, allow_arrays=True) + else: + local_H = np_to_list_dicts(local_H, allow_arrays=True) + return (local_H, persis_info, tag) diff --git a/libensemble/tests/functionality_tests/check_libE_stats.py b/libensemble/tests/functionality_tests/check_libE_stats.py index 424c07d8b1..304925dc1e 100644 --- a/libensemble/tests/functionality_tests/check_libE_stats.py +++ b/libensemble/tests/functionality_tests/check_libE_stats.py @@ -1,4 +1,4 @@ -""" Script to check format of libE_stats.txt +"""Script to check format of libE_stats.txt Checks matching start and end times existing for calculation and tasks if required. Checks that dates/times are in a valid format. diff --git a/libensemble/tests/functionality_tests/test_asktell_sampling.py b/libensemble/tests/functionality_tests/test_asktell_sampling.py index e4fb1a88b6..55e3b7afc3 100644 --- a/libensemble/tests/functionality_tests/test_asktell_sampling.py +++ b/libensemble/tests/functionality_tests/test_asktell_sampling.py @@ -14,8 +14,8 @@ # TESTSUITE_NPROCS: 2 4 import numpy as np -from generator_standard import Generator -from generator_standard.vocs import VOCS +from gest_api import Generator +from gest_api.vocs import VOCS # Import libEnsemble items for this test from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f diff --git a/libensemble/tests/functionality_tests/test_persistent_uniform_gen_decides_stop.py b/libensemble/tests/functionality_tests/test_persistent_uniform_gen_decides_stop.py index 68c8aaaa05..d9b9465080 100644 --- a/libensemble/tests/functionality_tests/test_persistent_uniform_gen_decides_stop.py +++ b/libensemble/tests/functionality_tests/test_persistent_uniform_gen_decides_stop.py @@ -82,9 +82,7 @@ assert ( sum(counts == init_batch_size) >= ngens ), "The initial batch of each gen should be common among initial_batch_size number of points" - assert ( - len(counts) > 1 - ), "All gen_ended_times are the same; they should be different for the async case" + assert len(counts) > 1, "All gen_ended_times are the same; they should be different for the async case" gen_workers = np.unique(H["gen_worker"]) print("Generators that issued points", gen_workers) diff --git a/libensemble/tests/regression_tests/test_asktell_aposmm_nlopt.py b/libensemble/tests/regression_tests/test_asktell_aposmm_nlopt.py index 0eec667f75..0f80e42ca1 100644 --- a/libensemble/tests/regression_tests/test_asktell_aposmm_nlopt.py +++ b/libensemble/tests/regression_tests/test_asktell_aposmm_nlopt.py @@ -28,7 +28,7 @@ libensemble.gen_funcs.rc.aposmm_optimizers = "nlopt" from time import time -from generator_standard.vocs import VOCS +from gest_api.vocs import VOCS from libensemble import Ensemble from libensemble.alloc_funcs.persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f @@ -53,12 +53,13 @@ workflow.exit_criteria = ExitCriteria(sim_max=2000) vocs = VOCS( - variables={"core": [-3, 3], "edge": [-2, 2]}, + variables={"core": [-3, 3], "edge": [-2, 2], "core_on_cube": [-3, 3], "edge_on_cube": [-2, 2]}, objectives={"energy": "MINIMIZE"}, ) aposmm = APOSMM( vocs, + variables_mapping={"x": ["core", "edge"], "x_on_cube": ["core_on_cube", "edge_on_cube"], "f": ["energy"]}, initial_sample_size=100, sample_points=minima, localopt_method="LN_BOBYQA", @@ -68,6 +69,7 @@ max_active_runs=workflow.nworkers, # should this match nworkers always? practically? ) + # SH TODO - dont want this stuff duplicated - pass with vocs instead workflow.gen_specs = GenSpecs( persis_in=["x", "x_on_cube", "sim_id", "local_min", "local_pt", "f"], generator=aposmm, diff --git a/libensemble/tests/regression_tests/test_asktell_gpCAM.py b/libensemble/tests/regression_tests/test_asktell_gpCAM.py index 3a10a1072d..b093a0df7a 100644 --- a/libensemble/tests/regression_tests/test_asktell_gpCAM.py +++ b/libensemble/tests/regression_tests/test_asktell_gpCAM.py @@ -22,7 +22,7 @@ import warnings import numpy as np -from generator_standard.vocs import VOCS +from gest_api.vocs import VOCS from libensemble.alloc_funcs.start_only_persistent import only_persistent_gens as alloc_f from libensemble.gen_classes.gpCAM import GP_CAM, GP_CAM_Covar diff --git a/libensemble/tests/regression_tests/test_persistent_gp_multitask_ax.py b/libensemble/tests/regression_tests/test_persistent_gp_multitask_ax.py index 8c589161ad..990493a176 100644 --- a/libensemble/tests/regression_tests/test_persistent_gp_multitask_ax.py +++ b/libensemble/tests/regression_tests/test_persistent_gp_multitask_ax.py @@ -50,7 +50,7 @@ def run_simulation(H, persis_info, sim_specs, libE_info): z = 8 elif task == "cheap_model": z = 1 - print('in sim', task) + print("in sim", task) libE_output = np.zeros(1, dtype=sim_specs["out"]) calc_status = WORKER_DONE diff --git a/libensemble/tests/unit_tests/test_asktell.py b/libensemble/tests/unit_tests/test_asktell.py index 3575bfc076..d8c90d741b 100644 --- a/libensemble/tests/unit_tests/test_asktell.py +++ b/libensemble/tests/unit_tests/test_asktell.py @@ -1,4 +1,5 @@ import numpy as np +from libensemble.utils.misc import unmap_numpy_array def _check_conversion(H, npp, mapping={}): @@ -92,6 +93,64 @@ def test_awkward_H(): _check_conversion(H, npp) +def test_unmap_numpy_array_basic(): + """Test basic unmapping of x and x_on_cube arrays""" + + dtype = [("sim_id", int), ("x", float, (3,)), ("x_on_cube", float, (3,)), ("f", float), ("grad", float, (3,))] + H = np.zeros(2, dtype=dtype) + H[0] = (0, [1.1, 2.2, 3.3], [0.1, 0.2, 0.3], 10.5, [0.1, 0.2, 0.3]) + H[1] = (1, [4.4, 5.5, 6.6], [0.4, 0.5, 0.6], 20.7, [0.4, 0.5, 0.6]) + + mapping = {"x": ["x0", "x1", "x2"], "x_on_cube": ["x0_cube", "x1_cube", "x2_cube"]} + H_unmapped = unmap_numpy_array(H, mapping) + + expected_fields = ["sim_id", "x0", "x1", "x2", "x0_cube", "x1_cube", "x2_cube", "f"] + assert all(field in H_unmapped.dtype.names for field in expected_fields) + + assert H_unmapped["x0"][0] == 1.1 + assert H_unmapped["x1"][0] == 2.2 + assert H_unmapped["x2"][0] == 3.3 + assert H_unmapped["x0_cube"][0] == 0.1 + assert H_unmapped["x1_cube"][0] == 0.2 + assert H_unmapped["x2_cube"][0] == 0.3 + # Test that non-mapped array fields are passed through unchanged + assert "grad" in H_unmapped.dtype.names + assert np.array_equal(H_unmapped["grad"], H["grad"]) + + +def test_unmap_numpy_array_single_dimension(): + """Test unmapping with single dimension""" + + dtype = [("sim_id", int), ("x", float, (1,)), ("f", float)] + H = np.zeros(1, dtype=dtype) + H[0] = (0, [5.5], 15.0) + + mapping = {"x": ["x0"]} + H_unmapped = unmap_numpy_array(H, mapping) + + assert "x0" in H_unmapped.dtype.names + assert H_unmapped["x0"][0] == 5.5 + + +def test_unmap_numpy_array_edge_cases(): + """Test edge cases for unmap_numpy_array""" + + dtype = [("sim_id", int), ("x", float, (2,)), ("f", float)] + H = np.zeros(1, dtype=dtype) + H[0] = (0, [1.0, 2.0], 10.0) + + # No mapping + H_no_mapping = unmap_numpy_array(H, {}) + assert H_no_mapping is H + + # None array + H_none = unmap_numpy_array(None, {"x": ["x0", "x1"]}) + assert H_none is None + + if __name__ == "__main__": test_awkward_list_dict() test_awkward_H() + test_unmap_numpy_array_basic() + test_unmap_numpy_array_single_dimension() + test_unmap_numpy_array_edge_cases() diff --git a/libensemble/tests/unit_tests/test_persistent_aposmm.py b/libensemble/tests/unit_tests/test_persistent_aposmm.py index d04d561986..8ea4eebed3 100644 --- a/libensemble/tests/unit_tests/test_persistent_aposmm.py +++ b/libensemble/tests/unit_tests/test_persistent_aposmm.py @@ -172,7 +172,7 @@ def test_standalone_persistent_aposmm_combined_func(): def test_asktell_with_persistent_aposmm(): from math import gamma, pi, sqrt - from generator_standard.vocs import VOCS + from gest_api.vocs import VOCS import libensemble.gen_funcs from libensemble.gen_classes import APOSMM @@ -185,13 +185,20 @@ def test_asktell_with_persistent_aposmm(): n = 2 eval_max = 2000 - variables = {"core": [-3, 3], "edge": [-2, 2]} + variables = {"core": [-3, 3], "edge": [-2, 2], "core_on_cube": [0, 1], "edge_on_cube": [0, 1]} objectives = {"energy": "MINIMIZE"} + variables_mapping = { + "x": ["core", "edge"], + "x_on_cube": ["core_on_cube", "edge_on_cube"], + "f": ["energy"], + } + vocs = VOCS(variables=variables, objectives=objectives) my_APOSMM = APOSMM( vocs, + variables_mapping=variables_mapping, initial_sample_size=100, sample_points=np.round(minima, 1), localopt_method="LN_BOBYQA", @@ -218,6 +225,8 @@ def test_asktell_with_persistent_aposmm(): while total_evals < eval_max: sample, detected_minima = my_APOSMM.suggest(6), my_APOSMM.suggest_updates() + if detected_minima: + print(f"sample {sample} detected_minima: {detected_minima}") if len(detected_minima): for m in detected_minima: potential_minima.append(m) @@ -225,7 +234,10 @@ def test_asktell_with_persistent_aposmm(): point["energy"] = six_hump_camel_func(np.array([point["core"], point["edge"]])) total_evals += 1 my_APOSMM.ingest(sample) - H, persis_info, exit_code = my_APOSMM.finalize() + my_APOSMM.finalize() + H, persis_info, exit_code = my_APOSMM.export() + + print(f"Number of local_min points in H: {np.sum(H['local_min'])}", flush=True) assert exit_code == FINISHED_PERSISTENT_GEN_TAG, "Standalone persistent_aposmm didn't exit correctly" assert persis_info.get("run_order"), "Standalone persistent_aposmm didn't do any localopt runs" @@ -243,9 +255,90 @@ def test_asktell_with_persistent_aposmm(): assert min_found >= 6, f"Found {min_found} minima" +def _run_aposmm_export_test(variables_mapping): + """Helper function to run APOSMM export tests with given variables_mapping""" + from gest_api.vocs import VOCS + from libensemble.gen_classes import APOSMM + + variables = { + "core": [-3, 3], + "edge": [-2, 2], + "core_on_cube": [0, 1], + "edge_on_cube": [0, 1], + } + objectives = {"energy": "MINIMIZE"} + + vocs = VOCS(variables=variables, objectives=objectives) + aposmm = APOSMM( + vocs, + variables_mapping=variables_mapping, + initial_sample_size=10, + localopt_method="LN_BOBYQA", + xtol_abs=1e-6, + ftol_abs=1e-6, + dist_to_bound_multiple=0.5, + max_active_runs=6, + ) + # Test basic export before finalize + H, _, _ = aposmm.export() + print(f"Export before finalize: {H}") # Debug + assert H is None # Should be None before finalize + # Test export after suggest/ingest cycle + sample = aposmm.suggest(5) + for point in sample: + point["energy"] = 1.0 # Mock evaluation + aposmm.ingest(sample) + aposmm.finalize() + + # Test export with unmapped fields + H, _, _ = aposmm.export() + if H is not None: + assert "x" in H.dtype.names and H["x"].ndim == 2 + assert "f" in H.dtype.names and H["f"].ndim == 1 + + # Test export with user_fields + H_unmapped, _, _ = aposmm.export(user_fields=True) + print(f"H_unmapped: {H_unmapped}") # Debug + if H_unmapped is not None: + assert "core" in H_unmapped.dtype.names + assert "edge" in H_unmapped.dtype.names + assert "energy" in H_unmapped.dtype.names + # Test export with as_dicts + H_dicts, _, _ = aposmm.export(as_dicts=True) + assert isinstance(H_dicts, list) + assert isinstance(H_dicts[0], dict) + assert "x" in H_dicts[0] # x remains as array + assert "f" in H_dicts[0] + # Test export with both options + H_both, _, _ = aposmm.export(user_fields=True, as_dicts=True) + assert isinstance(H_both, list) + assert "core" in H_both[0] + assert "edge" in H_both[0] + assert "energy" in H_both[0] + + +@pytest.mark.extra +def test_aposmm_export(): + """Test APOSMM export function with different options""" + + # Test with full variables_mapping + full_mapping = { + "x": ["core", "edge"], + "x_on_cube": ["core_on_cube", "edge_on_cube"], + "f": ["energy"], + } + _run_aposmm_export_test(full_mapping) + # Test with just x_on_cube mapping (should auto-map x and f) + minimal_mapping = { + "x_on_cube": ["core_on_cube", "edge_on_cube"], + } + _run_aposmm_export_test(minimal_mapping) + + if __name__ == "__main__": test_persis_aposmm_localopt_test() test_update_history_optimal() test_standalone_persistent_aposmm() test_standalone_persistent_aposmm_combined_func() test_asktell_with_persistent_aposmm() + test_aposmm_export() diff --git a/libensemble/tests/unit_tests_logger/test_logger.py b/libensemble/tests/unit_tests_logger/test_logger.py index e06331b3d2..fdf13725f9 100644 --- a/libensemble/tests/unit_tests_logger/test_logger.py +++ b/libensemble/tests/unit_tests_logger/test_logger.py @@ -124,7 +124,7 @@ def test_custom_log_levels(): logger_test.manager_warning("This manager_warning message should log") logger_test.vdebug("This vdebug message should log") - with open(LogConfig.config.filename, 'r') as f: + with open(LogConfig.config.filename, "r") as f: file_content = f.read() assert "This manager_warning message should log" in file_content assert "This vdebug message should log" in file_content diff --git a/libensemble/utils/misc.py b/libensemble/utils/misc.py index 0c03d63696..dfc39e5382 100644 --- a/libensemble/utils/misc.py +++ b/libensemble/utils/misc.py @@ -186,7 +186,49 @@ def _is_singledim(selection: npt.NDArray) -> bool: return (hasattr(selection, "__len__") and len(selection) == 1) or selection.shape == () -def np_to_list_dicts(array: npt.NDArray, mapping: dict = {}) -> List[dict]: +def unmap_numpy_array(array: npt.NDArray, mapping: dict = {}) -> npt.NDArray: + """Convert numpy array with mapped fields back to individual scalar fields. + Parameters + ---------- + array : npt.NDArray + Input array with mapped fields like x = [x0, x1, x2] + mapping : dict + Mapping from field names to variable names + Returns + ------- + npt.NDArray + Array with unmapped fields like x0, x1, x2 as individual scalars + """ + if not mapping or array is None: + return array + # Create new dtype with unmapped fields + new_fields = [] + for field in array.dtype.names: + if field in mapping: + for var_name in mapping[field]: + new_fields.append((var_name, array[field].dtype.type)) + else: + # Preserve the original field structure including per-row shape + field_dtype = array.dtype[field] + new_fields.append((field, field_dtype)) + unmapped_array = np.zeros(len(array), dtype=new_fields) + for field in array.dtype.names: + if field in mapping: + # Unmap array fields + if len(array[field].shape) == 1: + # Scalar field mapped to single variable + unmapped_array[mapping[field][0]] = array[field] + else: + # Multi-dimensional field + for i, var_name in enumerate(mapping[field]): + unmapped_array[var_name] = array[field][:, i] + else: + # Copy non-mapped fields + unmapped_array[field] = array[field] + return unmapped_array + + +def np_to_list_dicts(array: npt.NDArray, mapping: dict = {}, allow_arrays: bool = False) -> List[dict]: if array is None: return None out = [] @@ -196,9 +238,8 @@ def np_to_list_dicts(array: npt.NDArray, mapping: dict = {}) -> List[dict]: for field in row.dtype.names: # non-string arrays, lists, etc. - if field not in list(mapping.keys()): - if _is_multidim(row[field]): + if _is_multidim(row[field]) and not allow_arrays: for i, x in enumerate(row[field]): new_dict[field + str(i)] = x diff --git a/pyproject.toml b/pyproject.toml index 882bcbbb36..69be281995 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ authors = [{name = "Jeffrey Larson"}, {name = "Stephen Hudson"}, {name = "Stefan M. Wild"}, {name = "David Bindel"}, {name = "John-Luke Navarro"}] -dependencies = [ "numpy", "psutil", "pyyaml", "tomli", "campa-generator-standard @ git+https://github.com/campa-consortium/generator_standard@main", "pydantic"] +dependencies = ["numpy", "psutil", "pyyaml", "tomli", "gest @ git+https://github.com/campa-consortium/gest-api@main", "pydantic"] description = "A Python toolkit for coordinating asynchronous and dynamic ensembles of calculations." name = "libensemble" diff --git a/scripts/plot_libe_calcs_util_v_time.py b/scripts/plot_libe_calcs_util_v_time.py index 9f9f22edda..fc6750a10d 100755 --- a/scripts/plot_libe_calcs_util_v_time.py +++ b/scripts/plot_libe_calcs_util_v_time.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -""" User function utilization plot +"""User function utilization plot Script to produce utilization plot based on how many workers are running user functions (sim or gens) at any given time. The plot is written to a file. diff --git a/scripts/plot_libe_histogram.py b/scripts/plot_libe_histogram.py index e5145bc05d..9365571404 100755 --- a/scripts/plot_libe_histogram.py +++ b/scripts/plot_libe_histogram.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -""" Histogram of user function run-times (completed & killed). +"""Histogram of user function run-times (completed & killed). Script to produce a histogram plot giving a count of user function (sim or gen) calls by run-time intervals. Color shows completed versus killed versus diff --git a/scripts/plot_libe_tasks_util_v_time.py b/scripts/plot_libe_tasks_util_v_time.py index ece34bdafb..cb5ced7236 100644 --- a/scripts/plot_libe_tasks_util_v_time.py +++ b/scripts/plot_libe_tasks_util_v_time.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -""" User tasks utilization plot +"""User tasks utilization plot Script to produce utilisation plot based on how many workers are running user tasks (submitted via a libEnsemble executor) at any given time. This does not