diff --git a/docs/analysis/duplicate_detection.ipynb b/docs/analysis/duplicate_detection.ipynb index 8488ad6..cf2aa36 100644 --- a/docs/analysis/duplicate_detection.ipynb +++ b/docs/analysis/duplicate_detection.ipynb @@ -5,7 +5,7 @@ "metadata": {}, "source": [ "# Duplicate Individual Detection\n", - "This notebook demonstrates the use of the method `Population.get_num_unique()` to help detect when populations have duplicate individuals. This method uses `np.round` to deal with floating point accuracy issues." + "This page demonstrates the use of the method `Population.get_num_unique()` to help detect when populations have duplicate individuals. This method uses `np.round` to deal with floating point accuracy issues." ] }, { diff --git a/docs/analysis/inverted_generational_distance_analysis.ipynb b/docs/analysis/inverted_generational_distance_analysis.ipynb index bd57b03..e707972 100644 --- a/docs/analysis/inverted_generational_distance_analysis.ipynb +++ b/docs/analysis/inverted_generational_distance_analysis.ipynb @@ -5,7 +5,7 @@ "metadata": {}, "source": [ "# Analysis of Inverted Generational Distance\n", - "This notebook demonstrates how optimization algorithm benchmarking data can be loaded from disk, metrics (such as inverted generational distance) can be calculated for all reported populations, and the resulting data analyzed for comparisons between experiements. We demonstrate the process by comparing three algorithms against eachother on a single test problem. The resulting comparisons are displayed and it is shown how the table can be exported to latex for publication." + "This page demonstrates how optimization algorithm benchmarking data can be loaded from disk, metrics (such as inverted generational distance) can be calculated for all reported populations, and the resulting data analyzed for comparisons between experiements. We demonstrate the process by comparing three algorithms against eachother on a single test problem. The resulting comparisons are displayed and it is shown how the table can be exported to latex for publication." ] }, { diff --git a/docs/container_objects.ipynb b/docs/container_objects.ipynb index a68d04a..72b6520 100644 --- a/docs/container_objects.ipynb +++ b/docs/container_objects.ipynb @@ -5,7 +5,7 @@ "metadata": {}, "source": [ "# Container Objects\n", - "Some objects are included in ParetoBench to enable users to manipulate and save data related to multi-objective optimizations. We use this notebook to explain some of their usage." + "Some objects are included in ParetoBench to enable users to manipulate and save data related to multi-objective optimizations. We use this page to explain some of their usage." ] }, { diff --git a/docs/ext/xopt.ipynb b/docs/ext/xopt.ipynb new file mode 100644 index 0000000..27c810d --- /dev/null +++ b/docs/ext/xopt.ipynb @@ -0,0 +1,285 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Xopt External Connector\n", + "ParetoBench includes code to use its benchmark problems in [Xopt](https://xopt.xopt.org/) and load data output from some of its multiobjective genetic algorithms.\n", + "On this page, we will explore the use of the following ParetoBench features.\n", + "- `XoptProblemWrapper`: Use ParetoBench problems with Xopt.\n", + "- `import_nsga2_history_dir`: Load the output of one or more runs of `NSGA2Generator` into a `paretobench.History`.\n", + "- `import_cnsga_history`: Load the output of `CNSGAGenerator` into a `paretobench.History`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "import tempfile\n", + "import shutil\n", + "import numpy as np\n", + "import os\n", + "from xopt.base import Xopt\n", + "from xopt.evaluator import Evaluator\n", + "from xopt.generators.ga.cnsga import CNSGAGenerator\n", + "from xopt.generators.ga.nsga2 import NSGA2Generator\n", + "\n", + "from paretobench.ext.xopt import (\n", + " XoptProblemWrapper,\n", + " import_cnsga_history,\n", + " import_nsga2_history_dir,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "# Make a directory for our optimization data (cleaned up at end of notebook)\n", + "dir = tempfile.mkdtemp()" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "### Using ParetoBench Problems in Xopt\n", + "In Xopt, problems are described by a VOCS object with the problems variables, objectives, and constraints as well as an evaluation function which maps dicts of variables to dicts of objectives / constraints. `XoptProblemWrapper` adapts any ParetoBench problem to this interface. The VOCS object is available from the wrapper's `vocs` property and the wrapper itself may be passed directly to Xopt's `Evaluator`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "# Wrap one of ParetoBench's problems for use in Xopt\n", + "prob = XoptProblemWrapper.from_line_fmt(\"ZDT3 (n=3)\")\n", + "print(prob)\n", + "\n", + "# The VOCS object is constructed from the problem's variable bounds, objectives, and constraints\n", + "prob.vocs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "# The wrapper is a callable using Xopt's dict -> dict convention\n", + "print(prob({\"x0\": 1.0, \"x1\": 1.5, \"x2\": 2.0}))\n", + "\n", + "# Arrays are also accepted for use with vectorized evaluators\n", + "print(prob({\"x0\": np.array([1.0, 0.5]), \"x1\": np.array([1.5, 0.75]), \"x2\": np.array([2.0, 1.0])}))" + ] + }, + { + "cell_type": "markdown", + "id": "6", + "metadata": {}, + "source": [ + "### Loading Data from NSGA2Generator\n", + "`import_nsga2_history_dir` may be used to load optimization data from the `output_dir` of `NSGA2Generator` into a `paretobench.History` object. We will now run a short optimization on the wrapped problem and load the data to demonstrate the use of the function." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "# Generate optimization data for us to load later\n", + "population_size = 32\n", + "n_generations = 32\n", + "output_dir = os.path.join(dir, \"nsga2_zdt3\")\n", + "\n", + "# Set up and run the optimizer while writing data to `output_dir`\n", + "xx = Xopt(\n", + " generator=NSGA2Generator(vocs=prob.vocs, output_dir=output_dir, population_size=population_size),\n", + " evaluator=Evaluator(function=prob, max_workers=population_size, vectorized=True),\n", + " vocs=prob.vocs,\n", + ")\n", + "for _ in range(n_generations):\n", + " xx.step()\n", + "\n", + "# To avoid errors when running in notebook\n", + "xx.generator.close_log_file()\n", + "\n", + "print(f\"Ran NSGA2 for {n_generations} generations\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "# Load the run into a History object\n", + "hist = import_nsga2_history_dir(output_dir, problem=\"ZDT3 (n=3)\")\n", + "\n", + "# Show some stats from it\n", + "print(hist)\n", + "print(f\"Number of reports: {len(hist)}\")\n", + "print(f\"Individuals in final report: {len(hist.reports[-1])}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "# The method also allows for easy plotting of optimization\n", + "# Here we show all individuals along with the true Pareto front\n", + "import_nsga2_history_dir(output_dir, problem=\"ZDT3 (n=3)\").plot_obj_scatter(show_pf=True)" + ] + }, + { + "cell_type": "markdown", + "id": "10", + "metadata": {}, + "source": [ + "Multiple runs of `NSGA2Generator` created by restarting from checkpoints can be combined into a single `History` object. Pass a list of the output directories (in the order they were run) or a glob pattern such as `\"my_optimization_*\"` to `import_nsga2_history_dir`. The VOCS in each directory must match and the generation numbering must be continuous across the runs (as produced by checkpoint restarts)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "# Select the latest checkpoint file from the run\n", + "checkpoint_dir = os.path.join(output_dir, \"checkpoints\")\n", + "checkpoint_file = max(\n", + " (os.path.join(checkpoint_dir, f) for f in os.listdir(checkpoint_dir)),\n", + " key=os.path.getmtime,\n", + ")\n", + "\n", + "# Run for more generations after checkpoint\n", + "xx = Xopt(\n", + " generator=NSGA2Generator(vocs=prob.vocs, checkpoint_file=checkpoint_file),\n", + " evaluator=Evaluator(function=prob, max_workers=population_size, vectorized=True),\n", + " vocs=prob.vocs,\n", + ")\n", + "for _ in range(n_generations):\n", + " xx.step()\n", + "\n", + "# To avoid errors when running in notebook\n", + "xx.generator.close_log_file()\n", + "\n", + "# Show the newly created file\n", + "print(f\"Completed {n_generations} more generations\")\n", + "print(\"Contents of project directory:\")\n", + "for fname in sorted(os.listdir(dir)):\n", + " print(\" - \" + fname)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "# Plot data from both runs (Note \"*\" character after output directory to match all outputs)\n", + "import_nsga2_history_dir(os.path.join(dir, \"nsga2_zdt3*\"), problem=\"ZDT3 (n=3)\").plot_obj_scatter(show_pf=True)" + ] + }, + { + "cell_type": "markdown", + "id": "13", + "metadata": {}, + "source": [ + "### Loading Data from CNSGAGenerator\n", + "`CNSGAGenerator` saves each generation to a timestamped CSV file (`cnsga_population_*.csv`) when run with the `output_path` option. The function `import_cnsga_history` loads all of the population files in a directory into a `History` object. The population files do not contain the problem definition, so the VOCS object must be passed in as well. A path to a JSON file of the VOCS or the run's YAML config file (through the `config` keyword) also works." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14", + "metadata": {}, + "outputs": [], + "source": [ + "# Generate optimization data using CNSGA\n", + "output_path = os.path.join(dir, \"cnsga\")\n", + "xx = Xopt(\n", + " generator=CNSGAGenerator(vocs=prob.vocs, output_path=output_dir, population_size=population_size),\n", + " evaluator=Evaluator(function=prob, max_workers=population_size, vectorized=True),\n", + " vocs=prob.vocs,\n", + ")\n", + "for _ in range(n_generations):\n", + " xx.step()\n", + "\n", + "print(f\"Ran CNSGA for {n_generations} generations\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "# Load the run into a History object\n", + "import_cnsga_history(output_dir, vocs=prob.vocs, problem=\"ZDT3 (n=3)\").plot_obj_scatter(show_pf=True)" + ] + }, + { + "cell_type": "markdown", + "id": "16", + "metadata": {}, + "source": [ + "### Cleanup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "# Clean up optimization data\n", + "shutil.rmtree(dir)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "paretobench", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/index.md b/docs/index.md index 946564e..1f0a7d7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -53,7 +53,7 @@ Tools for plotting the data from multi-objective optimization algorithms are als - Color coding or animation for showing multiple populations - Markers and alpha to distinguish non-dominated / infeasible solutions -See more information in the following notebooks. +See more information on the following pages. - [Plotting populations](plotting/plotting_populations.ipynb) - [Plotting histories](plotting/plotting_histories.ipynb) diff --git a/docs/plotting/plotting_histories.ipynb b/docs/plotting/plotting_histories.ipynb index 0c1e934..32d7097 100644 --- a/docs/plotting/plotting_histories.ipynb +++ b/docs/plotting/plotting_histories.ipynb @@ -5,7 +5,7 @@ "metadata": {}, "source": [ "# Plotting of History Objects\n", - "In Paretobench, `History` objects contain multiple populations in an ordered list of \"reports\" which represent the progress of an optimization on solving a problem. This notebook demonstrates some options for plotting this data." + "In Paretobench, `History` objects contain multiple populations in an ordered list of \"reports\" which represent the progress of an optimization on solving a problem. This page demonstrates some options for plotting this data." ] }, { diff --git a/docs/plotting/plotting_metrics.ipynb b/docs/plotting/plotting_metrics.ipynb index b39cff6..6d7c83c 100644 --- a/docs/plotting/plotting_metrics.ipynb +++ b/docs/plotting/plotting_metrics.ipynb @@ -6,7 +6,7 @@ "metadata": {}, "source": [ "# Plotting Metrics Across Histories\n", - "This notebooks demonstrates the use of `plot_metric_history` which allows for the plotting the evolution of metrics over each population within a `History` object." + "This page demonstrates the use of `plot_metric_history` which allows for the plotting the evolution of metrics over each population within a `History` object." ] }, { diff --git a/docs/plotting/plotting_populations.ipynb b/docs/plotting/plotting_populations.ipynb index 0957296..dc71e67 100644 --- a/docs/plotting/plotting_populations.ipynb +++ b/docs/plotting/plotting_populations.ipynb @@ -5,7 +5,7 @@ "metadata": {}, "source": [ "# Plotting of Population Objects\n", - "This notebook demonstrates some of the functions available in ParetoBench for plotting the data in `Population` objects." + "This page demonstrates some of the functions available in ParetoBench for plotting the data in `Population` objects." ] }, { diff --git a/docs/problems/problems.ipynb b/docs/problems/problems.ipynb index 2ccd68e..6d9aed6 100644 --- a/docs/problems/problems.ipynb +++ b/docs/problems/problems.ipynb @@ -5,7 +5,7 @@ "metadata": {}, "source": [ "# Benchmark Problems\n", - "This notebook demonstrates the usage of the ParetoBench `Problem` objects which implement common multi-objective benchmark problems in a convient vectorized format." + "This page demonstrates the usage of the ParetoBench `Problem` objects which implement common multi-objective benchmark problems in a convient vectorized format." ] }, { @@ -134,7 +134,7 @@ "metadata": {}, "source": [ "### Analytic Pareto Fronts\n", - "Some of the test problems come with analytic Pareto fronts. These are useful for the evaluating convergence of optimization algorithms under test. In this notebook, we demonstrate how to query points from the Pareto fronts. Problems which can return points from the Pareto front inherit from `ProblemWithPF` and problems with a fixed number of points on the Pareto front inherit from `ProblemWithFixedPF`." + "Some of the test problems come with analytic Pareto fronts. These are useful for the evaluating convergence of optimization algorithms under test. On this page, we demonstrate how to query points from the Pareto fronts. Problems which can return points from the Pareto front inherit from `ProblemWithPF` and problems with a fixed number of points on the Pareto front inherit from `ProblemWithFixedPF`." ] }, { diff --git a/mkdocs.yml b/mkdocs.yml index 45139be..ac24cd2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -34,3 +34,5 @@ nav: - Analysis: - Duplicate Detection: analysis/duplicate_detection.ipynb - IGD Analysis: analysis/inverted_generational_distance_analysis.ipynb + - External Connectors: + - Xopt: ext/xopt.ipynb diff --git a/pyproject.toml b/pyproject.toml index 4582300..c5a235f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ docs = [ "mkdocs", "mkdocs-jupyter", "mkdocs-material", + "xopt" ] [tool.ruff] diff --git a/src/paretobench/ext/xopt.py b/src/paretobench/ext/xopt.py index 2d9c241..0527a0a 100644 --- a/src/paretobench/ext/xopt.py +++ b/src/paretobench/ext/xopt.py @@ -87,6 +87,9 @@ def __init__(self, problem: Problem): """ self.prob = problem + # Hack to enable Xopt serialization to work + self.__qualname__ = type(self).__qualname__ + @classmethod def from_line_fmt(cls, prob_name: str): return cls(Problem.from_line_fmt(prob_name)) @@ -138,7 +141,7 @@ def __repr__(self): return f"XoptProblemWrapper({self.prob.to_line_fmt()})" -def population_from_dataframe(df: pd.DataFrame, vocs: VOCS, errors_as_constraints: bool = False): +def population_from_dataframe(df: pd.DataFrame, vocs: VOCS, errors_as_constraints: bool = False) -> Population: """ Import a population file from an Xopt-style dataframe and VOCs object into a ParetoBench Population object. @@ -194,7 +197,7 @@ def import_cnsga_population( path: str | os.PathLike[str], vocs: VOCS | str | os.PathLike[str] | None = None, errors_as_constraints: bool = False, -): +) -> Population: """ Import a population file from Xopt's CNSGA generator into a ParetoBench Population object. @@ -226,7 +229,7 @@ def import_cnsga_history( config: str | os.PathLike[str] | None = None, problem: str = "", errors_as_constraints: bool = False, -): +) -> History: """ Import all population files in output_path from Xopt's CNSGA generator into a ParetoBench History object. @@ -415,7 +418,7 @@ def import_nsga2_history( config: str | os.PathLike[str] | None = None, problem: str = "", errors_as_constraints: bool = False, -): +) -> History: """ Import all populations from the output of NSGA2Generator. @@ -449,7 +452,7 @@ def import_nsga2_history_multi( config: str | os.PathLike[str] | None = None, problem: str = "", errors_as_constraints: bool = False, -): +) -> History: """ Import populations from several NSGA2Generator runs sharing one VOCS into a single History. @@ -476,11 +479,22 @@ def import_nsga2_history_multi( _vocs = _resolve_vocs(vocs, config) dfs = [] - for path in populations_paths: + for file_idx, path in enumerate(populations_paths): df = pd.read_csv(path) + df["_pb_file_idx"] = file_idx dfs.append(df) combined = pd.concat(dfs, ignore_index=True) + # Check for file with overlapping generation number + last_file_idx = combined.groupby("xopt_generation")["_pb_file_idx"].transform("max") + superseded = combined["_pb_file_idx"] != last_file_idx + if superseded.any(): + overlapping_gens = combined.loc[superseded, "xopt_generation"] + warnings.warn( + f"Generations {overlapping_gens.min()}-{overlapping_gens.max()} appear in multiple populations " + "files. This may result in populations with combined data from more than one file." + ) + hist = _history_from_populations_df(combined, _vocs, problem=problem, errors_as_constraints=errors_as_constraints) logger.info(f"Successfully loaded History object in {time.perf_counter()-start_t:.2f}s: {hist}") return hist @@ -490,7 +504,7 @@ def import_nsga2_history_dir( output_dir: str | os.PathLike[str] | list[str | os.PathLike[str]], problem: str = "", errors_as_constraints: bool = False, -): +) -> History: """ Import all populations from the output of NSGA2Generator (or multiple runs of NSGA2Generator) by specifying output directory. When multiple runs are loaded, they must have matching VOCS and `xopt_generation` must be correct as it is diff --git a/tests/ext/test_xopt.py b/tests/ext/test_xopt.py index 45aac68..ed1427d 100644 --- a/tests/ext/test_xopt.py +++ b/tests/ext/test_xopt.py @@ -7,7 +7,6 @@ from xopt.evaluator import Evaluator from xopt.generators.ga.cnsga import CNSGAGenerator from xopt.generators.ga.nsga2 import NSGA2Generator -from xopt.resources.test_functions.tnk import evaluate_TNK, tnk_vocs import numpy as np # Handle Xopt 2.x and 3.x style VOCS @@ -67,13 +66,18 @@ def _constraint_value(c): import_nsga2_history_dir, ) +# ParetoBench's TNK problem wrapped for use as a vectorized Xopt evaluator function +tnk_wrapper = XoptProblemWrapper.from_line_fmt("TNK") +tnk_vocs = tnk_wrapper.vocs + def _run_nsga2_to_dir(output_dir, population_size=16, n_generations=3, vocs=tnk_vocs): """Run NSGA2Generator into output_dir producing populations.csv and vocs.txt.""" generator = NSGA2Generator(vocs=vocs, output_dir=output_dir, population_size=population_size) - xx = Xopt(generator=generator, evaluator=Evaluator(function=evaluate_TNK, max_workers=1), vocs=vocs) + evaluator = Evaluator(function=tnk_wrapper, max_workers=population_size, vectorized=True) + xx = Xopt(generator=generator, evaluator=evaluator, vocs=vocs) try: - for _ in range(n_generations * population_size): + for _ in range(n_generations): xx.step() finally: xx.generator.close_log_file() @@ -115,7 +119,7 @@ def test_import_nsga2_history(): # Run a few optimization steps xx = Xopt( generator=generator, - evaluator=Evaluator(function=evaluate_TNK, max_workers=1), + evaluator=Evaluator(function=tnk_wrapper, max_workers=population_size, vectorized=True), vocs=tnk_vocs, ) @@ -125,9 +129,8 @@ def test_import_nsga2_history(): xopt_fs = [] xopt_gs = [] for _ in range(n_generations): - # Step the generator - for _ in range(population_size): - xx.step() + # Step the generator (one step evaluates a full generation) + xx.step() fevals += population_size # Get data from the population to test against @@ -175,8 +178,8 @@ def df_comp(df1, df2): # Confirm other metadata assert tp.obj_directions == "--" - assert tp.constraint_directions == "><" - assert all(tp.constraint_targets == [0.0, 0.5]) + assert tp.constraint_directions == "<<" + assert all(tp.constraint_targets == [0.0, 0.0]) assert tp.fevals == (idx + 1) * population_size # Confirm data is correct @@ -205,7 +208,7 @@ def test_import_nsga2_history_dir(): # Run a few optimization steps xx = Xopt( generator=generator, - evaluator=Evaluator(function=evaluate_TNK, max_workers=1), + evaluator=Evaluator(function=tnk_wrapper, max_workers=population_size, vectorized=True), vocs=tnk_vocs, ) @@ -215,9 +218,8 @@ def test_import_nsga2_history_dir(): xopt_fs = [] xopt_gs = [] for _ in range(n_generations): - # Step the generator - for _ in range(population_size): - xx.step() + # Step the generator (one step evaluates a full generation) + xx.step() fevals += population_size # Get data from the population to test against @@ -256,8 +258,8 @@ def df_comp(df1, df2): # Confirm other metadata assert tp.obj_directions == "--" - assert tp.constraint_directions == "><" - assert all(tp.constraint_targets == [0.0, 0.5]) + assert tp.constraint_directions == "<<" + assert all(tp.constraint_targets == [0.0, 0.0]) assert tp.fevals == (idx + 1) * population_size # Confirm data is correct @@ -284,7 +286,7 @@ def test_import_cnsga_history(): xx = Xopt( generator=generator, - evaluator=Evaluator(function=evaluate_TNK, max_workers=1), + evaluator=Evaluator(function=tnk_wrapper, max_workers=population_size, vectorized=True), vocs=tnk_vocs, ) @@ -296,8 +298,7 @@ def test_import_cnsga_history(): xopt_fs = [] xopt_gs = [] for _ in range(n_generations): - for _ in range(population_size): - xx.step() + xx.step() xopt_xs.append(_variable_data(xx.generator.vocs, xx.generator.population)) xopt_fs.append(_objective_data(xx.generator.vocs, xx.generator.population)) xopt_gs.append(_constraint_data(xx.generator.vocs, xx.generator.population)) @@ -337,8 +338,8 @@ def df_comp(df1, df2): # Confirm other metadata assert tp.obj_directions == "--" - assert tp.constraint_directions == "><" - assert all(tp.constraint_targets == [0.0, 0.5]) + assert tp.constraint_directions == "<<" + assert all(tp.constraint_targets == [0.0, 0.0]) assert tp.fevals == (idx + 1) * population_size # Confirm data is correct