From 7c1c53b1e524bcb27b05fbccfa2b3dac49ca37b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Mon, 2 Dec 2024 09:35:17 +0100 Subject: [PATCH 01/30] feat: Initial work on expyrementor --- Expyrementor/__init__.py | 0 Expyrementor/expyrementor.py | 77 ++++++++++++++++ Expyrementor/suggestors/__init__.py | 18 ++++ Expyrementor/suggestors/default_suggestor.py | 24 +++++ .../suggestors/initial_points_suggestor.py | 80 +++++++++++++++++ Expyrementor/suggestors/po_suggestor.py | 32 +++++++ Expyrementor/suggestors/random_strategizer.py | 33 +++++++ Expyrementor/suggestors/strategizer.py | 30 +++++++ Expyrementor/suggestors/suggestor.py | 38 ++++++++ Expyrementor/suggestors/suggestor_factory.py | 90 +++++++++++++++++++ ProcessOptimizer/tests/test_expyrementor.py | 81 +++++++++++++++++ .../tests/test_suggestors/__init__.py | 0 .../test_initial_point_suggestor.py | 51 +++++++++++ .../test_suggestors/test_random_suggestor.py | 28 ++++++ setup.py | 2 + 15 files changed, 584 insertions(+) create mode 100644 Expyrementor/__init__.py create mode 100644 Expyrementor/expyrementor.py create mode 100644 Expyrementor/suggestors/__init__.py create mode 100644 Expyrementor/suggestors/default_suggestor.py create mode 100644 Expyrementor/suggestors/initial_points_suggestor.py create mode 100644 Expyrementor/suggestors/po_suggestor.py create mode 100644 Expyrementor/suggestors/random_strategizer.py create mode 100644 Expyrementor/suggestors/strategizer.py create mode 100644 Expyrementor/suggestors/suggestor.py create mode 100644 Expyrementor/suggestors/suggestor_factory.py create mode 100644 ProcessOptimizer/tests/test_expyrementor.py create mode 100644 ProcessOptimizer/tests/test_suggestors/__init__.py create mode 100644 ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py create mode 100644 ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py diff --git a/Expyrementor/__init__.py b/Expyrementor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/Expyrementor/expyrementor.py b/Expyrementor/expyrementor.py new file mode 100644 index 00000000..97bec468 --- /dev/null +++ b/Expyrementor/expyrementor.py @@ -0,0 +1,77 @@ +import logging +import warnings +from typing import Any, Union + +import numpy as np +from ProcessOptimizer.space import space_factory, Space +from ProcessOptimizer.utils.get_rng import get_random_generator + +from .suggestors import BatchSuggestor, DefaultSuggestor, Suggestor, suggestor_factory + +logger = logging.getLogger(__name__) + +DEFAULT_SUGGESTOR = { + "type": "InitialPoint", + "initial_suggestor": {"type": "Default"}, + "ultimate_suggestor": {"type": "Default"}, + "n_initial_points": 5 +} + + +class Expyrementor: + """ + Expyrementor class for optimization experiments. This class is used to manage the + optimization process, including the search space, the suggestor, and the already + evaluated points. The ask-tell interface is used to interact with the optimization + process. The Expyrementor class is stateful and keeps track of the already evaluated + points and scores. + """ + def __init__( + self, + space: Union[Space, list], + suggestor: Union[Suggestor, dict, None] = None, + n_objectives: int = 1, + seed: Union[int, np.random.RandomState, np.random.Generator, None] = 42 + ): + """ + Initialize the Expyrementor with the search space and the suggestor. The suggestor + can be a Suggestor object, a dictionary with the suggestor configuration, or None. + If the suggestor is None, the default suggestor is used. The seed is used to + initialize the random number generator. + """ + space = space_factory(space) + rng = get_random_generator(seed) + suggestor = suggestor_factory(space, suggestor, n_objectives, rng) + if isinstance(suggestor, DefaultSuggestor): + logger.debug("Replacing DefaultSuggestor with InitialPointSuggestor") + suggestor = suggestor_factory(space, DEFAULT_SUGGESTOR, n_objectives, rng=rng) + self.suggestor = suggestor + self._suggested_experiments: list[list] = [] + self.Xi: list[list] = [] + # This is a list of points in the search space. Each point is a list of values for + # each dimension of the search space. + self.yi: list = [] + # We are agnostic to the type of the scores. They can be a float for single + # objective optimization or a list of floats for multiobjective optimization. + pass + + def ask(self, n: int = 1): + if n > 1 and not isinstance(self.suggestor, BatchSuggestor): + warnings.warn( + "No batch suggestor is defined. Asking for multiple suggestions might not work." + ) + if len(self._suggested_experiments) < n: + if isinstance(self.suggestor, BatchSuggestor): + self.suggestor.n_given = n - len(self._suggested_experiments) + self._suggested_experiments.extend(self.suggestor.suggest(self.Xi, self.yi)) + if len(self._suggested_experiments) < n: + raise ValueError("The suggestor did not return enough suggestions.") + return self._suggested_experiments[:n] + + def tell(self, x: list, y: Any): + self.Xi.append(x) + self.yi.append(y) + if len(x) > len(self._suggested_experiments): + self._suggested_experiments = [] + else: + self._suggested_experiments = self._suggested_experiments[len(x):] diff --git a/Expyrementor/suggestors/__init__.py b/Expyrementor/suggestors/__init__.py new file mode 100644 index 00000000..2c664fff --- /dev/null +++ b/Expyrementor/suggestors/__init__.py @@ -0,0 +1,18 @@ +from .default_suggestor import DefaultSuggestor +from .initial_points_suggestor import InitialPointSuggestor +from .po_suggestor import POSuggestor +from .random_strategizer import RandomStragegizer +from .suggestor import BatchSuggestor, Suggestor +from .strategizer import Strategizer +from .suggestor_factory import suggestor_factory + +__all__ = [ + "BatchSuggestor", + "DefaultSuggestor", + "InitialPointSuggestor", + "POSuggestor", + "RandomStragegizer", + "Strategizer", + "Suggestor", + "suggestor_factory", +] diff --git a/Expyrementor/suggestors/default_suggestor.py b/Expyrementor/suggestors/default_suggestor.py new file mode 100644 index 00000000..882d48e0 --- /dev/null +++ b/Expyrementor/suggestors/default_suggestor.py @@ -0,0 +1,24 @@ +import numpy as np +from ProcessOptimizer.space import Space + +from .suggestor import BatchSuggestor + + +class DefaultSuggestor(BatchSuggestor): + """ + Default suggestor class. It should only be used as a placeholder for use in + strategizers. It should be replaced with the appropriate suggestor before use. + """ + + def __init__(self, space: Space, rng: np.random.Generator): + # Space and random number generator are stored for use when replacing the + # DefaultSuggestor. + self.space = space + self.rng = rng + + def suggest(self, _, __): + raise NoDefaultSuggestorError("Default suggestor should not be used.") + + +class NoDefaultSuggestorError(NotImplementedError): + """ Raised when a DefaultSuggestor is used when it should have been replaced.""" diff --git a/Expyrementor/suggestors/initial_points_suggestor.py b/Expyrementor/suggestors/initial_points_suggestor.py new file mode 100644 index 00000000..7f1fdf01 --- /dev/null +++ b/Expyrementor/suggestors/initial_points_suggestor.py @@ -0,0 +1,80 @@ +import logging + +from .default_suggestor import DefaultSuggestor +from .po_suggestor import POSuggestor +from .suggestor import Suggestor +from .strategizer import Strategizer + +logger = logging.getLogger(__name__) + + +class InitialPointSuggestor(Strategizer): + """ + A strategizer that uses one suggestor for a fixed number of initial points and then + switches to another suggestor. + + Be careful when using a suggestor that suggest multiple points at once as the + inital suggestor. If it suggests more points than the number of remaining initial + points, its suggestions will be truncated so it only returns the remaining initial + points. This can be a problem with design of experiments suggestors, for example, as + all of their suggested points are needed for the analysis. + """ + def __init__( + self, + initial_suggestor: Suggestor, + ultimate_suggestor: Suggestor, + n_initial_points: int = 5, + ): + """ + Parameters + ---------- + initial_suggestor : Suggestor + The suggestor to use for the initial points. Default is a POSuggestor with + n_initial_points initial points. + ultimate_suggestor : Suggestor + The suggestor to use after the initial points have been suggested. Default + is a POSuggestor with no initial points. + n_initial_points : int + The number of initial points to suggest with the initial suggestor. Default + is 5. + """ + if isinstance(initial_suggestor, DefaultSuggestor): + # If the initial suggestor is the default suggestor, we replace it with a + # POSuggestor. It should be a much simpler suggestor, e.g. a Latin Hypercube + # Sampling suggestor. Replace the POSuggestor when available. + logger.debug( + "Initial suggestor is DefaultSuggestor, replacing with POSuggestor" + ) + initial_suggestor = POSuggestor( + initial_suggestor.space, + n_initial_points=n_initial_points, + rng=initial_suggestor.rng, + ) + if isinstance(ultimate_suggestor, DefaultSuggestor): + logger.debug( + "Ultimate suggestor is DefaultSuggestor, replacing with POSuggestor" + ) + ultimate_suggestor = POSuggestor( + ultimate_suggestor.space, n_initial_points=0, rng=ultimate_suggestor.rng + ) + self.n_initial_points = n_initial_points + self.initial_suggestor = initial_suggestor + self.ultimate_suggestor = ultimate_suggestor + + def next_suggestor(self, Xi, yi): + remaining_initial_points = self.n_initial_points - len(Xi) + if remaining_initial_points > 0: + logger.debug("Only %s points told, using initial suggestor", len(Xi)) + suggestion = self.initial_suggestor.suggest(Xi, yi) + if len(suggestion) > remaining_initial_points: + raise ValueError( + "Initial suggestor suggested %s points, but since %s points have" + "already been told, it should suggest at most %s points", + len(suggestion), + len(Xi), + remaining_initial_points, + ) + return self.initial_suggestor + else: + logger.debug("Using ultimate suggestor") + return self.ultimate_suggestor diff --git a/Expyrementor/suggestors/po_suggestor.py b/Expyrementor/suggestors/po_suggestor.py new file mode 100644 index 00000000..206d1396 --- /dev/null +++ b/Expyrementor/suggestors/po_suggestor.py @@ -0,0 +1,32 @@ +import logging + +import numpy as np +from ProcessOptimizer import Optimizer +from ProcessOptimizer.space import Space + +logger = logging.getLogger(__name__) + + +class POSuggestor: + def __init__(self, space: Space, rng: np.random.Generator, **kwargs): + self.optimizer = Optimizer( + dimensions=space, + random_state=np.random.RandomState(int(rng.random() * (2**32 - 1))), + # ProcessOptimizer uses a legacy random state object, so we need to convert + # the numpy random generator to a numpy random state object. + **kwargs, + ) + + def suggest(self, Xi: list[list], yi: list) -> list[list]: + if Xi != self.optimizer.Xi or yi != self.optimizer.yi: + self.optimizer.Xi = Xi + self.optimizer.yi = yi + self.optimizer.update_next() + point = self.optimizer.ask() + logger.debug( + "Given Xi = %s and yi = %s, POSugggestor suggests the point: %s", + Xi, + yi, + point, + ) + return [point] diff --git a/Expyrementor/suggestors/random_strategizer.py b/Expyrementor/suggestors/random_strategizer.py new file mode 100644 index 00000000..bb4a2cc0 --- /dev/null +++ b/Expyrementor/suggestors/random_strategizer.py @@ -0,0 +1,33 @@ +import warnings + +import numpy as np + +from .default_suggestor import DefaultSuggestor, NoDefaultSuggestorError +from .suggestor import Suggestor +from .strategizer import Strategizer + + +class RandomStragegizer(Strategizer): + def __init__( + self, suggestors: list[tuple[float, Suggestor]], n_objectives, rng: np.random.Generator + ): + self.total = sum(item[0] for item in suggestors) + if float(self.total) != 1.0 and float(self.total) != 100.0: + warnings.warn( + "Probabilities do not sum to 1.0 or 100.0. They will be normalized." + ) + if any( + isinstance(item[1], DefaultSuggestor) for item in suggestors + ): + raise NoDefaultSuggestorError( + "No DefaultSuggestor defined for RandomStrategizer." + ) + self.suggestors = suggestors + self.rng = rng + + def next_suggestor(self, _, __) -> Suggestor: + random_number = self.rng.random()*self.total + for weight, suggestor in self.suggestors: + if random_number < weight: + return suggestor + random_number -= weight diff --git a/Expyrementor/suggestors/strategizer.py b/Expyrementor/suggestors/strategizer.py new file mode 100644 index 00000000..ee7418a5 --- /dev/null +++ b/Expyrementor/suggestors/strategizer.py @@ -0,0 +1,30 @@ +import logging +from abc import ABC, abstractmethod + +from .suggestor import Suggestor + +logger = logging.getLogger(__name__) + + +# Strategizer is nearly superfluous, but having it done the same way helps with +# readability, and we also ensure that the logging is consistent. +class Strategizer(ABC): + """ + A strategizer is a suggestor that uses other suggestors to suggest points. + """ + @abstractmethod + def next_suggestor(self, Xi: list[list], Yi: list) -> Suggestor: + """ Select the next suggestor to use.""" + pass + + def suggest(self, Xi: list[list], Yi: list) -> list[list]: + suggestor = self.next_suggestor(Xi, Yi) + suggestion = suggestor.suggest(Xi, Yi) + logger.debug( + "Strategizer of type %s is using suggestor %s of type %s. It suggested: %s", + type(self), + suggestor, + type(suggestor), + suggestion, + ) + return suggestion diff --git a/Expyrementor/suggestors/suggestor.py b/Expyrementor/suggestors/suggestor.py new file mode 100644 index 00000000..e8c511c2 --- /dev/null +++ b/Expyrementor/suggestors/suggestor.py @@ -0,0 +1,38 @@ +from typing import Protocol, runtime_checkable + +import numpy as np +from ProcessOptimizer.space import Space + + +@runtime_checkable # Need to be runtime checkable for the factory to work +class Suggestor(Protocol): + """ + Protocol for suggestors. Suggestors are used to suggest new points to evaluate in the + optimization process. Suggestors should be stateless and only depend on the search + space and the already evaluated points. In particular, consecutive calls to the + suggest method with the same input should ideally return the same output, or at least + output the same number of points. + """ + def __init__(self, space: Space, n_objectives: int, rng: np.random.Generator, **kwargs): + """ + Initialize the suggestor with the search space. Suggestors can take other input + arguments as needed. + """ + pass + + def suggest(self, Xi: list[list], Yi: list) -> list[list]: + """ + Suggest a new point to evaluate. The input is a list of already evaluated points + and their corresponding scores. The output is a list of new points to evaluate. + The list can have the length of 1 or more. + """ + pass + + +class BatchSuggestor(Suggestor): + """ + Protocol for batch suggestors. Batch suggestors are suggestors that can be told how + many suggestions to make in a single call. This will usually be something building on + top of a regular suggestor, like constant liers or Krigging belivers. + """ + n_given: int diff --git a/Expyrementor/suggestors/suggestor_factory.py b/Expyrementor/suggestors/suggestor_factory.py new file mode 100644 index 00000000..6988a03d --- /dev/null +++ b/Expyrementor/suggestors/suggestor_factory.py @@ -0,0 +1,90 @@ +import logging +from typing import Any, Union + +import numpy as np +from ProcessOptimizer.space import Space + +from .default_suggestor import DefaultSuggestor +from .initial_points_suggestor import InitialPointSuggestor +from .po_suggestor import POSuggestor +from .random_strategizer import RandomStragegizer +from .suggestor import Suggestor + +logger = logging.getLogger(__name__) + + +def suggestor_factory( + space: Space, + definition: Union[Suggestor, dict[str, Any], None], + n_objectives: int, + rng: np.random.Generator, +) -> Suggestor: + """ + Create a suggestor from a definition dictionary. + + Definition is either a suggestor instance, a dict that specifies the suggestor type + and its parameters, or None. + + If definiton is a suggestor instance it is returned as is. + + If definition is a dict, it is used to create a suggestor. The dictionary must have + a 'name' key that specifies the type of suggestor. The other keys depend on the + suggestor type. It can be recursive if the suggestor is a strategizer, that is, a + suggestor that uses other suggestors. + + If definition is None, a DefaultSuggestor is created. This is useful as a + placeholder in strategizers, and should be replaced with a real suggestor before + use. + """ + if isinstance(definition, Suggestor): + return definition + elif not definition: # If definition is None or empty, return DefaultSuggestor. + logger.debug("Creating DefaultSuggestor") + return DefaultSuggestor(space, rng) + try: + suggestor_type = definition.pop("name") + except KeyError as e: + raise ValueError( + f"Missing 'name' key in suggestor definition: {definition}" + ) from e + if suggestor_type == "Default" or suggestor_type is None: + logger.debug("Creating DefaultSuggestor") + return DefaultSuggestor(space, rng) + elif suggestor_type == "InitialPoint": + logger.debug("Creating InitialPointSuggestor") + # If either of the necessary keys are missing, the default values are used. + initial_suggestor = definition.get("initial_suggestor", None) + ultimate_suggestor = definition.get("ultimate_suggestor", None) + return InitialPointSuggestor( + initial_suggestor=suggestor_factory( + space, initial_suggestor, n_objectives, rng + ), + ultimate_suggestor=suggestor_factory( + space, ultimate_suggestor, n_objectives, rng + ), + n_initial_points=definition["n_initial_points"], + ) + elif suggestor_type == "PO": + logger.debug("Creating POSuggestor") + return POSuggestor( + space=space, + rng=rng, + n_objectives=n_objectives, + **definition, + ) + elif suggestor_type == "RandomStrategizer" or suggestor_type == "Random": + logger.debug("Creating RandomStrategizer") + suggestors = [] + for suggestor in definition["suggestors"]: + usage_ratio = suggestor.pop("usage_ratio") + # Note that we are removing the key usage_ratio from the suggestor + # definition. If any suggestor uses this key, it will have to be redefined in + # the suggestor definition. + suggestors.append(( + usage_ratio, suggestor_factory(space, suggestor, n_objectives, rng) + )) + return RandomStragegizer( + suggestors=suggestors, n_objectives=n_objectives, rng=rng + ) + else: + raise ValueError(f"Unknown suggestor name: {suggestor_type}") diff --git a/ProcessOptimizer/tests/test_expyrementor.py b/ProcessOptimizer/tests/test_expyrementor.py new file mode 100644 index 00000000..9a7e7124 --- /dev/null +++ b/ProcessOptimizer/tests/test_expyrementor.py @@ -0,0 +1,81 @@ +from Expyrementor.expyrementor import Expyrementor +from Expyrementor.suggestors import InitialPointSuggestor, POSuggestor + + +class MockSuggestor: + def __init__(self, suggestions: list): + self.suggestions = suggestions + self.last_input = {} + + def suggest(self, Xi, yi): + self.last_input = {"Xi": Xi, "yi": yi} + return self.suggestions + + +def test_initialization(): + space = [[0, 1], [0, 1]] + exp = Expyrementor(space) + assert exp.Xi == [] + assert exp.yi == [] + assert isinstance(exp.suggestor, InitialPointSuggestor) + assert exp.suggestor.n_initial_points == 5 + assert isinstance(exp.suggestor.initial_suggestor, POSuggestor) + assert exp.suggestor.initial_suggestor.optimizer._n_initial_points >= 5 + assert isinstance(exp.suggestor.ultimate_suggestor, POSuggestor) + assert exp.suggestor.ultimate_suggestor.optimizer._n_initial_points == 0 + + +def test_tell_single_objective(): + space = [[0, 1], [0, 1]] + exp = Expyrementor(space) + exp.tell([0.5, 0.5], 1) + assert exp.Xi == [[0.5, 0.5]] + assert exp.yi == [1] + exp.tell([0.6, 0.6], 2) + assert exp.Xi == [[0.5, 0.5], [0.6, 0.6]] + assert exp.yi == [1, 2] + + +def test_tell_multiple_objectives(): + space = [[0, 1], [0, 1]] + exp = Expyrementor(space) + exp.tell([0.5, 0.5], [1, 2]) + assert exp.Xi == [[0.5, 0.5]] + assert exp.yi == [[1, 2]] + exp.tell([0.6, 0.6], [2, 3]) + assert exp.Xi == [[0.5, 0.5], [0.6, 0.6]] + assert exp.yi == [[1, 2], [2, 3]] + + +def test_ask_single_return(): + space = [[0, 1], [0, 1]] + exp = Expyrementor(space) + exp.suggestor = MockSuggestor([[0.5, 0.5]]) + assert exp.ask() == [0.5, 0.5] + + +def test_ask_multiple_returns(): + space = [[0, 1], [0, 1]] + exp = Expyrementor(space) + exp.suggestor = MockSuggestor([[0.5, 0.5], [0.6, 0.6]]) + # exp will now get two suggestions from the suggestor, and only return the first one + assert exp.ask() == [0.5, 0.5] + # We will now replace the suggestor + exp.suggestor = MockSuggestor([[0.7, 0.7]]) + # exp will now return the second suggestion from the first suggestor + assert exp.ask() == [0.6, 0.6] + # exp has now used all of the suggestions from the first suggestor, and will get a + # new suggestion from the second suggestor + assert exp.ask() == [0.7, 0.7] + + +def test_ask_passes_on_values(): + space = [[0, 1], [0, 1]] + exp = Expyrementor(space) + exp.suggestor = MockSuggestor([[0.5, 0.5]]) + exp.tell([0.6, 0.6], 2) + exp.ask() + assert exp.suggestor.last_input == {"Xi": [[0.6, 0.6]], "yi": [2]} + exp.tell([0.7, 0.7], 3) + exp.ask() + assert exp.suggestor.last_input == {"Xi": [[0.6, 0.6], [0.7, 0.7]], "yi": [2, 3]} diff --git a/ProcessOptimizer/tests/test_suggestors/__init__.py b/ProcessOptimizer/tests/test_suggestors/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py new file mode 100644 index 00000000..7dfd05d8 --- /dev/null +++ b/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py @@ -0,0 +1,51 @@ +import numpy as np +import pytest +from Expyrementor.suggestors import InitialPointSuggestor, DefaultSuggestor, POSuggestor + + +class MockSuggestor: + def __init__(self, suggestions: list): + self.suggestions = suggestions + self.last_input = {} + + def suggest(self, Xi, yi): + self.last_input = {"Xi": Xi, "yi": yi} + return self.suggestions + + +def test_initialization(): + space = [[0, 1], [0, 1]] + suggestor = InitialPointSuggestor( + initial_suggestor=DefaultSuggestor(space, rng=np.random.default_rng(1)), + ultimate_suggestor=DefaultSuggestor(space, rng=np.random.default_rng(2)), + ) + assert suggestor.n_initial_points == 5 + # Initial suggestor is a POSuggestor with more than 5 initial points. This means than + # we will only use the LHS part of this ProcessOptimizer. + assert isinstance(suggestor.initial_suggestor, POSuggestor) + assert suggestor.initial_suggestor.optimizer._n_initial_points == 5 + # Ultimate suggestor is a POSuggestor with no initial points, since + # InitialPointSuggestor handles the initial points. + assert isinstance(suggestor.ultimate_suggestor, POSuggestor) + assert suggestor.ultimate_suggestor.optimizer._n_initial_points == 0 + + +def test_suggestor_switch(): + suggestor = InitialPointSuggestor( + initial_suggestor=MockSuggestor([1]), + ultimate_suggestor=MockSuggestor([2]), + n_initial_points=3, + ) + assert suggestor.next_suggestor([], []) == suggestor.initial_suggestor + assert suggestor.next_suggestor([1, 2], []) == suggestor.initial_suggestor + assert suggestor.next_suggestor([1, 2, 3], []) == suggestor.ultimate_suggestor + + +def test_too_may_suggested_point(): + suggestor = InitialPointSuggestor( + initial_suggestor=MockSuggestor([1, 2]), + ultimate_suggestor=MockSuggestor([4]), + n_initial_points=3, + ) + with pytest.raises(ValueError): + suggestor.next_suggestor([1, 2], []) diff --git a/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py new file mode 100644 index 00000000..3baf8c14 --- /dev/null +++ b/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py @@ -0,0 +1,28 @@ +import numpy as np +import pytest +from Expyrementor.suggestors import RandomStragegizer, Suggestor + + +class MockSuggestor: + def __init__(self, suggestions: list): + self.suggestions = suggestions + self.last_input = {} + + def suggest(self, Xi, yi): + self.last_input = {"Xi": Xi, "yi": yi} + return self.suggestions + + +def test_random_strategizer(): + suggestor = RandomStragegizer( + suggestors=[(0.8, MockSuggestor([])), (0.2, MockSuggestor([]))], + n_objectives=1, + rng=np.random.default_rng(1) + ) + assert isinstance(suggestor, Suggestor) + # np.random.default_rng(1).random() gives 0.5118216247148916, 0.9504636963259353, + # and 0.14415961271963373 on the first three calls, so the first three calls + # should return the suggestors with weights 0.8, 0.2, and 0.8, respectively. + assert suggestor.next_suggestor([], []) == suggestor.suggestors[0][1] + assert suggestor.next_suggestor([], []) == suggestor.suggestors[1][1] + assert suggestor.next_suggestor([], []) == suggestor.suggestors[0][1] diff --git a/setup.py b/setup.py index 92762d2e..b203233d 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,8 @@ "ProcessOptimizer.space", "ProcessOptimizer.utils", "ProcessOptimizer.learning.gaussian_process", + "Expyrementor", + "Expyrementor.suggestors", ], install_requires=[ "numpy", From 8eab1da9991c6648c05756bf84cc037a83d13667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Tue, 3 Dec 2024 13:49:23 +0100 Subject: [PATCH 02/30] refactor: Moved caching suggested points to CachingStrategizer --- Expyrementor/__init__.py | 3 + Expyrementor/expyrementor.py | 48 +++++++------- Expyrementor/suggestors/__init__.py | 14 ++-- .../suggestors/caching_strategizer.py | 56 ++++++++++++++++ Expyrementor/suggestors/default_suggestor.py | 9 ++- ...estor.py => initial_points_strategizer.py} | 45 ++++++------- Expyrementor/suggestors/lhs_suggestor.py | 34 ++++++++++ Expyrementor/suggestors/po_suggestor.py | 9 +-- Expyrementor/suggestors/random_strategizer.py | 27 +++++--- Expyrementor/suggestors/strategizer.py | 30 --------- Expyrementor/suggestors/suggestor.py | 15 ++--- Expyrementor/suggestors/suggestor_factory.py | 22 +++++-- ProcessOptimizer/optimizer/optimizer.py | 1 + ProcessOptimizer/tests/test_expyrementor.py | 34 ++++++---- .../test_caching_strategizer.py | 60 +++++++++++++++++ .../test_initial_point_suggestor.py | 52 ++++++++------- .../test_suggestors/test_lhs_suggestor.py | 17 +++++ .../test_suggestors/test_random_suggestor.py | 25 +++++--- examples/expyrementor.ipynb | 64 +++++++++++++++++++ 19 files changed, 405 insertions(+), 160 deletions(-) create mode 100644 Expyrementor/suggestors/caching_strategizer.py rename Expyrementor/suggestors/{initial_points_suggestor.py => initial_points_strategizer.py} (68%) create mode 100644 Expyrementor/suggestors/lhs_suggestor.py delete mode 100644 Expyrementor/suggestors/strategizer.py create mode 100644 ProcessOptimizer/tests/test_suggestors/test_caching_strategizer.py create mode 100644 ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py create mode 100644 examples/expyrementor.ipynb diff --git a/Expyrementor/__init__.py b/Expyrementor/__init__.py index e69de29b..f4f104e5 100644 --- a/Expyrementor/__init__.py +++ b/Expyrementor/__init__.py @@ -0,0 +1,3 @@ +from .expyrementor import Expyrementor + +__all__ = ["Expyrementor"] diff --git a/Expyrementor/expyrementor.py b/Expyrementor/expyrementor.py index 97bec468..6a40cf66 100644 --- a/Expyrementor/expyrementor.py +++ b/Expyrementor/expyrementor.py @@ -1,19 +1,19 @@ import logging -import warnings from typing import Any, Union import numpy as np from ProcessOptimizer.space import space_factory, Space +from ProcessOptimizer.utils import is_2Dlistlike from ProcessOptimizer.utils.get_rng import get_random_generator -from .suggestors import BatchSuggestor, DefaultSuggestor, Suggestor, suggestor_factory +from .suggestors import DefaultSuggestor, Suggestor, suggestor_factory logger = logging.getLogger(__name__) DEFAULT_SUGGESTOR = { - "type": "InitialPoint", - "initial_suggestor": {"type": "Default"}, - "ultimate_suggestor": {"type": "Default"}, + "name": "InitialPoint", + "initial_suggestor": {"name": "Default"}, + "ultimate_suggestor": {"name": "Default"}, "n_initial_points": 5 } @@ -44,9 +44,10 @@ def __init__( suggestor = suggestor_factory(space, suggestor, n_objectives, rng) if isinstance(suggestor, DefaultSuggestor): logger.debug("Replacing DefaultSuggestor with InitialPointSuggestor") - suggestor = suggestor_factory(space, DEFAULT_SUGGESTOR, n_objectives, rng=rng) + suggestor = suggestor_factory( + space, DEFAULT_SUGGESTOR.copy(), n_objectives, rng=rng + ) self.suggestor = suggestor - self._suggested_experiments: list[list] = [] self.Xi: list[list] = [] # This is a list of points in the search space. Each point is a list of values for # each dimension of the search space. @@ -56,22 +57,23 @@ def __init__( pass def ask(self, n: int = 1): - if n > 1 and not isinstance(self.suggestor, BatchSuggestor): - warnings.warn( - "No batch suggestor is defined. Asking for multiple suggestions might not work." - ) - if len(self._suggested_experiments) < n: - if isinstance(self.suggestor, BatchSuggestor): - self.suggestor.n_given = n - len(self._suggested_experiments) - self._suggested_experiments.extend(self.suggestor.suggest(self.Xi, self.yi)) - if len(self._suggested_experiments) < n: - raise ValueError("The suggestor did not return enough suggestions.") - return self._suggested_experiments[:n] + """ + Ask the suggestor for new points to evaluate. The number of points to ask is + specified by the argument n. The method returns a list of new points to evaluate. + """ + return self.suggestor.suggest(Xi=self.Xi, Yi=self.yi, n_asked=n) def tell(self, x: list, y: Any): - self.Xi.append(x) - self.yi.append(y) - if len(x) > len(self._suggested_experiments): - self._suggested_experiments = [] + if is_2Dlistlike(x): + # If x is a list of points, we assume that y is a list of scores of the same + # length, and we add the members of x and y to the lists Xi and yi. + self.Xi.extend(x) + self.yi.extend(y) else: - self._suggested_experiments = self._suggested_experiments[len(x):] + # If x is a single point, we assume that y is a single score, and we add x + # and y to the lists Xi and yi. + self.Xi.append(x) + self.yi.append(y) + + def __str__(self): + return f"Expyrementor with suggestor {self.suggestor}" diff --git a/Expyrementor/suggestors/__init__.py b/Expyrementor/suggestors/__init__.py index 2c664fff..5925ce8e 100644 --- a/Expyrementor/suggestors/__init__.py +++ b/Expyrementor/suggestors/__init__.py @@ -1,18 +1,20 @@ +from .caching_strategizer import CachingStrategizer from .default_suggestor import DefaultSuggestor -from .initial_points_suggestor import InitialPointSuggestor +from .initial_points_strategizer import InitialPointStrategizer +from .lhs_suggestor import LHSSuggestor from .po_suggestor import POSuggestor from .random_strategizer import RandomStragegizer -from .suggestor import BatchSuggestor, Suggestor -from .strategizer import Strategizer +from .suggestor import IncompatibleNumberAsked, Suggestor from .suggestor_factory import suggestor_factory __all__ = [ - "BatchSuggestor", + "CachingStrategizer", "DefaultSuggestor", - "InitialPointSuggestor", + "IncompatibleNumberAsked", + "InitialPointStrategizer", + "LHSSuggestor", "POSuggestor", "RandomStragegizer", - "Strategizer", "Suggestor", "suggestor_factory", ] diff --git a/Expyrementor/suggestors/caching_strategizer.py b/Expyrementor/suggestors/caching_strategizer.py new file mode 100644 index 00000000..0bacd3ee --- /dev/null +++ b/Expyrementor/suggestors/caching_strategizer.py @@ -0,0 +1,56 @@ +from .default_suggestor import DefaultSuggestor +from .suggestor import Suggestor, IncompatibleNumberAsked + + +class CachingStrategizer(): + """ + Cache the suggestions of a suggestor and return them when asked for new points. + + The caching strategizer will ask the suggestor for new points a fixed number of times + and store the suggestions. When asked for new points, it will return the stored + suggestions until they run out, at which point it will ask the suggestor for new + points again. + + The caching strategizer can be used to convert a suggestor that suggests a fixed + number of points into a suggestor that can suggest a variable number of points. + Examples would be fitting an LHS suggestor into an iterative optimization process. + """ + def __init__(self, suggestor: Suggestor, ask_times: int = 1, **_): + """ + Initialize the caching strategizer with the suggestor to cache and the number of + times to ask the suggestor for new points. + + Parameters + ---------- + suggestor : Suggestor + The suggestor to cache. + ask_times : int, optional + The number of times to ask the suggestor for new points, by default 1. + If it is smaller than 0, the suggestor will be asked for new points + indefinitely. + """ + if isinstance(suggestor, DefaultSuggestor): + raise ValueError("CachingStrategizer has no default suggestor defined.") + self.suggestor = suggestor + self.ask_times_left = ask_times + self.cache = [] + + def suggest(self, Xi: list[list], Yi: list, n_asked: int = 1) -> list[list]: + suggested_points = [] + while len(suggested_points) < n_asked: + if n_asked <= len(self.cache): + n_points_from_cache = n_asked - len(suggested_points) + suggested_points.extend(self.cache[:n_points_from_cache]) + self.cache = self.cache[n_points_from_cache:] + else: + if self.ask_times_left != 0: # Not <= 0 to allow for infinite asking + # Use the exisiting cache, ask the suggestor for new points, put + # them in the cache, and decrement the number of times left to ask. + suggested_points.extend(self.cache) + self.cache = self.suggestor.suggest(Xi, Yi, -1) + self.ask_times_left -= 1 + else: + raise IncompatibleNumberAsked( + "CachingStrategizer ran out of ask times." + ) + return suggested_points diff --git a/Expyrementor/suggestors/default_suggestor.py b/Expyrementor/suggestors/default_suggestor.py index 882d48e0..e603d618 100644 --- a/Expyrementor/suggestors/default_suggestor.py +++ b/Expyrementor/suggestors/default_suggestor.py @@ -1,22 +1,21 @@ import numpy as np from ProcessOptimizer.space import Space -from .suggestor import BatchSuggestor - -class DefaultSuggestor(BatchSuggestor): +class DefaultSuggestor(): """ Default suggestor class. It should only be used as a placeholder for use in strategizers. It should be replaced with the appropriate suggestor before use. """ - def __init__(self, space: Space, rng: np.random.Generator): + def __init__(self, space: Space, n_objectives: int, rng: np.random.Generator, **kwargs): # Space and random number generator are stored for use when replacing the # DefaultSuggestor. self.space = space + self.n_objectives = n_objectives self.rng = rng - def suggest(self, _, __): + def suggest(self, Xi: list[list], Yi: list, n_asked: int = -1) -> list[list]: raise NoDefaultSuggestorError("Default suggestor should not be used.") diff --git a/Expyrementor/suggestors/initial_points_suggestor.py b/Expyrementor/suggestors/initial_points_strategizer.py similarity index 68% rename from Expyrementor/suggestors/initial_points_suggestor.py rename to Expyrementor/suggestors/initial_points_strategizer.py index 7f1fdf01..a10a9199 100644 --- a/Expyrementor/suggestors/initial_points_suggestor.py +++ b/Expyrementor/suggestors/initial_points_strategizer.py @@ -1,14 +1,15 @@ import logging +from .caching_strategizer import CachingStrategizer from .default_suggestor import DefaultSuggestor +from .lhs_suggestor import LHSSuggestor from .po_suggestor import POSuggestor from .suggestor import Suggestor -from .strategizer import Strategizer logger = logging.getLogger(__name__) -class InitialPointSuggestor(Strategizer): +class InitialPointStrategizer(): """ A strategizer that uses one suggestor for a fixed number of initial points and then switches to another suggestor. @@ -43,16 +44,18 @@ def __init__( # POSuggestor. It should be a much simpler suggestor, e.g. a Latin Hypercube # Sampling suggestor. Replace the POSuggestor when available. logger.debug( - "Initial suggestor is DefaultSuggestor, replacing with POSuggestor" + "Initial suggestor is DefaultSuggestor, replacing with " + "cached LHSSuggestor." ) - initial_suggestor = POSuggestor( - initial_suggestor.space, - n_initial_points=n_initial_points, + lhs_suggestor = LHSSuggestor( + space=initial_suggestor.space, rng=initial_suggestor.rng, + n_points=n_initial_points, ) + initial_suggestor = CachingStrategizer(lhs_suggestor) if isinstance(ultimate_suggestor, DefaultSuggestor): logger.debug( - "Ultimate suggestor is DefaultSuggestor, replacing with POSuggestor" + "Ultimate suggestor is DefaultSuggestor, replacing with POSuggestor." ) ultimate_suggestor = POSuggestor( ultimate_suggestor.space, n_initial_points=0, rng=ultimate_suggestor.rng @@ -61,20 +64,14 @@ def __init__( self.initial_suggestor = initial_suggestor self.ultimate_suggestor = ultimate_suggestor - def next_suggestor(self, Xi, yi): - remaining_initial_points = self.n_initial_points - len(Xi) - if remaining_initial_points > 0: - logger.debug("Only %s points told, using initial suggestor", len(Xi)) - suggestion = self.initial_suggestor.suggest(Xi, yi) - if len(suggestion) > remaining_initial_points: - raise ValueError( - "Initial suggestor suggested %s points, but since %s points have" - "already been told, it should suggest at most %s points", - len(suggestion), - len(Xi), - remaining_initial_points, - ) - return self.initial_suggestor - else: - logger.debug("Using ultimate suggestor") - return self.ultimate_suggestor + def suggest(self, Xi: list[list], Yi: list, n_asked: int = 1) -> list[list]: + n_initial_points = max(self.n_initial_points - len(Xi), 0) + n_ultimate_points = n_asked - n_initial_points + suggestions = [] + if n_initial_points > 0: + suggestions.extend(self.initial_suggestor.suggest(Xi, Yi, n_initial_points)) + if n_ultimate_points > 0: + suggestions.extend(self.ultimate_suggestor.suggest( + Xi, Yi, n_ultimate_points) + ) + return suggestions diff --git a/Expyrementor/suggestors/lhs_suggestor.py b/Expyrementor/suggestors/lhs_suggestor.py new file mode 100644 index 00000000..18216636 --- /dev/null +++ b/Expyrementor/suggestors/lhs_suggestor.py @@ -0,0 +1,34 @@ +import numpy as np +from ProcessOptimizer.space import Space + + +class LHSSuggestor(): + def __init__(self, space: Space, rng: np.random.Generator, n_points: int = 5): + self.space = space + self.rng = rng + self.n_points = n_points + + def suggest(self, Xi: list[list], Yi: list, n_asked: int = -1) -> list[list]: + if n_asked == -1: + n_asked = self.n_points + if n_asked != self.n_points: + raise ValueError( + f"LHSSuggestor can only provide {self.n_points} points. If you need " + "them split up, wrap it in a CachingStrategizer." + ) + samples = [] + # Create a list of evenly distributed points in the range [0, 1] to sample from + sample_indices = (np.arange(n_asked) + 0.5) / n_asked + for i in range(self.space.n_dims): + # Sample the points in the ith dimension + lhs_aranged = self.space.dimensions[i].sample(sample_indices) + # Shuffle the points in the ith dimension + samples.append([lhs_aranged[p] for p in self.rng.permutation(n_asked)]) + # Now we have a list of lists where each inner list is all the points in one + # dimension in random order. We need to transpose this so that we get a list of + # points in the space, where each point is a list of values from each dimension. + transposed_samples = [] + for i in range(n_asked): + row = [samples[j][i] for j in range(self.space.n_dims)] + transposed_samples.append(row) + return transposed_samples diff --git a/Expyrementor/suggestors/po_suggestor.py b/Expyrementor/suggestors/po_suggestor.py index 206d1396..c279e1bc 100644 --- a/Expyrementor/suggestors/po_suggestor.py +++ b/Expyrementor/suggestors/po_suggestor.py @@ -16,13 +16,14 @@ def __init__(self, space: Space, rng: np.random.Generator, **kwargs): # the numpy random generator to a numpy random state object. **kwargs, ) + self.n_given = 1 - def suggest(self, Xi: list[list], yi: list) -> list[list]: + def suggest(self, Xi: list[list], yi: list, n_asked: int = 1) -> list[list]: if Xi != self.optimizer.Xi or yi != self.optimizer.yi: - self.optimizer.Xi = Xi - self.optimizer.yi = yi + self.optimizer.Xi = Xi.copy() + self.optimizer.yi = yi.copy() self.optimizer.update_next() - point = self.optimizer.ask() + point = self.optimizer.ask(n_asked) logger.debug( "Given Xi = %s and yi = %s, POSugggestor suggests the point: %s", Xi, diff --git a/Expyrementor/suggestors/random_strategizer.py b/Expyrementor/suggestors/random_strategizer.py index bb4a2cc0..43a659b4 100644 --- a/Expyrementor/suggestors/random_strategizer.py +++ b/Expyrementor/suggestors/random_strategizer.py @@ -4,13 +4,12 @@ from .default_suggestor import DefaultSuggestor, NoDefaultSuggestorError from .suggestor import Suggestor -from .strategizer import Strategizer -class RandomStragegizer(Strategizer): +class RandomStragegizer(): def __init__( self, suggestors: list[tuple[float, Suggestor]], n_objectives, rng: np.random.Generator - ): + ): self.total = sum(item[0] for item in suggestors) if float(self.total) != 1.0 and float(self.total) != 100.0: warnings.warn( @@ -25,9 +24,21 @@ def __init__( self.suggestors = suggestors self.rng = rng - def next_suggestor(self, _, __) -> Suggestor: - random_number = self.rng.random()*self.total + def suggest(self, Xi: list[list], Yi: list, n_asked: int = 1) -> list[list]: + # Creating n_asked random indices in the range [0, total) + selector_indices = [ + relative_index*self.total for relative_index in self.rng.random(size=n_asked) + ] + suggested_points = [] + # Iterating over the suggestors, finding the number of suggested points for each + # suggestor, and suggesting them. + # Note that this will order the points in the order of the suggestors, the + # suggestion will start with points from the first suggestor, then the second, + # etc. for weight, suggestor in self.suggestors: - if random_number < weight: - return suggestor - random_number -= weight + selector_indices = [index-weight for index in selector_indices] + n_suggested = sum(index < 0 for index in selector_indices) + if n_suggested > 0: + suggested_points.extend(suggestor.suggest(Xi, Yi, n_suggested)) + selector_indices = [index for index in selector_indices if index >= 0] + return suggested_points diff --git a/Expyrementor/suggestors/strategizer.py b/Expyrementor/suggestors/strategizer.py deleted file mode 100644 index ee7418a5..00000000 --- a/Expyrementor/suggestors/strategizer.py +++ /dev/null @@ -1,30 +0,0 @@ -import logging -from abc import ABC, abstractmethod - -from .suggestor import Suggestor - -logger = logging.getLogger(__name__) - - -# Strategizer is nearly superfluous, but having it done the same way helps with -# readability, and we also ensure that the logging is consistent. -class Strategizer(ABC): - """ - A strategizer is a suggestor that uses other suggestors to suggest points. - """ - @abstractmethod - def next_suggestor(self, Xi: list[list], Yi: list) -> Suggestor: - """ Select the next suggestor to use.""" - pass - - def suggest(self, Xi: list[list], Yi: list) -> list[list]: - suggestor = self.next_suggestor(Xi, Yi) - suggestion = suggestor.suggest(Xi, Yi) - logger.debug( - "Strategizer of type %s is using suggestor %s of type %s. It suggested: %s", - type(self), - suggestor, - type(suggestor), - suggestion, - ) - return suggestion diff --git a/Expyrementor/suggestors/suggestor.py b/Expyrementor/suggestors/suggestor.py index e8c511c2..d08248ee 100644 --- a/Expyrementor/suggestors/suggestor.py +++ b/Expyrementor/suggestors/suggestor.py @@ -1,8 +1,5 @@ from typing import Protocol, runtime_checkable -import numpy as np -from ProcessOptimizer.space import Space - @runtime_checkable # Need to be runtime checkable for the factory to work class Suggestor(Protocol): @@ -13,14 +10,14 @@ class Suggestor(Protocol): suggest method with the same input should ideally return the same output, or at least output the same number of points. """ - def __init__(self, space: Space, n_objectives: int, rng: np.random.Generator, **kwargs): + def __init__(self, **kwargs): """ Initialize the suggestor with the search space. Suggestors can take other input arguments as needed. """ pass - def suggest(self, Xi: list[list], Yi: list) -> list[list]: + def suggest(self, Xi: list[list], Yi: list, n_asked: int) -> list[list]: """ Suggest a new point to evaluate. The input is a list of already evaluated points and their corresponding scores. The output is a list of new points to evaluate. @@ -29,10 +26,8 @@ def suggest(self, Xi: list[list], Yi: list) -> list[list]: pass -class BatchSuggestor(Suggestor): +class IncompatibleNumberAsked(ValueError): """ - Protocol for batch suggestors. Batch suggestors are suggestors that can be told how - many suggestions to make in a single call. This will usually be something building on - top of a regular suggestor, like constant liers or Krigging belivers. + Exception raised when a suggestor is asked to suggest more points than it can suggest. """ - n_given: int + pass diff --git a/Expyrementor/suggestors/suggestor_factory.py b/Expyrementor/suggestors/suggestor_factory.py index 6988a03d..7ffa50e2 100644 --- a/Expyrementor/suggestors/suggestor_factory.py +++ b/Expyrementor/suggestors/suggestor_factory.py @@ -4,8 +4,10 @@ import numpy as np from ProcessOptimizer.space import Space +from .caching_strategizer import CachingStrategizer from .default_suggestor import DefaultSuggestor -from .initial_points_suggestor import InitialPointSuggestor +from .initial_points_strategizer import InitialPointStrategizer +from .lhs_suggestor import LHSSuggestor from .po_suggestor import POSuggestor from .random_strategizer import RandomStragegizer from .suggestor import Suggestor @@ -27,10 +29,10 @@ def suggestor_factory( If definiton is a suggestor instance it is returned as is. - If definition is a dict, it is used to create a suggestor. The dictionary must have + If definition is a dict, it is used to create a suggestor. The dictionary must have a 'name' key that specifies the type of suggestor. The other keys depend on the suggestor type. It can be recursive if the suggestor is a strategizer, that is, a - suggestor that uses other suggestors. + suggestor that delegates ask() to other suggestors. If definition is None, a DefaultSuggestor is created. This is useful as a placeholder in strategizers, and should be replaced with a real suggestor before @@ -40,7 +42,7 @@ def suggestor_factory( return definition elif not definition: # If definition is None or empty, return DefaultSuggestor. logger.debug("Creating DefaultSuggestor") - return DefaultSuggestor(space, rng) + return DefaultSuggestor(space, n_objectives, rng) try: suggestor_type = definition.pop("name") except KeyError as e: @@ -49,13 +51,13 @@ def suggestor_factory( ) from e if suggestor_type == "Default" or suggestor_type is None: logger.debug("Creating DefaultSuggestor") - return DefaultSuggestor(space, rng) + return DefaultSuggestor(space, n_objectives, rng) elif suggestor_type == "InitialPoint": logger.debug("Creating InitialPointSuggestor") # If either of the necessary keys are missing, the default values are used. initial_suggestor = definition.get("initial_suggestor", None) ultimate_suggestor = definition.get("ultimate_suggestor", None) - return InitialPointSuggestor( + return InitialPointStrategizer( initial_suggestor=suggestor_factory( space, initial_suggestor, n_objectives, rng ), @@ -86,5 +88,13 @@ def suggestor_factory( return RandomStragegizer( suggestors=suggestors, n_objectives=n_objectives, rng=rng ) + elif suggestor_type == "LHS": + logger.debug("Creating a cached LHSSuggestor.") + return LHSSuggestor( + space=space, + rng=rng, + n_objectives=n_objectives, + **definition, + ) else: raise ValueError(f"Unknown suggestor name: {suggestor_type}") diff --git a/ProcessOptimizer/optimizer/optimizer.py b/ProcessOptimizer/optimizer/optimizer.py index fc0231a3..4c7c6e37 100644 --- a/ProcessOptimizer/optimizer/optimizer.py +++ b/ProcessOptimizer/optimizer/optimizer.py @@ -1053,6 +1053,7 @@ def get_constraints(self): def update_next(self): """Updates the value returned by opt.ask(). Useful if a parameter was updated after ask was called.""" self.cache_ = {} + self._n_initial_points = self.n_initial_points_-len(self.Xi) # Ask for a new next_x. Usefull if new constraints have been added or lenght_scale has been tweaked. # We only need to overwrite _next_x if it exists. if hasattr(self, "_next_x"): diff --git a/ProcessOptimizer/tests/test_expyrementor.py b/ProcessOptimizer/tests/test_expyrementor.py index 9a7e7124..d3cdd9e0 100644 --- a/ProcessOptimizer/tests/test_expyrementor.py +++ b/ProcessOptimizer/tests/test_expyrementor.py @@ -1,5 +1,5 @@ from Expyrementor.expyrementor import Expyrementor -from Expyrementor.suggestors import InitialPointSuggestor, POSuggestor +from Expyrementor.suggestors import InitialPointStrategizer, POSuggestor, CachingStrategizer, LHSSuggestor class MockSuggestor: @@ -7,9 +7,9 @@ def __init__(self, suggestions: list): self.suggestions = suggestions self.last_input = {} - def suggest(self, Xi, yi): - self.last_input = {"Xi": Xi, "yi": yi} - return self.suggestions + def suggest(self, Xi, Yi, n_asked=1): + self.last_input = {"Xi": Xi, "Yi": Yi} + return self.suggestions[:n_asked] def test_initialization(): @@ -17,10 +17,11 @@ def test_initialization(): exp = Expyrementor(space) assert exp.Xi == [] assert exp.yi == [] - assert isinstance(exp.suggestor, InitialPointSuggestor) + assert isinstance(exp.suggestor, InitialPointStrategizer) assert exp.suggestor.n_initial_points == 5 - assert isinstance(exp.suggestor.initial_suggestor, POSuggestor) - assert exp.suggestor.initial_suggestor.optimizer._n_initial_points >= 5 + assert isinstance(exp.suggestor.initial_suggestor, CachingStrategizer) + assert isinstance(exp.suggestor.initial_suggestor.suggestor, LHSSuggestor) + assert exp.suggestor.initial_suggestor.suggestor.n_points == 5 assert isinstance(exp.suggestor.ultimate_suggestor, POSuggestor) assert exp.suggestor.ultimate_suggestor.optimizer._n_initial_points == 0 @@ -51,7 +52,7 @@ def test_ask_single_return(): space = [[0, 1], [0, 1]] exp = Expyrementor(space) exp.suggestor = MockSuggestor([[0.5, 0.5]]) - assert exp.ask() == [0.5, 0.5] + assert exp.ask() == [[0.5, 0.5]] def test_ask_multiple_returns(): @@ -59,14 +60,12 @@ def test_ask_multiple_returns(): exp = Expyrementor(space) exp.suggestor = MockSuggestor([[0.5, 0.5], [0.6, 0.6]]) # exp will now get two suggestions from the suggestor, and only return the first one - assert exp.ask() == [0.5, 0.5] + assert exp.ask() == [[0.5, 0.5]] # We will now replace the suggestor exp.suggestor = MockSuggestor([[0.7, 0.7]]) - # exp will now return the second suggestion from the first suggestor - assert exp.ask() == [0.6, 0.6] # exp has now used all of the suggestions from the first suggestor, and will get a # new suggestion from the second suggestor - assert exp.ask() == [0.7, 0.7] + assert exp.ask() == [[0.7, 0.7]] def test_ask_passes_on_values(): @@ -75,7 +74,14 @@ def test_ask_passes_on_values(): exp.suggestor = MockSuggestor([[0.5, 0.5]]) exp.tell([0.6, 0.6], 2) exp.ask() - assert exp.suggestor.last_input == {"Xi": [[0.6, 0.6]], "yi": [2]} + assert exp.suggestor.last_input == {"Xi": [[0.6, 0.6]], "Yi": [2]} exp.tell([0.7, 0.7], 3) exp.ask() - assert exp.suggestor.last_input == {"Xi": [[0.6, 0.6], [0.7, 0.7]], "yi": [2, 3]} + assert exp.suggestor.last_input == {"Xi": [[0.6, 0.6], [0.7, 0.7]], "Yi": [2, 3]} + + +def test_ask_multiple(): + space = [[0, 1], [0, 1]] + exp = Expyrementor(space) + exp.suggestor = MockSuggestor([[0.5, 0.5], [0.6, 0.6]]) + assert exp.ask(2) == [[0.5, 0.5], [0.6, 0.6]] diff --git a/ProcessOptimizer/tests/test_suggestors/test_caching_strategizer.py b/ProcessOptimizer/tests/test_suggestors/test_caching_strategizer.py new file mode 100644 index 00000000..23540d6d --- /dev/null +++ b/ProcessOptimizer/tests/test_suggestors/test_caching_strategizer.py @@ -0,0 +1,60 @@ +from Expyrementor.suggestors import ( + CachingStrategizer, DefaultSuggestor, IncompatibleNumberAsked +) +import pytest + + +class MockSuggestor: + def __init__(self, suggestions: list): + self.suggestions = suggestions + self.last_input = {} + + def suggest(self, Xi, Yi, n_asked=-1): + self.last_input = {"Xi": Xi, "Yi": Yi} + return self.suggestions + + +def test_initialization(): + suggestor = CachingStrategizer(MockSuggestor([[1]])) + assert suggestor.ask_times_left == 1 + + +def test_suggest(): + suggestor = CachingStrategizer(MockSuggestor([[1]])) + assert suggestor.suggest([], []) == [[1]] + + +def test_ask_times(): + suggestor = CachingStrategizer(MockSuggestor([[1]])) + assert suggestor.suggest([], []) == [[1]] + with pytest.raises(IncompatibleNumberAsked): + suggestor.suggest([], []) + + suggestor = CachingStrategizer(MockSuggestor([[1]]), ask_times=2) + assert suggestor.suggest([], []) == [[1]] + assert suggestor.suggest([], []) == [[1]] + with pytest.raises(IncompatibleNumberAsked): + suggestor.suggest([], []) + + +def test_cache(): + suggestor = CachingStrategizer(MockSuggestor([[1], [2]])) + assert suggestor.suggest([], []) == [[1]] + suggestor.suggestor = MockSuggestor([]) + assert suggestor.suggest([], []) == [[2]] + with pytest.raises(IncompatibleNumberAsked): + suggestor.suggest([], []) + + +def test_num_asked(): + suggestor = CachingStrategizer(MockSuggestor([[1], [2], [3]]), ask_times=2) + assert suggestor.suggest([], [], n_asked=2) == [[1], [2]] + assert suggestor.suggest([], [], n_asked=2) == [[3], [1]] + assert suggestor.suggest([], [], n_asked=2) == [[2], [3]] + with pytest.raises(IncompatibleNumberAsked): + suggestor.suggest([], [], n_asked=2) + + +def test_default_suggestor(): + with pytest.raises(ValueError): + CachingStrategizer(DefaultSuggestor(space=[[0, 1]], n_objectives=1, rng=None)) diff --git a/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py index 7dfd05d8..f2c14a96 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py +++ b/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py @@ -1,6 +1,12 @@ import numpy as np import pytest -from Expyrementor.suggestors import InitialPointSuggestor, DefaultSuggestor, POSuggestor +from Expyrementor.suggestors import ( + InitialPointStrategizer, + DefaultSuggestor, + POSuggestor, + CachingStrategizer, + LHSSuggestor, +) class MockSuggestor: @@ -8,22 +14,23 @@ def __init__(self, suggestions: list): self.suggestions = suggestions self.last_input = {} - def suggest(self, Xi, yi): - self.last_input = {"Xi": Xi, "yi": yi} - return self.suggestions + def suggest(self, Xi, Yi, n_asked=1): + self.last_input = {"Xi": Xi, "Yi": Yi} + return self.suggestions[:n_asked] def test_initialization(): space = [[0, 1], [0, 1]] - suggestor = InitialPointSuggestor( - initial_suggestor=DefaultSuggestor(space, rng=np.random.default_rng(1)), - ultimate_suggestor=DefaultSuggestor(space, rng=np.random.default_rng(2)), + suggestor = InitialPointStrategizer( + initial_suggestor=DefaultSuggestor(space, n_objectives=1, rng=np.random.default_rng(1)), + ultimate_suggestor=DefaultSuggestor(space, n_objectives=1, rng=np.random.default_rng(2)), ) assert suggestor.n_initial_points == 5 # Initial suggestor is a POSuggestor with more than 5 initial points. This means than # we will only use the LHS part of this ProcessOptimizer. - assert isinstance(suggestor.initial_suggestor, POSuggestor) - assert suggestor.initial_suggestor.optimizer._n_initial_points == 5 + assert isinstance(suggestor.initial_suggestor, CachingStrategizer) + assert isinstance(suggestor.initial_suggestor.suggestor, LHSSuggestor) + assert suggestor.initial_suggestor.suggestor.n_points == 5 # Ultimate suggestor is a POSuggestor with no initial points, since # InitialPointSuggestor handles the initial points. assert isinstance(suggestor.ultimate_suggestor, POSuggestor) @@ -31,21 +38,22 @@ def test_initialization(): def test_suggestor_switch(): - suggestor = InitialPointSuggestor( - initial_suggestor=MockSuggestor([1]), - ultimate_suggestor=MockSuggestor([2]), + suggestor = InitialPointStrategizer( + initial_suggestor=MockSuggestor([[1]]), + ultimate_suggestor=MockSuggestor([[2]]), n_initial_points=3, ) - assert suggestor.next_suggestor([], []) == suggestor.initial_suggestor - assert suggestor.next_suggestor([1, 2], []) == suggestor.initial_suggestor - assert suggestor.next_suggestor([1, 2, 3], []) == suggestor.ultimate_suggestor + assert suggestor.suggest([], []) == [[1]] + assert suggestor.suggest([1], []) == [[1]] + assert suggestor.suggest([1, 2], []) == [[1]] + assert suggestor.suggest([1, 2, 3], []) == [[2]] -def test_too_may_suggested_point(): - suggestor = InitialPointSuggestor( - initial_suggestor=MockSuggestor([1, 2]), - ultimate_suggestor=MockSuggestor([4]), - n_initial_points=3, +def test_bridging_the_switch(): + suggestor = InitialPointStrategizer( + initial_suggestor=MockSuggestor([[1], [1]]), + ultimate_suggestor=MockSuggestor([[2]]), + n_initial_points=2, ) - with pytest.raises(ValueError): - suggestor.next_suggestor([1, 2], []) + assert suggestor.suggest([], [], n_asked=2) == [[1], [1]] + assert suggestor.suggest([1], [], n_asked=2) == [[1], [2]] diff --git a/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py new file mode 100644 index 00000000..54cb440e --- /dev/null +++ b/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py @@ -0,0 +1,17 @@ +import numpy as np +from ProcessOptimizer.space import space_factory +from Expyrementor.suggestors import LHSSuggestor + + +def test_initializaton(): + LHSSuggestor(space=[], rng=None, n_points=5) + + +def test_suggest(): + space = space_factory([[0, 10], [0.0, 1.0], ["cat", "dog"]]) + suggestor = LHSSuggestor(space, rng=np.random.default_rng(1), n_points=5) + suggestions = suggestor.suggest([], []) + assert len(suggestions) == 5 + assert set(suggestion[0] for suggestion in suggestions) == {1, 3, 5, 7, 9} + assert set(suggestion[1] for suggestion in suggestions) == {0.1, 0.3, 0.5, 0.7, 0.9} + assert set(suggestion[2] for suggestion in suggestions) == {"cat", "dog"} diff --git a/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py index 3baf8c14..bf91c5f9 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py +++ b/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py @@ -1,5 +1,4 @@ import numpy as np -import pytest from Expyrementor.suggestors import RandomStragegizer, Suggestor @@ -8,14 +7,14 @@ def __init__(self, suggestions: list): self.suggestions = suggestions self.last_input = {} - def suggest(self, Xi, yi): - self.last_input = {"Xi": Xi, "yi": yi} - return self.suggestions + def suggest(self, Xi, Yi, n_asked=1): + self.last_input = {"Xi": Xi, "Yi": Yi} + return self.suggestions*n_asked def test_random_strategizer(): suggestor = RandomStragegizer( - suggestors=[(0.8, MockSuggestor([])), (0.2, MockSuggestor([]))], + suggestors=[(0.8, MockSuggestor([[1]])), (0.2, MockSuggestor([[2]]))], n_objectives=1, rng=np.random.default_rng(1) ) @@ -23,6 +22,16 @@ def test_random_strategizer(): # np.random.default_rng(1).random() gives 0.5118216247148916, 0.9504636963259353, # and 0.14415961271963373 on the first three calls, so the first three calls # should return the suggestors with weights 0.8, 0.2, and 0.8, respectively. - assert suggestor.next_suggestor([], []) == suggestor.suggestors[0][1] - assert suggestor.next_suggestor([], []) == suggestor.suggestors[1][1] - assert suggestor.next_suggestor([], []) == suggestor.suggestors[0][1] + assert suggestor.suggest([], []) == [[1]] + assert suggestor.suggest([], []) == [[2]] + assert suggestor.suggest([], []) == [[1]] + + +def test_random_multiple_ask(): + suggestor = RandomStragegizer( + suggestors=[(0.8, MockSuggestor([[1]])), (0.2, MockSuggestor([[2]]))], + n_objectives=1, + rng=np.random.default_rng(1) + ) + assert suggestor.suggest([], [], n_asked=2) == [[1], [2]] + assert suggestor.suggest([], [], n_asked=3) == [[1], [1], [2]] diff --git a/examples/expyrementor.ipynb b/examples/expyrementor.ipynb new file mode 100644 index 00000000..b1dc5464 --- /dev/null +++ b/examples/expyrementor.ipynb @@ -0,0 +1,64 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Expyrementor with suggestor InitialPointStrategizer strategizer\n" + ] + }, + { + "ename": "TypeError", + "evalue": "set_n_asked() missing 2 required positional arguments: 'Yi' and 'n'", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mTypeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[1;32mIn[1], line 7\u001b[0m\n\u001b[0;32m 5\u001b[0m director \u001b[38;5;241m=\u001b[39m Expyrementor(space)\n\u001b[0;32m 6\u001b[0m \u001b[38;5;28mprint\u001b[39m(director)\n\u001b[1;32m----> 7\u001b[0m first_suggested_params \u001b[38;5;241m=\u001b[39m \u001b[43mdirector\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mask\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 8\u001b[0m \u001b[38;5;28mprint\u001b[39m(first_suggested_params)\n\u001b[0;32m 9\u001b[0m director\u001b[38;5;241m.\u001b[39mtell(first_suggested_params[\u001b[38;5;241m0\u001b[39m], \u001b[38;5;241m0.5\u001b[39m)\n", + "File \u001b[1;32mc:\\users\\srfu\\programming\\processoptimizer\\Expyrementor\\expyrementor.py:63\u001b[0m, in \u001b[0;36mExpyrementor.ask\u001b[1;34m(self, n)\u001b[0m\n\u001b[0;32m 61\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mask\u001b[39m(\u001b[38;5;28mself\u001b[39m, n: \u001b[38;5;28mint\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m):\n\u001b[0;32m 62\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_suggested_experiments) \u001b[38;5;241m<\u001b[39m n:\n\u001b[1;32m---> 63\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msuggestor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mset_n_asked\u001b[49m\u001b[43m(\u001b[49m\u001b[43mn\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_suggested_experiments\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 64\u001b[0m new_suggestions \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msuggestor\u001b[38;5;241m.\u001b[39msuggest(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mXi, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39myi)\n\u001b[0;32m 65\u001b[0m \u001b[38;5;66;03m# If any new suggestions are already in the list of suggested experiments that\u001b[39;00m\n\u001b[0;32m 66\u001b[0m \u001b[38;5;66;03m# hasn't been evaluated yet, we only keep the copies from the new suggestions.\u001b[39;00m\n", + "\u001b[1;31mTypeError\u001b[0m: set_n_asked() missing 2 required positional arguments: 'Yi' and 'n'" + ] + } + ], + "source": [ + "from Expyrementor import Expyrementor\n", + "\n", + "space = [[1.0, 100.0], [100, 200], [\"cat\", \"dog\", \"fish\"]]\n", + "\n", + "director = Expyrementor(space)\n", + "print(director)\n", + "first_suggested_params = director.ask()\n", + "print(first_suggested_params)\n", + "director.tell(first_suggested_params[0], 0.5)\n", + "second_suggested_params = director.ask()\n", + "print(second_suggested_params)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 7ca2a2ce72ef1b18a9e741b614f1fc5494bdc2cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Tue, 3 Dec 2024 13:58:44 +0100 Subject: [PATCH 03/30] fix: Fixed bug in InitialPointSuggestor --- .../suggestors/initial_points_strategizer.py | 2 +- .../test_initial_point_suggestor.py | 11 +++++++++++ examples/expyrementor.ipynb | 19 +++++++++++-------- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/Expyrementor/suggestors/initial_points_strategizer.py b/Expyrementor/suggestors/initial_points_strategizer.py index a10a9199..72b2e9a1 100644 --- a/Expyrementor/suggestors/initial_points_strategizer.py +++ b/Expyrementor/suggestors/initial_points_strategizer.py @@ -65,7 +65,7 @@ def __init__( self.ultimate_suggestor = ultimate_suggestor def suggest(self, Xi: list[list], Yi: list, n_asked: int = 1) -> list[list]: - n_initial_points = max(self.n_initial_points - len(Xi), 0) + n_initial_points = min(self.n_initial_points - len(Xi), n_asked) n_ultimate_points = n_asked - n_initial_points suggestions = [] if n_initial_points > 0: diff --git a/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py index f2c14a96..c49a6e56 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py +++ b/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py @@ -57,3 +57,14 @@ def test_bridging_the_switch(): ) assert suggestor.suggest([], [], n_asked=2) == [[1], [1]] assert suggestor.suggest([1], [], n_asked=2) == [[1], [2]] + + +def test_multiple_initial(): + suggestor = InitialPointStrategizer( + initial_suggestor=MockSuggestor([[1], [1]]), + ultimate_suggestor=MockSuggestor([[2]]), + n_initial_points=2, + ) + assert suggestor.suggest([], []) == [[1]] + assert suggestor.suggest([1], []) == [[1]] + assert suggestor.suggest([1, 1], []) == [[2]] \ No newline at end of file diff --git a/examples/expyrementor.ipynb b/examples/expyrementor.ipynb index b1dc5464..7e0c335c 100644 --- a/examples/expyrementor.ipynb +++ b/examples/expyrementor.ipynb @@ -2,26 +2,29 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Expyrementor with suggestor InitialPointStrategizer strategizer\n" + "Expyrementor with suggestor \n", + "[[70.3, 150, 'dog'], [50.5, 190, 'fish'], [10.9, 110, 'cat'], [90.10000000000001, 130, 'fish'], [30.7, 170, 'cat']]\n" ] }, { - "ename": "TypeError", - "evalue": "set_n_asked() missing 2 required positional arguments: 'Yi' and 'n'", + "ename": "IncompatibleNumberAsked", + "evalue": "CachingStrategizer ran out of ask times.", "output_type": "error", "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mTypeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[1;32mIn[1], line 7\u001b[0m\n\u001b[0;32m 5\u001b[0m director \u001b[38;5;241m=\u001b[39m Expyrementor(space)\n\u001b[0;32m 6\u001b[0m \u001b[38;5;28mprint\u001b[39m(director)\n\u001b[1;32m----> 7\u001b[0m first_suggested_params \u001b[38;5;241m=\u001b[39m \u001b[43mdirector\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mask\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 8\u001b[0m \u001b[38;5;28mprint\u001b[39m(first_suggested_params)\n\u001b[0;32m 9\u001b[0m director\u001b[38;5;241m.\u001b[39mtell(first_suggested_params[\u001b[38;5;241m0\u001b[39m], \u001b[38;5;241m0.5\u001b[39m)\n", - "File \u001b[1;32mc:\\users\\srfu\\programming\\processoptimizer\\Expyrementor\\expyrementor.py:63\u001b[0m, in \u001b[0;36mExpyrementor.ask\u001b[1;34m(self, n)\u001b[0m\n\u001b[0;32m 61\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mask\u001b[39m(\u001b[38;5;28mself\u001b[39m, n: \u001b[38;5;28mint\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m):\n\u001b[0;32m 62\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_suggested_experiments) \u001b[38;5;241m<\u001b[39m n:\n\u001b[1;32m---> 63\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msuggestor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mset_n_asked\u001b[49m\u001b[43m(\u001b[49m\u001b[43mn\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_suggested_experiments\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 64\u001b[0m new_suggestions \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msuggestor\u001b[38;5;241m.\u001b[39msuggest(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mXi, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39myi)\n\u001b[0;32m 65\u001b[0m \u001b[38;5;66;03m# If any new suggestions are already in the list of suggested experiments that\u001b[39;00m\n\u001b[0;32m 66\u001b[0m \u001b[38;5;66;03m# hasn't been evaluated yet, we only keep the copies from the new suggestions.\u001b[39;00m\n", - "\u001b[1;31mTypeError\u001b[0m: set_n_asked() missing 2 required positional arguments: 'Yi' and 'n'" + "\u001b[1;31mIncompatibleNumberAsked\u001b[0m Traceback (most recent call last)", + "Cell \u001b[1;32mIn[2], line 10\u001b[0m\n\u001b[0;32m 8\u001b[0m \u001b[38;5;28mprint\u001b[39m(first_suggested_params)\n\u001b[0;32m 9\u001b[0m director\u001b[38;5;241m.\u001b[39mtell(first_suggested_params[\u001b[38;5;241m0\u001b[39m], \u001b[38;5;241m0.5\u001b[39m)\n\u001b[1;32m---> 10\u001b[0m second_suggested_params \u001b[38;5;241m=\u001b[39m \u001b[43mdirector\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mask\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 11\u001b[0m \u001b[38;5;28mprint\u001b[39m(second_suggested_params)\n", + "File \u001b[1;32mc:\\users\\srfu\\programming\\processoptimizer\\Expyrementor\\expyrementor.py:64\u001b[0m, in \u001b[0;36mExpyrementor.ask\u001b[1;34m(self, n)\u001b[0m\n\u001b[0;32m 59\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mask\u001b[39m(\u001b[38;5;28mself\u001b[39m, n: \u001b[38;5;28mint\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m):\n\u001b[0;32m 60\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m 61\u001b[0m \u001b[38;5;124;03m Ask the suggestor for new points to evaluate. The number of points to ask is\u001b[39;00m\n\u001b[0;32m 62\u001b[0m \u001b[38;5;124;03m specified by the argument n. The method returns a list of new points to evaluate.\u001b[39;00m\n\u001b[0;32m 63\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[1;32m---> 64\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msuggestor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msuggest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mXi\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mXi\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mYi\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43myi\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mn_asked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mn\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[1;32mc:\\users\\srfu\\programming\\processoptimizer\\Expyrementor\\suggestors\\initial_points_strategizer.py:72\u001b[0m, in \u001b[0;36mInitialPointStrategizer.suggest\u001b[1;34m(self, Xi, Yi, n_asked)\u001b[0m\n\u001b[0;32m 70\u001b[0m suggestions \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m 71\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m n_initial_points \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m---> 72\u001b[0m suggestions\u001b[38;5;241m.\u001b[39mextend(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minitial_suggestor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msuggest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mXi\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mYi\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mn_initial_points\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[0;32m 73\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m n_ultimate_points \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m 74\u001b[0m suggestions\u001b[38;5;241m.\u001b[39mextend(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39multimate_suggestor\u001b[38;5;241m.\u001b[39msuggest(\n\u001b[0;32m 75\u001b[0m Xi, Yi, n_ultimate_points)\n\u001b[0;32m 76\u001b[0m )\n", + "File \u001b[1;32mc:\\users\\srfu\\programming\\processoptimizer\\Expyrementor\\suggestors\\caching_strategizer.py:53\u001b[0m, in \u001b[0;36mCachingStrategizer.suggest\u001b[1;34m(self, Xi, Yi, n_asked)\u001b[0m\n\u001b[0;32m 51\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mask_times_left \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[0;32m 52\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m---> 53\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m IncompatibleNumberAsked(\n\u001b[0;32m 54\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCachingStrategizer ran out of ask times.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 55\u001b[0m )\n\u001b[0;32m 56\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m suggested_points\n", + "\u001b[1;31mIncompatibleNumberAsked\u001b[0m: CachingStrategizer ran out of ask times." ] } ], From 5351c812eed123b50eaf2e921698edebd4a70927 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Wed, 4 Dec 2024 13:17:01 +0100 Subject: [PATCH 04/30] feat: removed caching strategizer --- Expyrementor/suggestors/__init__.py | 2 - .../suggestors/caching_strategizer.py | 56 ---------------- .../suggestors/initial_points_strategizer.py | 7 +- Expyrementor/suggestors/lhs_suggestor.py | 41 ++++++++---- Expyrementor/suggestors/suggestor_factory.py | 13 ++-- ProcessOptimizer/tests/test_expyrementor.py | 7 +- .../test_caching_strategizer.py | 60 ----------------- .../test_initial_point_suggestor.py | 29 +++++++-- .../test_suggestors/test_lhs_suggestor.py | 42 ++++++++++-- .../test_suggestors/test_random_suggestor.py | 65 ++++++++++++++++++- examples/expyrementor.ipynb | 21 ++---- 11 files changed, 168 insertions(+), 175 deletions(-) delete mode 100644 Expyrementor/suggestors/caching_strategizer.py delete mode 100644 ProcessOptimizer/tests/test_suggestors/test_caching_strategizer.py diff --git a/Expyrementor/suggestors/__init__.py b/Expyrementor/suggestors/__init__.py index 5925ce8e..c1796645 100644 --- a/Expyrementor/suggestors/__init__.py +++ b/Expyrementor/suggestors/__init__.py @@ -1,4 +1,3 @@ -from .caching_strategizer import CachingStrategizer from .default_suggestor import DefaultSuggestor from .initial_points_strategizer import InitialPointStrategizer from .lhs_suggestor import LHSSuggestor @@ -8,7 +7,6 @@ from .suggestor_factory import suggestor_factory __all__ = [ - "CachingStrategizer", "DefaultSuggestor", "IncompatibleNumberAsked", "InitialPointStrategizer", diff --git a/Expyrementor/suggestors/caching_strategizer.py b/Expyrementor/suggestors/caching_strategizer.py deleted file mode 100644 index 0bacd3ee..00000000 --- a/Expyrementor/suggestors/caching_strategizer.py +++ /dev/null @@ -1,56 +0,0 @@ -from .default_suggestor import DefaultSuggestor -from .suggestor import Suggestor, IncompatibleNumberAsked - - -class CachingStrategizer(): - """ - Cache the suggestions of a suggestor and return them when asked for new points. - - The caching strategizer will ask the suggestor for new points a fixed number of times - and store the suggestions. When asked for new points, it will return the stored - suggestions until they run out, at which point it will ask the suggestor for new - points again. - - The caching strategizer can be used to convert a suggestor that suggests a fixed - number of points into a suggestor that can suggest a variable number of points. - Examples would be fitting an LHS suggestor into an iterative optimization process. - """ - def __init__(self, suggestor: Suggestor, ask_times: int = 1, **_): - """ - Initialize the caching strategizer with the suggestor to cache and the number of - times to ask the suggestor for new points. - - Parameters - ---------- - suggestor : Suggestor - The suggestor to cache. - ask_times : int, optional - The number of times to ask the suggestor for new points, by default 1. - If it is smaller than 0, the suggestor will be asked for new points - indefinitely. - """ - if isinstance(suggestor, DefaultSuggestor): - raise ValueError("CachingStrategizer has no default suggestor defined.") - self.suggestor = suggestor - self.ask_times_left = ask_times - self.cache = [] - - def suggest(self, Xi: list[list], Yi: list, n_asked: int = 1) -> list[list]: - suggested_points = [] - while len(suggested_points) < n_asked: - if n_asked <= len(self.cache): - n_points_from_cache = n_asked - len(suggested_points) - suggested_points.extend(self.cache[:n_points_from_cache]) - self.cache = self.cache[n_points_from_cache:] - else: - if self.ask_times_left != 0: # Not <= 0 to allow for infinite asking - # Use the exisiting cache, ask the suggestor for new points, put - # them in the cache, and decrement the number of times left to ask. - suggested_points.extend(self.cache) - self.cache = self.suggestor.suggest(Xi, Yi, -1) - self.ask_times_left -= 1 - else: - raise IncompatibleNumberAsked( - "CachingStrategizer ran out of ask times." - ) - return suggested_points diff --git a/Expyrementor/suggestors/initial_points_strategizer.py b/Expyrementor/suggestors/initial_points_strategizer.py index 72b2e9a1..0f537e10 100644 --- a/Expyrementor/suggestors/initial_points_strategizer.py +++ b/Expyrementor/suggestors/initial_points_strategizer.py @@ -1,6 +1,5 @@ import logging -from .caching_strategizer import CachingStrategizer from .default_suggestor import DefaultSuggestor from .lhs_suggestor import LHSSuggestor from .po_suggestor import POSuggestor @@ -47,12 +46,11 @@ def __init__( "Initial suggestor is DefaultSuggestor, replacing with " "cached LHSSuggestor." ) - lhs_suggestor = LHSSuggestor( + initial_suggestor = LHSSuggestor( space=initial_suggestor.space, rng=initial_suggestor.rng, n_points=n_initial_points, ) - initial_suggestor = CachingStrategizer(lhs_suggestor) if isinstance(ultimate_suggestor, DefaultSuggestor): logger.debug( "Ultimate suggestor is DefaultSuggestor, replacing with POSuggestor." @@ -65,7 +63,8 @@ def __init__( self.ultimate_suggestor = ultimate_suggestor def suggest(self, Xi: list[list], Yi: list, n_asked: int = 1) -> list[list]: - n_initial_points = min(self.n_initial_points - len(Xi), n_asked) + initial_points_left = max(self.n_initial_points - len(Xi), 0) + n_initial_points = min(n_asked, initial_points_left) n_ultimate_points = n_asked - n_initial_points suggestions = [] if n_initial_points > 0: diff --git a/Expyrementor/suggestors/lhs_suggestor.py b/Expyrementor/suggestors/lhs_suggestor.py index 18216636..d12fac7c 100644 --- a/Expyrementor/suggestors/lhs_suggestor.py +++ b/Expyrementor/suggestors/lhs_suggestor.py @@ -1,34 +1,47 @@ import numpy as np from ProcessOptimizer.space import Space +from .suggestor import IncompatibleNumberAsked + class LHSSuggestor(): - def __init__(self, space: Space, rng: np.random.Generator, n_points: int = 5): + def __init__( + self, + space: Space, + n_objectives: int, + rng: np.random.Generator, + n_points: int = 5 + ): self.space = space self.rng = rng self.n_points = n_points + self.cache = self.find_lhs_points() - def suggest(self, Xi: list[list], Yi: list, n_asked: int = -1) -> list[list]: - if n_asked == -1: - n_asked = self.n_points - if n_asked != self.n_points: - raise ValueError( - f"LHSSuggestor can only provide {self.n_points} points. If you need " - "them split up, wrap it in a CachingStrategizer." - ) - samples = [] + def find_lhs_points(self): # Create a list of evenly distributed points in the range [0, 1] to sample from - sample_indices = (np.arange(n_asked) + 0.5) / n_asked + sample_indices = (np.arange(self.n_points) + 0.5) / self.n_points + samples = [] for i in range(self.space.n_dims): # Sample the points in the ith dimension lhs_aranged = self.space.dimensions[i].sample(sample_indices) # Shuffle the points in the ith dimension - samples.append([lhs_aranged[p] for p in self.rng.permutation(n_asked)]) + samples.append( + [lhs_aranged[p] for p in self.rng.permutation(self.n_points)] + ) # Now we have a list of lists where each inner list is all the points in one # dimension in random order. We need to transpose this so that we get a list of - # points in the space, where each point is a list of values from each dimension. + # points in the space, where each point is a list of values from each + # dimension. transposed_samples = [] - for i in range(n_asked): + for i in range(self.n_points): row = [samples[j][i] for j in range(self.space.n_dims)] transposed_samples.append(row) return transposed_samples + + def suggest(self, Xi: list[list], Yi: list, n_asked: int = 1) -> list[list]: + if n_asked + len(Xi) > self.n_points: + raise IncompatibleNumberAsked( + "The number of points requested is greater than the number of points " + "in the LHS cache." + ) + return self.cache[len(Xi):len(Xi) + n_asked] diff --git a/Expyrementor/suggestors/suggestor_factory.py b/Expyrementor/suggestors/suggestor_factory.py index 7ffa50e2..cfd9acb9 100644 --- a/Expyrementor/suggestors/suggestor_factory.py +++ b/Expyrementor/suggestors/suggestor_factory.py @@ -1,10 +1,9 @@ import logging -from typing import Any, Union +from typing import Any, Union, Optional import numpy as np from ProcessOptimizer.space import Space -from .caching_strategizer import CachingStrategizer from .default_suggestor import DefaultSuggestor from .initial_points_strategizer import InitialPointStrategizer from .lhs_suggestor import LHSSuggestor @@ -18,8 +17,8 @@ def suggestor_factory( space: Space, definition: Union[Suggestor, dict[str, Any], None], - n_objectives: int, - rng: np.random.Generator, + n_objectives: int = 1, + rng: Optional[np.random.Generator] = None, ) -> Suggestor: """ Create a suggestor from a definition dictionary. @@ -40,6 +39,8 @@ def suggestor_factory( """ if isinstance(definition, Suggestor): return definition + if rng is None: + rng = np.random.default_rng(1) elif not definition: # If definition is None or empty, return DefaultSuggestor. logger.debug("Creating DefaultSuggestor") return DefaultSuggestor(space, n_objectives, rng) @@ -70,8 +71,8 @@ def suggestor_factory( logger.debug("Creating POSuggestor") return POSuggestor( space=space, - rng=rng, n_objectives=n_objectives, + rng=rng, **definition, ) elif suggestor_type == "RandomStrategizer" or suggestor_type == "Random": @@ -92,8 +93,8 @@ def suggestor_factory( logger.debug("Creating a cached LHSSuggestor.") return LHSSuggestor( space=space, - rng=rng, n_objectives=n_objectives, + rng=rng, **definition, ) else: diff --git a/ProcessOptimizer/tests/test_expyrementor.py b/ProcessOptimizer/tests/test_expyrementor.py index d3cdd9e0..a0dbce70 100644 --- a/ProcessOptimizer/tests/test_expyrementor.py +++ b/ProcessOptimizer/tests/test_expyrementor.py @@ -1,5 +1,5 @@ from Expyrementor.expyrementor import Expyrementor -from Expyrementor.suggestors import InitialPointStrategizer, POSuggestor, CachingStrategizer, LHSSuggestor +from Expyrementor.suggestors import InitialPointStrategizer, POSuggestor, LHSSuggestor class MockSuggestor: @@ -19,9 +19,8 @@ def test_initialization(): assert exp.yi == [] assert isinstance(exp.suggestor, InitialPointStrategizer) assert exp.suggestor.n_initial_points == 5 - assert isinstance(exp.suggestor.initial_suggestor, CachingStrategizer) - assert isinstance(exp.suggestor.initial_suggestor.suggestor, LHSSuggestor) - assert exp.suggestor.initial_suggestor.suggestor.n_points == 5 + assert isinstance(exp.suggestor.initial_suggestor, LHSSuggestor) + assert exp.suggestor.initial_suggestor.n_points == 5 assert isinstance(exp.suggestor.ultimate_suggestor, POSuggestor) assert exp.suggestor.ultimate_suggestor.optimizer._n_initial_points == 0 diff --git a/ProcessOptimizer/tests/test_suggestors/test_caching_strategizer.py b/ProcessOptimizer/tests/test_suggestors/test_caching_strategizer.py deleted file mode 100644 index 23540d6d..00000000 --- a/ProcessOptimizer/tests/test_suggestors/test_caching_strategizer.py +++ /dev/null @@ -1,60 +0,0 @@ -from Expyrementor.suggestors import ( - CachingStrategizer, DefaultSuggestor, IncompatibleNumberAsked -) -import pytest - - -class MockSuggestor: - def __init__(self, suggestions: list): - self.suggestions = suggestions - self.last_input = {} - - def suggest(self, Xi, Yi, n_asked=-1): - self.last_input = {"Xi": Xi, "Yi": Yi} - return self.suggestions - - -def test_initialization(): - suggestor = CachingStrategizer(MockSuggestor([[1]])) - assert suggestor.ask_times_left == 1 - - -def test_suggest(): - suggestor = CachingStrategizer(MockSuggestor([[1]])) - assert suggestor.suggest([], []) == [[1]] - - -def test_ask_times(): - suggestor = CachingStrategizer(MockSuggestor([[1]])) - assert suggestor.suggest([], []) == [[1]] - with pytest.raises(IncompatibleNumberAsked): - suggestor.suggest([], []) - - suggestor = CachingStrategizer(MockSuggestor([[1]]), ask_times=2) - assert suggestor.suggest([], []) == [[1]] - assert suggestor.suggest([], []) == [[1]] - with pytest.raises(IncompatibleNumberAsked): - suggestor.suggest([], []) - - -def test_cache(): - suggestor = CachingStrategizer(MockSuggestor([[1], [2]])) - assert suggestor.suggest([], []) == [[1]] - suggestor.suggestor = MockSuggestor([]) - assert suggestor.suggest([], []) == [[2]] - with pytest.raises(IncompatibleNumberAsked): - suggestor.suggest([], []) - - -def test_num_asked(): - suggestor = CachingStrategizer(MockSuggestor([[1], [2], [3]]), ask_times=2) - assert suggestor.suggest([], [], n_asked=2) == [[1], [2]] - assert suggestor.suggest([], [], n_asked=2) == [[3], [1]] - assert suggestor.suggest([], [], n_asked=2) == [[2], [3]] - with pytest.raises(IncompatibleNumberAsked): - suggestor.suggest([], [], n_asked=2) - - -def test_default_suggestor(): - with pytest.raises(ValueError): - CachingStrategizer(DefaultSuggestor(space=[[0, 1]], n_objectives=1, rng=None)) diff --git a/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py index c49a6e56..ce5fa6a9 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py +++ b/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py @@ -1,12 +1,13 @@ import numpy as np -import pytest from Expyrementor.suggestors import ( + suggestor_factory, InitialPointStrategizer, DefaultSuggestor, POSuggestor, - CachingStrategizer, + Suggestor, LHSSuggestor, ) +from ProcessOptimizer.space import space_factory class MockSuggestor: @@ -20,23 +21,37 @@ def suggest(self, Xi, Yi, n_asked=1): def test_initialization(): - space = [[0, 1], [0, 1]] + space = space_factory([[0, 1], [0, 1]]) suggestor = InitialPointStrategizer( initial_suggestor=DefaultSuggestor(space, n_objectives=1, rng=np.random.default_rng(1)), ultimate_suggestor=DefaultSuggestor(space, n_objectives=1, rng=np.random.default_rng(2)), ) + assert isinstance(suggestor, Suggestor) assert suggestor.n_initial_points == 5 # Initial suggestor is a POSuggestor with more than 5 initial points. This means than # we will only use the LHS part of this ProcessOptimizer. - assert isinstance(suggestor.initial_suggestor, CachingStrategizer) - assert isinstance(suggestor.initial_suggestor.suggestor, LHSSuggestor) - assert suggestor.initial_suggestor.suggestor.n_points == 5 + assert isinstance(suggestor.initial_suggestor, LHSSuggestor) + assert suggestor.initial_suggestor.n_points == 5 # Ultimate suggestor is a POSuggestor with no initial points, since # InitialPointSuggestor handles the initial points. assert isinstance(suggestor.ultimate_suggestor, POSuggestor) assert suggestor.ultimate_suggestor.optimizer._n_initial_points == 0 +def test_factory(): + space = space_factory([[0, 1], [0, 1]]) + suggestor = suggestor_factory( + space=space, + definition={"name": "InitialPoint", "n_initial_points": 10}, + ) + assert isinstance(suggestor, InitialPointStrategizer) + assert suggestor.n_initial_points == 10 + assert isinstance(suggestor.initial_suggestor, LHSSuggestor) + assert suggestor.initial_suggestor.n_points == 10 + assert isinstance(suggestor.ultimate_suggestor, POSuggestor) + assert suggestor.ultimate_suggestor.optimizer._n_initial_points == 0 + + def test_suggestor_switch(): suggestor = InitialPointStrategizer( initial_suggestor=MockSuggestor([[1]]), @@ -67,4 +82,4 @@ def test_multiple_initial(): ) assert suggestor.suggest([], []) == [[1]] assert suggestor.suggest([1], []) == [[1]] - assert suggestor.suggest([1, 1], []) == [[2]] \ No newline at end of file + assert suggestor.suggest([1, 1], []) == [[2]] diff --git a/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py index 54cb440e..62eaf4e6 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py +++ b/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py @@ -1,17 +1,51 @@ import numpy as np +import pytest from ProcessOptimizer.space import space_factory -from Expyrementor.suggestors import LHSSuggestor +from Expyrementor.suggestors import LHSSuggestor, Suggestor, suggestor_factory, IncompatibleNumberAsked def test_initializaton(): - LHSSuggestor(space=[], rng=None, n_points=5) + suggestor = LHSSuggestor( + space=space_factory([[1, 2], [1, 2]]), + n_objectives=1, + rng=np.random.default_rng(1), + ) + assert isinstance(suggestor, Suggestor) + assert suggestor.n_points == 5 + + +def test_factory(): + space = space_factory([[1, 2], [1, 2]]) + suggestor = suggestor_factory( + space=space, + definition={"name": "LHS", "n_points": 10}, + ) + assert isinstance(suggestor, LHSSuggestor) + assert suggestor.n_points == 10 def test_suggest(): space = space_factory([[0, 10], [0.0, 1.0], ["cat", "dog"]]) - suggestor = LHSSuggestor(space, rng=np.random.default_rng(1), n_points=5) + suggestor = LHSSuggestor( + space, n_objectives=1, rng=np.random.default_rng(1), n_points=5 + ) suggestions = suggestor.suggest([], []) - assert len(suggestions) == 5 + assert len(suggestions) == 1 + assert len(suggestions[0]) == 3 + assert suggestions[0] in space + suggestions = suggestor.suggest([], [], n_asked=5) + # Testing that the values for each dimension is regularly spaced over the range assert set(suggestion[0] for suggestion in suggestions) == {1, 3, 5, 7, 9} assert set(suggestion[1] for suggestion in suggestions) == {0.1, 0.3, 0.5, 0.7, 0.9} assert set(suggestion[2] for suggestion in suggestions) == {"cat", "dog"} + + +def test_suggest_too_many(): + space = space_factory([[0, 10], [0.0, 1.0], ["cat", "dog"]]) + suggestor = LHSSuggestor( + space, n_objectives=1, rng=np.random.default_rng(1), n_points=5 + ) + with pytest.raises(IncompatibleNumberAsked): + suggestor.suggest([], [], n_asked=6) + with pytest.raises(IncompatibleNumberAsked): + suggestor.suggest([[1, 0.0, "cat"]], [1], n_asked=5) diff --git a/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py index bf91c5f9..1feb6843 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py +++ b/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py @@ -1,5 +1,16 @@ import numpy as np -from Expyrementor.suggestors import RandomStragegizer, Suggestor +import pytest +import warnings +from Expyrementor.suggestors import ( + RandomStragegizer, + Suggestor, + suggestor_factory, + POSuggestor, + LHSSuggestor, + DefaultSuggestor +) +from Expyrementor.suggestors.default_suggestor import NoDefaultSuggestorError +from ProcessOptimizer.space import space_factory class MockSuggestor: @@ -27,6 +38,22 @@ def test_random_strategizer(): assert suggestor.suggest([], []) == [[1]] +def test_factory(): + space = space_factory([[0, 1], [0, 1]]) + suggestor = suggestor_factory( + space=space, + definition={"name": "Random", "suggestors": [ + {"usage_ratio": 0.8, "name": "PO"}, + {"usage_ratio": 0.2, "name": "LHS"},]}, + ) + assert isinstance(suggestor, RandomStragegizer) + assert len(suggestor.suggestors) == 2 + assert suggestor.suggestors[0][0] == 0.8 + assert suggestor.suggestors[1][0] == 0.2 + assert isinstance(suggestor.suggestors[0][1], POSuggestor) + assert isinstance(suggestor.suggestors[1][1], LHSSuggestor) + + def test_random_multiple_ask(): suggestor = RandomStragegizer( suggestors=[(0.8, MockSuggestor([[1]])), (0.2, MockSuggestor([[2]]))], @@ -35,3 +62,39 @@ def test_random_multiple_ask(): ) assert suggestor.suggest([], [], n_asked=2) == [[1], [2]] assert suggestor.suggest([], [], n_asked=3) == [[1], [1], [2]] + + +def test_default_suggestor(): + with pytest.raises(NoDefaultSuggestorError): + RandomStragegizer( + suggestors=[ + (0.8, MockSuggestor([[1]])), + (0.2, DefaultSuggestor(space=[], n_objectives=1, rng=None)) + ], + n_objectives=1, + rng=np.random.default_rng(1) + ) + + +def test_wrong_sum(): + with pytest.warns(UserWarning): + # Warning if the sum of usage ratios is not 1 or 100 + RandomStragegizer( + suggestors=[(0.8, MockSuggestor([[1]])), (0.3, MockSuggestor([[2]]))], + n_objectives=1, + rng=np.random.default_rng(1) + ) + with warnings.catch_warnings(): + warnings.simplefilter("error") + # No warnings if the sum of usage ratios is 1 + RandomStragegizer( + suggestors=[(0.8, MockSuggestor([[1]])), (0.2, MockSuggestor([[2]]))], + n_objectives=1, + rng=np.random.default_rng(1) + ) + # No warnings if the sum of usage ratios is 100 + RandomStragegizer( + suggestors=[(80, MockSuggestor([[1]])), (20, MockSuggestor([[2]]))], + n_objectives=1, + rng=np.random.default_rng(1) + ) \ No newline at end of file diff --git a/examples/expyrementor.ipynb b/examples/expyrementor.ipynb index 7e0c335c..dcc6da2f 100644 --- a/examples/expyrementor.ipynb +++ b/examples/expyrementor.ipynb @@ -2,29 +2,16 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Expyrementor with suggestor \n", - "[[70.3, 150, 'dog'], [50.5, 190, 'fish'], [10.9, 110, 'cat'], [90.10000000000001, 130, 'fish'], [30.7, 170, 'cat']]\n" - ] - }, - { - "ename": "IncompatibleNumberAsked", - "evalue": "CachingStrategizer ran out of ask times.", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mIncompatibleNumberAsked\u001b[0m Traceback (most recent call last)", - "Cell \u001b[1;32mIn[2], line 10\u001b[0m\n\u001b[0;32m 8\u001b[0m \u001b[38;5;28mprint\u001b[39m(first_suggested_params)\n\u001b[0;32m 9\u001b[0m director\u001b[38;5;241m.\u001b[39mtell(first_suggested_params[\u001b[38;5;241m0\u001b[39m], \u001b[38;5;241m0.5\u001b[39m)\n\u001b[1;32m---> 10\u001b[0m second_suggested_params \u001b[38;5;241m=\u001b[39m \u001b[43mdirector\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mask\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 11\u001b[0m \u001b[38;5;28mprint\u001b[39m(second_suggested_params)\n", - "File \u001b[1;32mc:\\users\\srfu\\programming\\processoptimizer\\Expyrementor\\expyrementor.py:64\u001b[0m, in \u001b[0;36mExpyrementor.ask\u001b[1;34m(self, n)\u001b[0m\n\u001b[0;32m 59\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mask\u001b[39m(\u001b[38;5;28mself\u001b[39m, n: \u001b[38;5;28mint\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m):\n\u001b[0;32m 60\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m 61\u001b[0m \u001b[38;5;124;03m Ask the suggestor for new points to evaluate. The number of points to ask is\u001b[39;00m\n\u001b[0;32m 62\u001b[0m \u001b[38;5;124;03m specified by the argument n. The method returns a list of new points to evaluate.\u001b[39;00m\n\u001b[0;32m 63\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[1;32m---> 64\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msuggestor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msuggest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mXi\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mXi\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mYi\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43myi\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mn_asked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mn\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[1;32mc:\\users\\srfu\\programming\\processoptimizer\\Expyrementor\\suggestors\\initial_points_strategizer.py:72\u001b[0m, in \u001b[0;36mInitialPointStrategizer.suggest\u001b[1;34m(self, Xi, Yi, n_asked)\u001b[0m\n\u001b[0;32m 70\u001b[0m suggestions \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m 71\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m n_initial_points \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m---> 72\u001b[0m suggestions\u001b[38;5;241m.\u001b[39mextend(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minitial_suggestor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msuggest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mXi\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mYi\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mn_initial_points\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[0;32m 73\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m n_ultimate_points \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m 74\u001b[0m suggestions\u001b[38;5;241m.\u001b[39mextend(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39multimate_suggestor\u001b[38;5;241m.\u001b[39msuggest(\n\u001b[0;32m 75\u001b[0m Xi, Yi, n_ultimate_points)\n\u001b[0;32m 76\u001b[0m )\n", - "File \u001b[1;32mc:\\users\\srfu\\programming\\processoptimizer\\Expyrementor\\suggestors\\caching_strategizer.py:53\u001b[0m, in \u001b[0;36mCachingStrategizer.suggest\u001b[1;34m(self, Xi, Yi, n_asked)\u001b[0m\n\u001b[0;32m 51\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mask_times_left \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[0;32m 52\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m---> 53\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m IncompatibleNumberAsked(\n\u001b[0;32m 54\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCachingStrategizer ran out of ask times.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 55\u001b[0m )\n\u001b[0;32m 56\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m suggested_points\n", - "\u001b[1;31mIncompatibleNumberAsked\u001b[0m: CachingStrategizer ran out of ask times." + "Expyrementor with suggestor \n", + "[[70.3, 150, 'dog']]\n", + "[[50.5, 190, 'fish']]\n" ] } ], From fb4c3b2ce3e516578dcdf0c9f0ed6fc637ddb621 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Wed, 4 Dec 2024 14:59:43 +0100 Subject: [PATCH 05/30] fix: Made Optmizer.update_next more flexible --- ProcessOptimizer/optimizer/optimizer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ProcessOptimizer/optimizer/optimizer.py b/ProcessOptimizer/optimizer/optimizer.py index 4c7c6e37..85017641 100644 --- a/ProcessOptimizer/optimizer/optimizer.py +++ b/ProcessOptimizer/optimizer/optimizer.py @@ -1054,10 +1054,12 @@ def update_next(self): """Updates the value returned by opt.ask(). Useful if a parameter was updated after ask was called.""" self.cache_ = {} self._n_initial_points = self.n_initial_points_-len(self.Xi) + copy = self.copy(random_state=self.rng) + self.models = copy.models # Ask for a new next_x. Usefull if new constraints have been added or lenght_scale has been tweaked. # We only need to overwrite _next_x if it exists. - if hasattr(self, "_next_x"): - opt = self.copy(random_state=self.rng) + if hasattr(copy, "_next_x"): + opt = copy self._next_x = opt._next_x def get_result(self): From afffb9eeffdc5e03e32cdacaac248f65f37b22da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Wed, 4 Dec 2024 15:00:05 +0100 Subject: [PATCH 06/30] feat: Small improvements in Expyrementor and suggestors --- Expyrementor/expyrementor.py | 12 +++- Expyrementor/suggestors/lhs_suggestor.py | 3 +- Expyrementor/suggestors/po_suggestor.py | 17 ++++- Expyrementor/suggestors/random_strategizer.py | 2 +- Expyrementor/suggestors/suggestor_factory.py | 1 - ProcessOptimizer/tests/test_expyrementor.py | 6 ++ .../test_suggestors/test_lhs_suggestor.py | 5 +- .../test_suggestors/test_po_suggestor.py | 37 ++++++++++ examples/expyrementor.ipynb | 72 +++++++++++++++++-- 9 files changed, 138 insertions(+), 17 deletions(-) create mode 100644 ProcessOptimizer/tests/test_suggestors/test_po_suggestor.py diff --git a/Expyrementor/expyrementor.py b/Expyrementor/expyrementor.py index 6a40cf66..4ac58fc5 100644 --- a/Expyrementor/expyrementor.py +++ b/Expyrementor/expyrementor.py @@ -1,12 +1,13 @@ import logging from typing import Any, Union +import warnings import numpy as np from ProcessOptimizer.space import space_factory, Space from ProcessOptimizer.utils import is_2Dlistlike from ProcessOptimizer.utils.get_rng import get_random_generator -from .suggestors import DefaultSuggestor, Suggestor, suggestor_factory +from .suggestors import DefaultSuggestor, Suggestor, suggestor_factory, POSuggestor logger = logging.getLogger(__name__) @@ -47,6 +48,13 @@ def __init__( suggestor = suggestor_factory( space, DEFAULT_SUGGESTOR.copy(), n_objectives, rng=rng ) + if isinstance(suggestor, POSuggestor): + warnings.warn( + "POSuggestor is not recommended for use as a base. Use " + "InitialPointSuggestor with a POSuggestor as ultimate_suggestor " + "instead. Unless explicitly set, n_initial_points in POSuggestor will " + "be set to 0." + ) self.suggestor = suggestor self.Xi: list[list] = [] # This is a list of points in the search space. Each point is a list of values for @@ -76,4 +84,4 @@ def tell(self, x: list, y: Any): self.yi.append(y) def __str__(self): - return f"Expyrementor with suggestor {self.suggestor}" + return f"Expyrementor with a {self.suggestor.__class__.__name__} suggestor." diff --git a/Expyrementor/suggestors/lhs_suggestor.py b/Expyrementor/suggestors/lhs_suggestor.py index d12fac7c..75ffb3af 100644 --- a/Expyrementor/suggestors/lhs_suggestor.py +++ b/Expyrementor/suggestors/lhs_suggestor.py @@ -8,10 +8,9 @@ class LHSSuggestor(): def __init__( self, space: Space, - n_objectives: int, rng: np.random.Generator, n_points: int = 5 - ): + ): self.space = space self.rng = rng self.n_points = n_points diff --git a/Expyrementor/suggestors/po_suggestor.py b/Expyrementor/suggestors/po_suggestor.py index c279e1bc..cb595bb3 100644 --- a/Expyrementor/suggestors/po_suggestor.py +++ b/Expyrementor/suggestors/po_suggestor.py @@ -8,15 +8,23 @@ class POSuggestor: - def __init__(self, space: Space, rng: np.random.Generator, **kwargs): + def __init__( + self, + space: Space, + rng: np.random.Generator, + n_initial_points: int = 0, + **kwargs + ): + # We set the number of initial points to 0 by default, as initial points should + # be generated by the initial point suggestor. self.optimizer = Optimizer( dimensions=space, + n_initial_points=n_initial_points, random_state=np.random.RandomState(int(rng.random() * (2**32 - 1))), # ProcessOptimizer uses a legacy random state object, so we need to convert # the numpy random generator to a numpy random state object. **kwargs, ) - self.n_given = 1 def suggest(self, Xi: list[list], yi: list, n_asked: int = 1) -> list[list]: if Xi != self.optimizer.Xi or yi != self.optimizer.yi: @@ -30,4 +38,7 @@ def suggest(self, Xi: list[list], yi: list, n_asked: int = 1) -> list[list]: yi, point, ) - return [point] + if n_asked == 1: + return [point] + else: + return point diff --git a/Expyrementor/suggestors/random_strategizer.py b/Expyrementor/suggestors/random_strategizer.py index 43a659b4..7f4051f4 100644 --- a/Expyrementor/suggestors/random_strategizer.py +++ b/Expyrementor/suggestors/random_strategizer.py @@ -39,6 +39,6 @@ def suggest(self, Xi: list[list], Yi: list, n_asked: int = 1) -> list[list]: selector_indices = [index-weight for index in selector_indices] n_suggested = sum(index < 0 for index in selector_indices) if n_suggested > 0: - suggested_points.extend(suggestor.suggest(Xi, Yi, n_suggested)) + suggested_points.extend(suggestor.suggest(Xi, Yi, int(n_suggested))) selector_indices = [index for index in selector_indices if index >= 0] return suggested_points diff --git a/Expyrementor/suggestors/suggestor_factory.py b/Expyrementor/suggestors/suggestor_factory.py index cfd9acb9..3d5d221b 100644 --- a/Expyrementor/suggestors/suggestor_factory.py +++ b/Expyrementor/suggestors/suggestor_factory.py @@ -93,7 +93,6 @@ def suggestor_factory( logger.debug("Creating a cached LHSSuggestor.") return LHSSuggestor( space=space, - n_objectives=n_objectives, rng=rng, **definition, ) diff --git a/ProcessOptimizer/tests/test_expyrementor.py b/ProcessOptimizer/tests/test_expyrementor.py index a0dbce70..097f9c8c 100644 --- a/ProcessOptimizer/tests/test_expyrementor.py +++ b/ProcessOptimizer/tests/test_expyrementor.py @@ -1,3 +1,4 @@ +import pytest from Expyrementor.expyrementor import Expyrementor from Expyrementor.suggestors import InitialPointStrategizer, POSuggestor, LHSSuggestor @@ -84,3 +85,8 @@ def test_ask_multiple(): exp = Expyrementor(space) exp.suggestor = MockSuggestor([[0.5, 0.5], [0.6, 0.6]]) assert exp.ask(2) == [[0.5, 0.5], [0.6, 0.6]] + + +def test_warning_if_raw_POSuggestor(): + with pytest.warns(UserWarning): + Expyrementor([[0, 1], [0, 1]], suggestor={"name": "PO"}) diff --git a/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py index 62eaf4e6..724fe612 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py +++ b/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py @@ -7,7 +7,6 @@ def test_initializaton(): suggestor = LHSSuggestor( space=space_factory([[1, 2], [1, 2]]), - n_objectives=1, rng=np.random.default_rng(1), ) assert isinstance(suggestor, Suggestor) @@ -27,7 +26,7 @@ def test_factory(): def test_suggest(): space = space_factory([[0, 10], [0.0, 1.0], ["cat", "dog"]]) suggestor = LHSSuggestor( - space, n_objectives=1, rng=np.random.default_rng(1), n_points=5 + space, rng=np.random.default_rng(1), n_points=5 ) suggestions = suggestor.suggest([], []) assert len(suggestions) == 1 @@ -43,7 +42,7 @@ def test_suggest(): def test_suggest_too_many(): space = space_factory([[0, 10], [0.0, 1.0], ["cat", "dog"]]) suggestor = LHSSuggestor( - space, n_objectives=1, rng=np.random.default_rng(1), n_points=5 + space, rng=np.random.default_rng(1), n_points=5 ) with pytest.raises(IncompatibleNumberAsked): suggestor.suggest([], [], n_asked=6) diff --git a/ProcessOptimizer/tests/test_suggestors/test_po_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_po_suggestor.py new file mode 100644 index 00000000..ecc27d42 --- /dev/null +++ b/ProcessOptimizer/tests/test_suggestors/test_po_suggestor.py @@ -0,0 +1,37 @@ +import numpy as np +from Expyrementor.suggestors import POSuggestor, suggestor_factory +from ProcessOptimizer.space import space_factory + + +def test_initialization(): + space = space_factory([[0, 1], [0, 1]]) + suggestor = POSuggestor(space, rng=np.random.default_rng(1)) + assert isinstance(suggestor, POSuggestor) + assert suggestor.optimizer._n_initial_points == 0 + nonstandard_suggestor = POSuggestor( + space, rng=np.random.default_rng(1), n_initial_points=5 + ) + assert nonstandard_suggestor.optimizer._n_initial_points == 5 + + +def test_factory(): + space = space_factory([[0, 1], [0, 1]]) + suggestor = suggestor_factory( + space=space, + definition={"name": "PO"}, + ) + assert isinstance(suggestor, POSuggestor) + assert suggestor.optimizer._n_initial_points == 0 + + +def test_suggest(): + space = space_factory([[0, 1], [0, 1]]) + suggestor = POSuggestor(space, rng=np.random.default_rng(1)) + suggestions = suggestor.suggest([[1, 1]], [1]) + assert len(suggestions) == 1 + assert len(suggestions[0]) == 2 + assert suggestions[0] in space + suggestions = suggestor.suggest([[1, 1]], [1], n_asked=5) + assert len(suggestions) == 5 + for suggestion in suggestions: + assert suggestion in space diff --git a/examples/expyrementor.ipynb b/examples/expyrementor.ipynb index dcc6da2f..34e93ffa 100644 --- a/examples/expyrementor.ipynb +++ b/examples/expyrementor.ipynb @@ -1,17 +1,27 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Expyrementor\n", + "\n", + "Expyrementor implements a more modular way to handle design of experiments. The basic usage is designed to be as close to ProcessOptimizer's Optimizer as possible:" + ] + }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Expyrementor with suggestor \n", - "[[70.3, 150, 'dog']]\n", - "[[50.5, 190, 'fish']]\n" + "Expyrementor with suggestor \n", + "[[90.10000000000001, 170, 'dog']]\n", + "[[50.5, 110, 'fish']]\n", + "[[70.3, 130, 'cat'], [30.7, 150, 'cat']]\n" ] } ], @@ -22,11 +32,63 @@ "\n", "director = Expyrementor(space)\n", "print(director)\n", + "# Asking for the first parameter set to test\n", "first_suggested_params = director.ask()\n", "print(first_suggested_params)\n", + "# Telling the director how well the first parameter set performed\n", "director.tell(first_suggested_params[0], 0.5)\n", + "# Asking for the next parameter set to test\n", "second_suggested_params = director.ask()\n", - "print(second_suggested_params)" + "print(second_suggested_params)\n", + "# Telling the director how well the second parameter set performed\n", + "director.tell(first_suggested_params[0], 0.5)\n", + "# Asking for the next two parameter sets to test\n", + "third_and_fourth_suggested_params = director.ask(n=2)\n", + "print(third_and_fourth_suggested_params)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "However, we can also make more complicated suggestors. As an example, we are going to make\n", + "a suggestor that starts with a 3 point Latin Hypercube Sampling, and after that\n", + "randomly chooses a exploring (xi = 10) 20% of the time, ans an exploiting (chi = 0.00001)\n", + "Optimizer 80% of the time." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[95.05, 135, 'dog'], [15.85, 105, 'dog'], [85.14999999999999, 115, 'cat'], [33.99800434958006, 104, 'dog'], [3.429739342486463, 120, 'fish'], [83.55809954099502, 105, 'cat'], [12.47231893940161, 170, 'dog'], [49.818785128244784, 114, 'cat'], [91.57420742670749, 149, 'dog'], [21.859534469711146, 136, 'cat']]\n" + ] + } + ], + "source": [ + "suggestor_definition = {\n", + " \"name\": \"InitialPoint\",\n", + " \"n_initial_points\": 10,\n", + " \"initial_suggestor\": {\n", + " \"name\": \"LHS\", \"n_points\": 10\n", + " },\n", + " \"ultimate_suggestor\": {\n", + " \"name\": \"Random\",\n", + " \"suggestors\": [\n", + " {\"usage_ratio\": 20, \"name\": \"PO\", \"acq_func_kwargs\": {\"xi\": 10}, \"n_initial_points\": 0},\n", + " {\"usage_ratio\": 80, \"name\": \"PO\", \"acq_func_kwargs\": {\"xi\": 0.00001}, \"n_initial_points\": 0},\n", + " ]\n", + " }\n", + "}\n", + "director = Expyrementor(space, suggestor_definition)\n", + "intial_suggestions = director.ask(7)\n", + "director.tell(intial_suggestions, [0.5, 0.6, -0.7, 0.8, 0.9, 1.0, 1.1])\n", + "print(director.ask(10))" ] } ], From f2bd641845fbdc894c52a137bb4720bd38f9f851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Wed, 4 Dec 2024 15:19:06 +0100 Subject: [PATCH 07/30] doc: Removed redundant n_initial_points fron Expyrementor example --- examples/expyrementor.ipynb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/expyrementor.ipynb b/examples/expyrementor.ipynb index 34e93ffa..b109453f 100644 --- a/examples/expyrementor.ipynb +++ b/examples/expyrementor.ipynb @@ -11,14 +11,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Expyrementor with suggestor \n", + "Expyrementor with a InitialPointStrategizer suggestor.\n", "[[90.10000000000001, 170, 'dog']]\n", "[[50.5, 110, 'fish']]\n", "[[70.3, 130, 'cat'], [30.7, 150, 'cat']]\n" @@ -59,7 +59,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -80,8 +80,8 @@ " \"ultimate_suggestor\": {\n", " \"name\": \"Random\",\n", " \"suggestors\": [\n", - " {\"usage_ratio\": 20, \"name\": \"PO\", \"acq_func_kwargs\": {\"xi\": 10}, \"n_initial_points\": 0},\n", - " {\"usage_ratio\": 80, \"name\": \"PO\", \"acq_func_kwargs\": {\"xi\": 0.00001}, \"n_initial_points\": 0},\n", + " {\"usage_ratio\": 20, \"name\": \"PO\", \"acq_func_kwargs\": {\"xi\": 10}},\n", + " {\"usage_ratio\": 80, \"name\": \"PO\", \"acq_func_kwargs\": {\"xi\": 0.00001}},\n", " ]\n", " }\n", "}\n", From 1556e7191105e35e78f7fae9ae26ab60b92188c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Wed, 4 Dec 2024 15:19:16 +0100 Subject: [PATCH 08/30] chore: Removed testing for python 3.8 --- .github/workflows/python-package-tests.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package-tests.yml b/.github/workflows/python-package-tests.yml index 0cd065a2..a0db520f 100644 --- a/.github/workflows/python-package-tests.yml +++ b/.github/workflows/python-package-tests.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index b370597b..89b81b85 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ PyPI version Tests built with Python3 -Runs on +Runs on PyPI - License Scikit-Optimize DOI From 4672fa54a1de342b8763af2b0b447c604a777943 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Thu, 5 Dec 2024 12:47:41 +0100 Subject: [PATCH 09/30] chore: Expyrementor fully switched to return numpy.ndarray --- Expyrementor/suggestors/default_suggestor.py | 2 +- .../suggestors/initial_points_strategizer.py | 21 ++- Expyrementor/suggestors/lhs_suggestor.py | 8 +- Expyrementor/suggestors/po_suggestor.py | 14 +- Expyrementor/suggestors/random_strategizer.py | 9 +- Expyrementor/suggestors/suggestor.py | 2 +- ProcessOptimizer/space/space.py | 27 +++- ProcessOptimizer/tests/test_space.py | 8 +- .../test_initial_point_suggestor.py | 4 +- .../test_suggestors/test_random_suggestor.py | 4 +- ProcessOptimizer/utils/utils.py | 2 +- examples/expyrementor.ipynb | 126 +++++++++++++++++- 12 files changed, 202 insertions(+), 25 deletions(-) diff --git a/Expyrementor/suggestors/default_suggestor.py b/Expyrementor/suggestors/default_suggestor.py index e603d618..37e6e388 100644 --- a/Expyrementor/suggestors/default_suggestor.py +++ b/Expyrementor/suggestors/default_suggestor.py @@ -15,7 +15,7 @@ def __init__(self, space: Space, n_objectives: int, rng: np.random.Generator, ** self.n_objectives = n_objectives self.rng = rng - def suggest(self, Xi: list[list], Yi: list, n_asked: int = -1) -> list[list]: + def suggest(self, Xi: list[list], Yi: list, n_asked: int = -1) -> np.ndarray: raise NoDefaultSuggestorError("Default suggestor should not be used.") diff --git a/Expyrementor/suggestors/initial_points_strategizer.py b/Expyrementor/suggestors/initial_points_strategizer.py index 0f537e10..1c695ca1 100644 --- a/Expyrementor/suggestors/initial_points_strategizer.py +++ b/Expyrementor/suggestors/initial_points_strategizer.py @@ -1,5 +1,7 @@ import logging +import numpy as np + from .default_suggestor import DefaultSuggestor from .lhs_suggestor import LHSSuggestor from .po_suggestor import POSuggestor @@ -62,7 +64,7 @@ def __init__( self.initial_suggestor = initial_suggestor self.ultimate_suggestor = ultimate_suggestor - def suggest(self, Xi: list[list], Yi: list, n_asked: int = 1) -> list[list]: + def suggest(self, Xi: list[list], Yi: list, n_asked: int = 1) -> np.ndarray: initial_points_left = max(self.n_initial_points - len(Xi), 0) n_initial_points = min(n_asked, initial_points_left) n_ultimate_points = n_asked - n_initial_points @@ -73,4 +75,19 @@ def suggest(self, Xi: list[list], Yi: list, n_asked: int = 1) -> list[list]: suggestions.extend(self.ultimate_suggestor.suggest( Xi, Yi, n_ultimate_points) ) - return suggestions + return np.array(suggestions, dtype=object) + + def __str__(self): + return ( + f"InitialPointStrategizer with a {self.initial_suggestor.__class__.__name__} " + f"as inital suggestor and a {self.ultimate_suggestor.__class__.__name__} as " + "ultimate suggestor." + ) + + def __repr__(self): + return ( + f"InitialPointStrategizer(" + f"initial_suggestor={self.initial_suggestor.__class__.__name__}(...), " + f"ultimate_suggestor={self.ultimate_suggestor.__class__.__name__}(...), " + f"n_initial_points={self.n_initial_points})" + ) diff --git a/Expyrementor/suggestors/lhs_suggestor.py b/Expyrementor/suggestors/lhs_suggestor.py index 75ffb3af..e5713152 100644 --- a/Expyrementor/suggestors/lhs_suggestor.py +++ b/Expyrementor/suggestors/lhs_suggestor.py @@ -37,10 +37,16 @@ def find_lhs_points(self): transposed_samples.append(row) return transposed_samples - def suggest(self, Xi: list[list], Yi: list, n_asked: int = 1) -> list[list]: + def suggest(self, Xi: list[list], Yi: list, n_asked: int = 1) -> np.ndarray: if n_asked + len(Xi) > self.n_points: raise IncompatibleNumberAsked( "The number of points requested is greater than the number of points " "in the LHS cache." ) return self.cache[len(Xi):len(Xi) + n_asked] + + def __str__(self): + return f"Latin Hypercube Suggestor with {self.n_points} points" + + def __repr__(self): + return f"LHSSuggestor(space={self.space}, rng={self.rng}, n_points={self.n_points})" diff --git a/Expyrementor/suggestors/po_suggestor.py b/Expyrementor/suggestors/po_suggestor.py index cb595bb3..d5e078c1 100644 --- a/Expyrementor/suggestors/po_suggestor.py +++ b/Expyrementor/suggestors/po_suggestor.py @@ -26,7 +26,7 @@ def __init__( **kwargs, ) - def suggest(self, Xi: list[list], yi: list, n_asked: int = 1) -> list[list]: + def suggest(self, Xi: list[list], yi: list, n_asked: int = 1) -> np.ndarray: if Xi != self.optimizer.Xi or yi != self.optimizer.yi: self.optimizer.Xi = Xi.copy() self.optimizer.yi = yi.copy() @@ -39,6 +39,18 @@ def suggest(self, Xi: list[list], yi: list, n_asked: int = 1) -> list[list]: point, ) if n_asked == 1: + # PO returns a singe point as a list, so we wrap it in another list to + # maintain the same interface as the other suggestors. return [point] else: return point + + def __str__(self): + return "ProcessOptimizer Suggestor" + + def __repr__(self): + return ( + f"POSuggestor(space={self.optimizer.space}, " + f"rng=..., " + f"n_initial_points={self.optimizer.n_initial_points_})" + ) diff --git a/Expyrementor/suggestors/random_strategizer.py b/Expyrementor/suggestors/random_strategizer.py index 7f4051f4..c22e6574 100644 --- a/Expyrementor/suggestors/random_strategizer.py +++ b/Expyrementor/suggestors/random_strategizer.py @@ -24,7 +24,7 @@ def __init__( self.suggestors = suggestors self.rng = rng - def suggest(self, Xi: list[list], Yi: list, n_asked: int = 1) -> list[list]: + def suggest(self, Xi: list[list], Yi: list, n_asked: int = 1) -> np.ndarray: # Creating n_asked random indices in the range [0, total) selector_indices = [ relative_index*self.total for relative_index in self.rng.random(size=n_asked) @@ -41,4 +41,9 @@ def suggest(self, Xi: list[list], Yi: list, n_asked: int = 1) -> list[list]: if n_suggested > 0: suggested_points.extend(suggestor.suggest(Xi, Yi, int(n_suggested))) selector_indices = [index for index in selector_indices if index >= 0] - return suggested_points + return np.array(suggested_points) + + def __str__(self): + return "Random Strategizer with suggestors: " + ", ".join( + suggestor.__class__.__name__ for _, suggestor in self.suggestors + ) \ No newline at end of file diff --git a/Expyrementor/suggestors/suggestor.py b/Expyrementor/suggestors/suggestor.py index d08248ee..5c0528b5 100644 --- a/Expyrementor/suggestors/suggestor.py +++ b/Expyrementor/suggestors/suggestor.py @@ -17,7 +17,7 @@ def __init__(self, **kwargs): """ pass - def suggest(self, Xi: list[list], Yi: list, n_asked: int) -> list[list]: + def suggest(self, Xi: list[list], Yi: list, n_asked: int) -> np.ndarray: """ Suggest a new point to evaluate. The input is a list of already evaluated points and their corresponding scores. The output is a list of new points to evaluate. diff --git a/ProcessOptimizer/space/space.py b/ProcessOptimizer/space/space.py index 42f23b57..e6946867 100644 --- a/ProcessOptimizer/space/space.py +++ b/ProcessOptimizer/space/space.py @@ -1,3 +1,4 @@ +from __future__ import annotations from abc import ABC, abstractmethod from typing import Iterable, List, Union @@ -11,7 +12,7 @@ from .transformers import Identity from .transformers import Log10 from .transformers import Pipeline -from ..utils import get_random_generator +from ..utils import get_random_generator, is_2Dlistlike # helper class to be able to print [1, ..., 4] instead of [1, '...', 4] @@ -39,7 +40,7 @@ def space_factory(input: Union["Space", List]) -> "Space": return Space(input) -def check_dimension(dimension, transform=None): +def check_dimension(dimension, transform=None) -> Dimension: """Turn a provided dimension description into a dimension object. Checks that the provided dimension falls into one of the @@ -201,7 +202,7 @@ def sample( unique_points.append(point) seen.add(point) sampled_points = unique_points - return np.array(sampled_points) + return np.array(sampled_points, dtype=object) @abstractmethod def _sample(self, points: Iterable[float]) -> np.array: @@ -817,6 +818,26 @@ def inverse_transform(self, Xt): return rows + def sample(self, points: Iterable[Union[float, Iterable[float]]]) -> np.ndarray: + """Draw points from the space. + + Parameters + ---------- + * `points` [float or list[float]]: + A single point or a list of points to sample. All must be between 0 and 1. + + Returns + ------- + * `sampled_points` [np.ndarray]: + The sampled points. + """ + if not is_2Dlistlike(points): + points = [points] + sampled_points = [] + for dim in self.dimensions: + sampled_points.append(dim.sample([p[0] for p in points])) + return np.array(sampled_points, dtype = object).transpose() + @property def n_dims(self): """The dimensionality of the original space.""" diff --git a/ProcessOptimizer/tests/test_space.py b/ProcessOptimizer/tests/test_space.py index f1c34ee8..bb77c9c3 100644 --- a/ProcessOptimizer/tests/test_space.py +++ b/ProcessOptimizer/tests/test_space.py @@ -87,14 +87,14 @@ def test_real_bounds(): @pytest.mark.parametrize( "dimension, ismember, point_type", [ - (Real(1, 10), lambda x: 1 <= x <= 10, np.float64), + (Real(1, 10), lambda x: 1 <= x <= 10, float), ( Real(10**-5, 10**5, prior="log-uniform"), lambda x: 10**-5 <= x <= 10**5, - np.float64, + float, ), - (Integer(1, 10), lambda x: 1 <= x <= 10, np.integer), - (Integer(1, 10, transform="normalize"), lambda x: 0 <= x <= 10, np.integer), + (Integer(1, 10), lambda x: 1 <= x <= 10, int), + (Integer(1, 10, transform="normalize"), lambda x: 0 <= x <= 10, int), (Categorical(["cat", "dog", "rat"]), lambda x: x in ["cat", "dog", "rat"], str), ], ) diff --git a/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py index ce5fa6a9..8c4aec56 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py +++ b/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py @@ -70,8 +70,8 @@ def test_bridging_the_switch(): ultimate_suggestor=MockSuggestor([[2]]), n_initial_points=2, ) - assert suggestor.suggest([], [], n_asked=2) == [[1], [1]] - assert suggestor.suggest([1], [], n_asked=2) == [[1], [2]] + assert all(suggestor.suggest([], [], n_asked=2) == [[1], [1]]) + assert all(suggestor.suggest([1], [], n_asked=2) == [[1], [2]]) def test_multiple_initial(): diff --git a/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py index 1feb6843..5e1b90d4 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py +++ b/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py @@ -60,8 +60,8 @@ def test_random_multiple_ask(): n_objectives=1, rng=np.random.default_rng(1) ) - assert suggestor.suggest([], [], n_asked=2) == [[1], [2]] - assert suggestor.suggest([], [], n_asked=3) == [[1], [1], [2]] + assert all(suggestor.suggest([], [], n_asked=2) == [[1], [2]]) + assert all(suggestor.suggest([], [], n_asked=3) == [[1], [1], [2]]) def test_default_suggestor(): diff --git a/ProcessOptimizer/utils/utils.py b/ProcessOptimizer/utils/utils.py index 8563eb5d..66d38618 100644 --- a/ProcessOptimizer/utils/utils.py +++ b/ProcessOptimizer/utils/utils.py @@ -192,7 +192,7 @@ def load(filename, **kwargs): def is_listlike(x): - return isinstance(x, (list, tuple)) + return isinstance(x, (list, tuple, np.ndarray)) def is_2Dlistlike(x): diff --git a/examples/expyrementor.ipynb b/examples/expyrementor.ipynb index b109453f..4c4ab700 100644 --- a/examples/expyrementor.ipynb +++ b/examples/expyrementor.ipynb @@ -19,9 +19,10 @@ "output_type": "stream", "text": [ "Expyrementor with a InitialPointStrategizer suggestor.\n", - "[[90.10000000000001, 170, 'dog']]\n", - "[[50.5, 110, 'fish']]\n", - "[[70.3, 130, 'cat'], [30.7, 150, 'cat']]\n" + "[[90.10000000000001 170 'dog']]\n", + "[[50.5 110 'fish']]\n", + "[[70.3 130 'cat']\n", + " [30.7 150 'cat']]\n" ] } ], @@ -59,14 +60,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "[[95.05, 135, 'dog'], [15.85, 105, 'dog'], [85.14999999999999, 115, 'cat'], [33.99800434958006, 104, 'dog'], [3.429739342486463, 120, 'fish'], [83.55809954099502, 105, 'cat'], [12.47231893940161, 170, 'dog'], [49.818785128244784, 114, 'cat'], [91.57420742670749, 149, 'dog'], [21.859534469711146, 136, 'cat']]\n" + "[[95.05 135 'dog']\n", + " [15.85 105 'dog']\n", + " [85.14999999999999 115 'cat']\n", + " ['33.99800434958006' '104' 'dog']\n", + " ['3.429739342486463' '120' 'fish']\n", + " ['83.55809954099502' '105' 'cat']\n", + " ['12.47231893940161' '170' 'dog']\n", + " ['49.818785128244784' '114' 'cat']\n", + " ['91.57420742670749' '149' 'dog']\n", + " ['21.859534469711146' '136' 'cat']]\n" ] } ], @@ -90,6 +100,112 @@ "director.tell(intial_suggestions, [0.5, 0.6, -0.7, 0.8, 0.9, 1.0, 1.1])\n", "print(director.ask(10))" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also make your own suggestor and mix it with the built-in ones. Just make sure\n", + "that your suggestor implements the Suggestor protocol.\n", + "\n", + "Specifically, it has to have an `__init__` method, and a `suggest` method which accepts\n", + "the input arugments Xi (list of all tested parameters), Yi (list of the results of\n", + "testing the parameters) and `n_asked` (the number of suggestions to make)." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ConstantSuggestor is a Suggestor: True\n", + "[[50.5 150 'dog']\n", + " [50.5 150 'dog']\n", + " [50.5 150 'dog']\n", + " [50.5 150 'dog']\n", + " [50.5 150 'dog']\n", + " [1.0 100 'cat']\n", + " [1.0 100 'cat']\n", + " [1.0 100 'cat']\n", + " [1.0 100 'cat']\n", + " [1.0 100 'cat']]\n" + ] + } + ], + "source": [ + "from Expyrementor.suggestors import Suggestor\n", + "from ProcessOptimizer.space import Space, space_factory\n", + "\n", + "class ConstantSuggestor(Suggestor):\n", + " def __init__(self, space: Space, constant: int = 0.5):\n", + " self.space = space\n", + " self.constant = constant\n", + "\n", + " def suggest(self, Xi, Yi, n_asked):\n", + " return self.space.sample([[self.constant] * len(self.space)]*n_asked)\n", + "\n", + "print(f\"ConstantSuggestor is a Suggestor: {issubclass(ConstantSuggestor, Suggestor)}\")\n", + "\n", + "space = space_factory([[1.0, 100.0], [100, 200], [\"cat\", \"dog\", \"fish\"]])\n", + "\n", + "suggestor_definition = {\n", + " \"name\": \"InitialPoint\",\n", + " \"n_initial_points\": 5,\n", + " \"initial_suggestor\": ConstantSuggestor(space),\n", + " \"ultimate_suggestor\": ConstantSuggestor(space, constant=0.)\n", + " }\n", + "director = Expyrementor(space, suggestor_definition)\n", + "print(director.ask(10))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Initial Points Suggestor has the option of uding default suggestors for either the\n", + "suggestor used to suggest the initial points (initiali suggestor), or the suggestor used\n", + "after the intial points are used up (ultimate suggestor), or both.\n", + "\n", + "The default initial suggestor is a Lating Hypercube Sampling suggestor with as many points\n", + "as the Initial Point Suggestor will use it.\n", + "\n", + "The default ultimate suggestor is a ProcessOptimizer Optimizer suggestor with no initial\n", + "points (the Initial Point Suggestor takes care of the initial points).\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Director is a Expyrementor with a InitialPointStrategizer suggestor.\n", + "Its suggestor is a InitialPointStrategizer with a LHSSuggestor as inital suggestor and a POSuggestor as ultimate suggestor.\n", + "In turn, that has a Latin Hypercube Suggestor with 11 points as intial suggestor and a ProcessOptimizer Suggestor as ultimate suggestor\n" + ] + } + ], + "source": [ + "space = space_factory([[1.0, 100.0], [100, 200], [\"cat\", \"dog\", \"fish\"]])\n", + "suggestor_definition = {\n", + " \"name\": \"InitialPoint\",\n", + " \"n_initial_points\": 11,\n", + " \"initial_suggestor\": {\"name\": \"Default\"},\n", + " \"ultimate_suggestor\": {\"name\": \"Default\"},\n", + " }\n", + "director = Expyrementor(space, suggestor_definition)\n", + "print(f\"Director is a {director}\")\n", + "print(f\"Its suggestor is a {director.suggestor}\")\n", + "print(f\"In turn, that has a {director.suggestor.initial_suggestor} as intial suggestor and a {director.suggestor.ultimate_suggestor} as ultimate suggestor\")" + ] } ], "metadata": { From e43faf6bc509bd27ed83160fa8ff7e5ad6a407da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Thu, 5 Dec 2024 13:11:37 +0100 Subject: [PATCH 10/30] feat: Allow for instantiated suggestors in RandomStrategizer --- Expyrementor/suggestors/suggestor.py | 1 + Expyrementor/suggestors/suggestor_factory.py | 9 +++++ .../test_suggestors/test_random_suggestor.py | 31 ++++++++++++++++ examples/expyrementor.ipynb | 35 ++++++++++++------- 4 files changed, 63 insertions(+), 13 deletions(-) diff --git a/Expyrementor/suggestors/suggestor.py b/Expyrementor/suggestors/suggestor.py index 5c0528b5..0cf3fccc 100644 --- a/Expyrementor/suggestors/suggestor.py +++ b/Expyrementor/suggestors/suggestor.py @@ -1,3 +1,4 @@ +import numpy as np from typing import Protocol, runtime_checkable diff --git a/Expyrementor/suggestors/suggestor_factory.py b/Expyrementor/suggestors/suggestor_factory.py index 3d5d221b..96a83ed0 100644 --- a/Expyrementor/suggestors/suggestor_factory.py +++ b/Expyrementor/suggestors/suggestor_factory.py @@ -83,6 +83,15 @@ def suggestor_factory( # Note that we are removing the key usage_ratio from the suggestor # definition. If any suggestor uses this key, it will have to be redefined in # the suggestor definition. + if "suggestor" in suggestor: + if len(suggestor) > 1: + raise ValueError( + "If a suggestor definition for a RandomStrategizer has a " + "'suggestor' key, it should only have that key and " + "'usage_ratio', but it has the keys `usage_ratio`, " + f"{suggestor.keys()}." + ) + suggestor = suggestor["suggestor"] suggestors.append(( usage_ratio, suggestor_factory(space, suggestor, n_objectives, rng) )) diff --git a/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py index 5e1b90d4..0508c71f 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py +++ b/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py @@ -97,4 +97,35 @@ def test_wrong_sum(): suggestors=[(80, MockSuggestor([[1]])), (20, MockSuggestor([[2]]))], n_objectives=1, rng=np.random.default_rng(1) + ) + + +def test_random_with_suggestor_given(): + space = space_factory([[0, 1], [0, 1]]) + suggestor = suggestor_factory( + space=space, + definition={"name": "Random", "suggestors": [ + {"usage_ratio": 0.8, "suggestor": MockSuggestor([[1]])}, + {"usage_ratio": 0.2, "suggestor": MockSuggestor([[2]])},]}, + ) + assert isinstance(suggestor, RandomStragegizer) + assert len(suggestor.suggestors) == 2 + assert suggestor.suggestors[0][0] == 0.8 + assert suggestor.suggestors[1][0] == 0.2 + assert isinstance(suggestor.suggestors[0][1], MockSuggestor) + assert isinstance(suggestor.suggestors[1][1], MockSuggestor) + + +def test_random_with_suggestor_given_wrong_keys(): + space = space_factory([[0, 1], [0, 1]]) + with pytest.raises(ValueError): + suggestor_factory( + space=space, + definition={"name": "Random", "suggestors": [ + {"usage_ratio": 0.8, "suggestor": MockSuggestor([[1]])}, + { + "usage_ratio": 0.2, + "suggestor": MockSuggestor([[2]]), + "additional_key": "Can't have this key", + },]}, ) \ No newline at end of file diff --git a/examples/expyrementor.ipynb b/examples/expyrementor.ipynb index 4c4ab700..57cbaa26 100644 --- a/examples/expyrementor.ipynb +++ b/examples/expyrementor.ipynb @@ -115,7 +115,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -123,16 +123,16 @@ "output_type": "stream", "text": [ "ConstantSuggestor is a Suggestor: True\n", - "[[50.5 150 'dog']\n", + "[[83.5 116 'dog']\n", + " [50.5 184 'fish']\n", + " [17.5 150 'cat']\n", " [50.5 150 'dog']\n", " [50.5 150 'dog']\n", - " [50.5 150 'dog']\n", - " [50.5 150 'dog']\n", - " [1.0 100 'cat']\n", - " [1.0 100 'cat']\n", - " [1.0 100 'cat']\n", - " [1.0 100 'cat']\n", - " [1.0 100 'cat']]\n" + " [90.10000000000001 190 'fish']\n", + " [90.10000000000001 190 'fish']\n", + " [90.10000000000001 190 'fish']\n", + " [90.10000000000001 190 'fish']\n", + " [90.10000000000001 190 'fish']]\n" ] } ], @@ -154,10 +154,19 @@ "\n", "suggestor_definition = {\n", " \"name\": \"InitialPoint\",\n", - " \"n_initial_points\": 5,\n", - " \"initial_suggestor\": ConstantSuggestor(space),\n", - " \"ultimate_suggestor\": ConstantSuggestor(space, constant=0.)\n", + " \"n_initial_points\": 3,\n", + " \"initial_suggestor\": {\n", + " \"name\": \"LHS\", \"n_points\": 3\n", + " },\n", + " \"ultimate_suggestor\": {\n", + " \"name\": \"Random\",\n", + " \"suggestors\": [\n", + " {\"usage_ratio\": 30, \"suggestor\": ConstantSuggestor(space)},\n", + " {\"usage_ratio\": 70, \"suggestor\": ConstantSuggestor(space, constant=0.9)},\n", + " ]\n", " }\n", + "}\n", + "\n", "director = Expyrementor(space, suggestor_definition)\n", "print(director.ask(10))" ] @@ -180,7 +189,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [ { From d121b85f92b2f4a68a171ddf2801f057eb945196 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Thu, 5 Dec 2024 13:29:59 +0100 Subject: [PATCH 11/30] doc: Documneting "forbidden" keys in Suggestor definition dicts --- Expyrementor/suggestors/suggestor_factory.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Expyrementor/suggestors/suggestor_factory.py b/Expyrementor/suggestors/suggestor_factory.py index 96a83ed0..ad432b8e 100644 --- a/Expyrementor/suggestors/suggestor_factory.py +++ b/Expyrementor/suggestors/suggestor_factory.py @@ -36,6 +36,10 @@ def suggestor_factory( If definition is None, a DefaultSuggestor is created. This is useful as a placeholder in strategizers, and should be replaced with a real suggestor before use. + + The keys `name`, `usage_ratio`, and `suggestor` are reserved and should not be used + in __init__ of suggestors. They might be removed from the definition dict before + passing it to the suggestor. """ if isinstance(definition, Suggestor): return definition From 3ff20bccc13a8e607e5178ecf838633ae25cda91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Thu, 5 Dec 2024 13:30:18 +0100 Subject: [PATCH 12/30] fix: One more dtype = object for numpy --- Expyrementor/suggestors/random_strategizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Expyrementor/suggestors/random_strategizer.py b/Expyrementor/suggestors/random_strategizer.py index c22e6574..305e04ec 100644 --- a/Expyrementor/suggestors/random_strategizer.py +++ b/Expyrementor/suggestors/random_strategizer.py @@ -41,7 +41,7 @@ def suggest(self, Xi: list[list], Yi: list, n_asked: int = 1) -> np.ndarray: if n_suggested > 0: suggested_points.extend(suggestor.suggest(Xi, Yi, int(n_suggested))) selector_indices = [index for index in selector_indices if index >= 0] - return np.array(suggested_points) + return np.array(suggested_points, dtype=object) def __str__(self): return "Random Strategizer with suggestors: " + ", ".join( From ed4427e782eb0d73d03955e7b564bd891a0cc78b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Thu, 5 Dec 2024 13:30:43 +0100 Subject: [PATCH 13/30] doc: Better description for Expyrementor notebook --- examples/expyrementor.ipynb | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/examples/expyrementor.ipynb b/examples/expyrementor.ipynb index 57cbaa26..60cfd055 100644 --- a/examples/expyrementor.ipynb +++ b/examples/expyrementor.ipynb @@ -54,8 +54,8 @@ "source": [ "However, we can also make more complicated suggestors. As an example, we are going to make\n", "a suggestor that starts with a 3 point Latin Hypercube Sampling, and after that\n", - "randomly chooses a exploring (xi = 10) 20% of the time, ans an exploiting (chi = 0.00001)\n", - "Optimizer 80% of the time." + "randomly chooses an exploring Optimizer (xi = 10) 20% of the time, ans an exploiting\n", + "Optimizer (chi = 0.00001) 80% of the time." ] }, { @@ -70,13 +70,13 @@ "[[95.05 135 'dog']\n", " [15.85 105 'dog']\n", " [85.14999999999999 115 'cat']\n", - " ['33.99800434958006' '104' 'dog']\n", - " ['3.429739342486463' '120' 'fish']\n", - " ['83.55809954099502' '105' 'cat']\n", - " ['12.47231893940161' '170' 'dog']\n", - " ['49.818785128244784' '114' 'cat']\n", - " ['91.57420742670749' '149' 'dog']\n", - " ['21.859534469711146' '136' 'cat']]\n" + " [33.99800434958006 104 'dog']\n", + " [3.429739342486463 120 'fish']\n", + " [83.55809954099502 105 'cat']\n", + " [12.47231893940161 170 'dog']\n", + " [49.818785128244784 114 'cat']\n", + " [91.57420742670749 149 'dog']\n", + " [21.859534469711146 136 'cat']]\n" ] } ], @@ -109,13 +109,13 @@ "that your suggestor implements the Suggestor protocol.\n", "\n", "Specifically, it has to have an `__init__` method, and a `suggest` method which accepts\n", - "the input arugments Xi (list of all tested parameters), Yi (list of the results of\n", + "the input arguments Xi (list of all tested parameters), Yi (list of the results of\n", "testing the parameters) and `n_asked` (the number of suggestions to make)." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -137,6 +137,7 @@ } ], "source": [ + "import numpy as np\n", "from Expyrementor.suggestors import Suggestor\n", "from ProcessOptimizer.space import Space, space_factory\n", "\n", @@ -145,7 +146,7 @@ " self.space = space\n", " self.constant = constant\n", "\n", - " def suggest(self, Xi, Yi, n_asked):\n", + " def suggest(self, Xi: list[list], Yi: list, n_asked: int) -> np.ndarray:\n", " return self.space.sample([[self.constant] * len(self.space)]*n_asked)\n", "\n", "print(f\"ConstantSuggestor is a Suggestor: {issubclass(ConstantSuggestor, Suggestor)}\")\n", @@ -189,7 +190,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [ { From e3651da031be8834e59fca7d5db20b3b888d581a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Thu, 5 Dec 2024 15:48:44 +0100 Subject: [PATCH 14/30] Chore: rename Expyrementor to XpyriMentor --- Expyrementor/__init__.py | 3 -- ProcessOptimizer/tests/test_expyrementor.py | 20 +++++------ .../test_initial_point_suggestor.py | 2 +- .../test_suggestors/test_lhs_suggestor.py | 2 +- .../test_suggestors/test_po_suggestor.py | 2 +- .../test_suggestors/test_random_suggestor.py | 4 +-- XpyriMentor/__init__.py | 3 ++ .../suggestors/__init__.py | 0 .../suggestors/default_suggestor.py | 0 .../suggestors/initial_points_strategizer.py | 2 +- .../suggestors/lhs_suggestor.py | 0 .../suggestors/po_suggestor.py | 0 .../suggestors/random_strategizer.py | 0 .../suggestors/suggestor.py | 0 .../suggestors/suggestor_factory.py | 0 .../xpyrimentor.py | 10 +++--- examples/expyrementor.ipynb | 36 +++++++++---------- setup.py | 4 +-- 18 files changed, 44 insertions(+), 44 deletions(-) delete mode 100644 Expyrementor/__init__.py create mode 100644 XpyriMentor/__init__.py rename {Expyrementor => XpyriMentor}/suggestors/__init__.py (100%) rename {Expyrementor => XpyriMentor}/suggestors/default_suggestor.py (100%) rename {Expyrementor => XpyriMentor}/suggestors/initial_points_strategizer.py (99%) rename {Expyrementor => XpyriMentor}/suggestors/lhs_suggestor.py (100%) rename {Expyrementor => XpyriMentor}/suggestors/po_suggestor.py (100%) rename {Expyrementor => XpyriMentor}/suggestors/random_strategizer.py (100%) rename {Expyrementor => XpyriMentor}/suggestors/suggestor.py (100%) rename {Expyrementor => XpyriMentor}/suggestors/suggestor_factory.py (100%) rename Expyrementor/expyrementor.py => XpyriMentor/xpyrimentor.py (89%) diff --git a/Expyrementor/__init__.py b/Expyrementor/__init__.py deleted file mode 100644 index f4f104e5..00000000 --- a/Expyrementor/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .expyrementor import Expyrementor - -__all__ = ["Expyrementor"] diff --git a/ProcessOptimizer/tests/test_expyrementor.py b/ProcessOptimizer/tests/test_expyrementor.py index 097f9c8c..ccb60447 100644 --- a/ProcessOptimizer/tests/test_expyrementor.py +++ b/ProcessOptimizer/tests/test_expyrementor.py @@ -1,6 +1,6 @@ import pytest -from Expyrementor.expyrementor import Expyrementor -from Expyrementor.suggestors import InitialPointStrategizer, POSuggestor, LHSSuggestor +from XpyriMentor.XpyriMentor import XpyriMentor +from XpyriMentor.suggestors import InitialPointStrategizer, POSuggestor, LHSSuggestor class MockSuggestor: @@ -15,7 +15,7 @@ def suggest(self, Xi, Yi, n_asked=1): def test_initialization(): space = [[0, 1], [0, 1]] - exp = Expyrementor(space) + exp = XpyriMentor(space) assert exp.Xi == [] assert exp.yi == [] assert isinstance(exp.suggestor, InitialPointStrategizer) @@ -28,7 +28,7 @@ def test_initialization(): def test_tell_single_objective(): space = [[0, 1], [0, 1]] - exp = Expyrementor(space) + exp = XpyriMentor(space) exp.tell([0.5, 0.5], 1) assert exp.Xi == [[0.5, 0.5]] assert exp.yi == [1] @@ -39,7 +39,7 @@ def test_tell_single_objective(): def test_tell_multiple_objectives(): space = [[0, 1], [0, 1]] - exp = Expyrementor(space) + exp = XpyriMentor(space) exp.tell([0.5, 0.5], [1, 2]) assert exp.Xi == [[0.5, 0.5]] assert exp.yi == [[1, 2]] @@ -50,14 +50,14 @@ def test_tell_multiple_objectives(): def test_ask_single_return(): space = [[0, 1], [0, 1]] - exp = Expyrementor(space) + exp = XpyriMentor(space) exp.suggestor = MockSuggestor([[0.5, 0.5]]) assert exp.ask() == [[0.5, 0.5]] def test_ask_multiple_returns(): space = [[0, 1], [0, 1]] - exp = Expyrementor(space) + exp = XpyriMentor(space) exp.suggestor = MockSuggestor([[0.5, 0.5], [0.6, 0.6]]) # exp will now get two suggestions from the suggestor, and only return the first one assert exp.ask() == [[0.5, 0.5]] @@ -70,7 +70,7 @@ def test_ask_multiple_returns(): def test_ask_passes_on_values(): space = [[0, 1], [0, 1]] - exp = Expyrementor(space) + exp = XpyriMentor(space) exp.suggestor = MockSuggestor([[0.5, 0.5]]) exp.tell([0.6, 0.6], 2) exp.ask() @@ -82,11 +82,11 @@ def test_ask_passes_on_values(): def test_ask_multiple(): space = [[0, 1], [0, 1]] - exp = Expyrementor(space) + exp = XpyriMentor(space) exp.suggestor = MockSuggestor([[0.5, 0.5], [0.6, 0.6]]) assert exp.ask(2) == [[0.5, 0.5], [0.6, 0.6]] def test_warning_if_raw_POSuggestor(): with pytest.warns(UserWarning): - Expyrementor([[0, 1], [0, 1]], suggestor={"name": "PO"}) + XpyriMentor([[0, 1], [0, 1]], suggestor={"name": "PO"}) diff --git a/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py index 8c4aec56..e042d119 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py +++ b/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py @@ -1,5 +1,5 @@ import numpy as np -from Expyrementor.suggestors import ( +from XpyriMentor.suggestors import ( suggestor_factory, InitialPointStrategizer, DefaultSuggestor, diff --git a/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py index 724fe612..ce975840 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py +++ b/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py @@ -1,7 +1,7 @@ import numpy as np import pytest from ProcessOptimizer.space import space_factory -from Expyrementor.suggestors import LHSSuggestor, Suggestor, suggestor_factory, IncompatibleNumberAsked +from XpyriMentor.suggestors import LHSSuggestor, Suggestor, suggestor_factory, IncompatibleNumberAsked def test_initializaton(): diff --git a/ProcessOptimizer/tests/test_suggestors/test_po_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_po_suggestor.py index ecc27d42..569b0c0b 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_po_suggestor.py +++ b/ProcessOptimizer/tests/test_suggestors/test_po_suggestor.py @@ -1,5 +1,5 @@ import numpy as np -from Expyrementor.suggestors import POSuggestor, suggestor_factory +from XpyriMentor.suggestors import POSuggestor, suggestor_factory from ProcessOptimizer.space import space_factory diff --git a/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py index 0508c71f..4cbc72a9 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py +++ b/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py @@ -1,7 +1,7 @@ import numpy as np import pytest import warnings -from Expyrementor.suggestors import ( +from XpyriMentor.suggestors import ( RandomStragegizer, Suggestor, suggestor_factory, @@ -9,7 +9,7 @@ LHSSuggestor, DefaultSuggestor ) -from Expyrementor.suggestors.default_suggestor import NoDefaultSuggestorError +from XpyriMentor.suggestors.default_suggestor import NoDefaultSuggestorError from ProcessOptimizer.space import space_factory diff --git a/XpyriMentor/__init__.py b/XpyriMentor/__init__.py new file mode 100644 index 00000000..82d40afd --- /dev/null +++ b/XpyriMentor/__init__.py @@ -0,0 +1,3 @@ +from .xpyrimentor import XpyriMentor + +__all__ = ["XpyriMentor"] diff --git a/Expyrementor/suggestors/__init__.py b/XpyriMentor/suggestors/__init__.py similarity index 100% rename from Expyrementor/suggestors/__init__.py rename to XpyriMentor/suggestors/__init__.py diff --git a/Expyrementor/suggestors/default_suggestor.py b/XpyriMentor/suggestors/default_suggestor.py similarity index 100% rename from Expyrementor/suggestors/default_suggestor.py rename to XpyriMentor/suggestors/default_suggestor.py diff --git a/Expyrementor/suggestors/initial_points_strategizer.py b/XpyriMentor/suggestors/initial_points_strategizer.py similarity index 99% rename from Expyrementor/suggestors/initial_points_strategizer.py rename to XpyriMentor/suggestors/initial_points_strategizer.py index 1c695ca1..7df8eb36 100644 --- a/Expyrementor/suggestors/initial_points_strategizer.py +++ b/XpyriMentor/suggestors/initial_points_strategizer.py @@ -81,7 +81,7 @@ def __str__(self): return ( f"InitialPointStrategizer with a {self.initial_suggestor.__class__.__name__} " f"as inital suggestor and a {self.ultimate_suggestor.__class__.__name__} as " - "ultimate suggestor." + "ultimate suggestor" ) def __repr__(self): diff --git a/Expyrementor/suggestors/lhs_suggestor.py b/XpyriMentor/suggestors/lhs_suggestor.py similarity index 100% rename from Expyrementor/suggestors/lhs_suggestor.py rename to XpyriMentor/suggestors/lhs_suggestor.py diff --git a/Expyrementor/suggestors/po_suggestor.py b/XpyriMentor/suggestors/po_suggestor.py similarity index 100% rename from Expyrementor/suggestors/po_suggestor.py rename to XpyriMentor/suggestors/po_suggestor.py diff --git a/Expyrementor/suggestors/random_strategizer.py b/XpyriMentor/suggestors/random_strategizer.py similarity index 100% rename from Expyrementor/suggestors/random_strategizer.py rename to XpyriMentor/suggestors/random_strategizer.py diff --git a/Expyrementor/suggestors/suggestor.py b/XpyriMentor/suggestors/suggestor.py similarity index 100% rename from Expyrementor/suggestors/suggestor.py rename to XpyriMentor/suggestors/suggestor.py diff --git a/Expyrementor/suggestors/suggestor_factory.py b/XpyriMentor/suggestors/suggestor_factory.py similarity index 100% rename from Expyrementor/suggestors/suggestor_factory.py rename to XpyriMentor/suggestors/suggestor_factory.py diff --git a/Expyrementor/expyrementor.py b/XpyriMentor/xpyrimentor.py similarity index 89% rename from Expyrementor/expyrementor.py rename to XpyriMentor/xpyrimentor.py index 4ac58fc5..98704c02 100644 --- a/Expyrementor/expyrementor.py +++ b/XpyriMentor/xpyrimentor.py @@ -19,12 +19,12 @@ } -class Expyrementor: +class XpyriMentor: """ - Expyrementor class for optimization experiments. This class is used to manage the + XpyriMentor class for optimization experiments. This class is used to manage the optimization process, including the search space, the suggestor, and the already evaluated points. The ask-tell interface is used to interact with the optimization - process. The Expyrementor class is stateful and keeps track of the already evaluated + process. The XpyriMentor class is stateful and keeps track of the already evaluated points and scores. """ def __init__( @@ -35,7 +35,7 @@ def __init__( seed: Union[int, np.random.RandomState, np.random.Generator, None] = 42 ): """ - Initialize the Expyrementor with the search space and the suggestor. The suggestor + Initialize the XpyriMentor with the search space and the suggestor. The suggestor can be a Suggestor object, a dictionary with the suggestor configuration, or None. If the suggestor is None, the default suggestor is used. The seed is used to initialize the random number generator. @@ -84,4 +84,4 @@ def tell(self, x: list, y: Any): self.yi.append(y) def __str__(self): - return f"Expyrementor with a {self.suggestor.__class__.__name__} suggestor." + return f"XpyriMentor with a {self.suggestor.__class__.__name__} suggestor" diff --git a/examples/expyrementor.ipynb b/examples/expyrementor.ipynb index 60cfd055..9441f011 100644 --- a/examples/expyrementor.ipynb +++ b/examples/expyrementor.ipynb @@ -4,9 +4,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Expyrementor\n", + "# XpyriMentor\n", "\n", - "Expyrementor implements a more modular way to handle design of experiments. The basic usage is designed to be as close to ProcessOptimizer's Optimizer as possible:" + "XpyriMentor implements a more modular way to handle design of experiments. The basic usage is designed to be as close to ProcessOptimizer's Optimizer as possible:" ] }, { @@ -18,7 +18,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Expyrementor with a InitialPointStrategizer suggestor.\n", + "XpyriMentor with a InitialPointStrategizer suggestor.\n", "[[90.10000000000001 170 'dog']]\n", "[[50.5 110 'fish']]\n", "[[70.3 130 'cat']\n", @@ -27,11 +27,11 @@ } ], "source": [ - "from Expyrementor import Expyrementor\n", + "from XpyriMentor import XpyriMentor\n", "\n", "space = [[1.0, 100.0], [100, 200], [\"cat\", \"dog\", \"fish\"]]\n", "\n", - "director = Expyrementor(space)\n", + "director = XpyriMentor(space)\n", "print(director)\n", "# Asking for the first parameter set to test\n", "first_suggested_params = director.ask()\n", @@ -55,7 +55,7 @@ "However, we can also make more complicated suggestors. As an example, we are going to make\n", "a suggestor that starts with a 3 point Latin Hypercube Sampling, and after that\n", "randomly chooses an exploring Optimizer (xi = 10) 20% of the time, ans an exploiting\n", - "Optimizer (chi = 0.00001) 80% of the time." + "Optimizer (xi = 0.00001) 80% of the time." ] }, { @@ -95,7 +95,7 @@ " ]\n", " }\n", "}\n", - "director = Expyrementor(space, suggestor_definition)\n", + "director = XpyriMentor(space, suggestor_definition)\n", "intial_suggestions = director.ask(7)\n", "director.tell(intial_suggestions, [0.5, 0.6, -0.7, 0.8, 0.9, 1.0, 1.1])\n", "print(director.ask(10))" @@ -138,10 +138,10 @@ ], "source": [ "import numpy as np\n", - "from Expyrementor.suggestors import Suggestor\n", + "from XpyriMentor.suggestors import Suggestor\n", "from ProcessOptimizer.space import Space, space_factory\n", "\n", - "class ConstantSuggestor(Suggestor):\n", + "class ConstantSuggestor():\n", " def __init__(self, space: Space, constant: int = 0.5):\n", " self.space = space\n", " self.constant = constant\n", @@ -168,7 +168,7 @@ " }\n", "}\n", "\n", - "director = Expyrementor(space, suggestor_definition)\n", + "director = XpyriMentor(space, suggestor_definition)\n", "print(director.ask(10))" ] }, @@ -190,16 +190,16 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Director is a Expyrementor with a InitialPointStrategizer suggestor.\n", - "Its suggestor is a InitialPointStrategizer with a LHSSuggestor as inital suggestor and a POSuggestor as ultimate suggestor.\n", - "In turn, that has a Latin Hypercube Suggestor with 11 points as intial suggestor and a ProcessOptimizer Suggestor as ultimate suggestor\n" + "Director is an XpyriMentor with a InitialPointStrategizer suggestor..\n", + "Its suggestor is a InitialPointStrategizer with a LHSSuggestor as inital suggestor and a POSuggestor as ultimate suggestor..\n", + "More specifically, it has a Latin Hypercube Suggestor with 11 points as intial suggestor and a ProcessOptimizer Suggestor as ultimate suggestor.\n" ] } ], @@ -211,10 +211,10 @@ " \"initial_suggestor\": {\"name\": \"Default\"},\n", " \"ultimate_suggestor\": {\"name\": \"Default\"},\n", " }\n", - "director = Expyrementor(space, suggestor_definition)\n", - "print(f\"Director is a {director}\")\n", - "print(f\"Its suggestor is a {director.suggestor}\")\n", - "print(f\"In turn, that has a {director.suggestor.initial_suggestor} as intial suggestor and a {director.suggestor.ultimate_suggestor} as ultimate suggestor\")" + "director = XpyriMentor(space, suggestor_definition)\n", + "print(f\"Director is an {director}.\")\n", + "print(f\"Its suggestor is a {director.suggestor}.\")\n", + "print(f\"More specifically, it has a {director.suggestor.initial_suggestor} as intial suggestor and a {director.suggestor.ultimate_suggestor} as ultimate suggestor.\")" ] } ], diff --git a/setup.py b/setup.py index b203233d..5ca2677e 100644 --- a/setup.py +++ b/setup.py @@ -26,8 +26,8 @@ "ProcessOptimizer.space", "ProcessOptimizer.utils", "ProcessOptimizer.learning.gaussian_process", - "Expyrementor", - "Expyrementor.suggestors", + "XpyriMentor", + "XpyriMentor.suggestors", ], install_requires=[ "numpy", From 63a88f005ba971b3f675dc934c4b932eab6c5e50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Thu, 5 Dec 2024 15:58:09 +0100 Subject: [PATCH 15/30] chore: Rename test file --- .../tests/{test_expyrementor.py => test_xpyrimentor.py} | 2 +- examples/expyrementor.ipynb | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) rename ProcessOptimizer/tests/{test_expyrementor.py => test_xpyrimentor.py} (98%) diff --git a/ProcessOptimizer/tests/test_expyrementor.py b/ProcessOptimizer/tests/test_xpyrimentor.py similarity index 98% rename from ProcessOptimizer/tests/test_expyrementor.py rename to ProcessOptimizer/tests/test_xpyrimentor.py index ccb60447..2643275a 100644 --- a/ProcessOptimizer/tests/test_expyrementor.py +++ b/ProcessOptimizer/tests/test_xpyrimentor.py @@ -1,5 +1,5 @@ import pytest -from XpyriMentor.XpyriMentor import XpyriMentor +from XpyriMentor.xpyrimentor import XpyriMentor from XpyriMentor.suggestors import InitialPointStrategizer, POSuggestor, LHSSuggestor diff --git a/examples/expyrementor.ipynb b/examples/expyrementor.ipynb index 9441f011..3dbd9daa 100644 --- a/examples/expyrementor.ipynb +++ b/examples/expyrementor.ipynb @@ -18,7 +18,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "XpyriMentor with a InitialPointStrategizer suggestor.\n", + "XpyriMentor with a InitialPointStrategizer suggestor\n", "[[90.10000000000001 170 'dog']]\n", "[[50.5 110 'fish']]\n", "[[70.3 130 'cat']\n", @@ -190,15 +190,15 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Director is an XpyriMentor with a InitialPointStrategizer suggestor..\n", - "Its suggestor is a InitialPointStrategizer with a LHSSuggestor as inital suggestor and a POSuggestor as ultimate suggestor..\n", + "Director is an XpyriMentor with a InitialPointStrategizer suggestor.\n", + "Its suggestor is a InitialPointStrategizer with a LHSSuggestor as inital suggestor and a POSuggestor as ultimate suggestor.\n", "More specifically, it has a Latin Hypercube Suggestor with 11 points as intial suggestor and a ProcessOptimizer Suggestor as ultimate suggestor.\n" ] } From 7611efb73dd964e38b610ac195e4bc14d555dd06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Thu, 5 Dec 2024 15:58:37 +0100 Subject: [PATCH 16/30] chore: Removed n_objectives from RandomStrategizzer init since it isn't needed --- .../tests/test_suggestors/test_random_suggestor.py | 5 ----- XpyriMentor/suggestors/random_strategizer.py | 2 +- XpyriMentor/suggestors/suggestor_factory.py | 4 +--- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py index 4cbc72a9..3cb30747 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py +++ b/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py @@ -57,7 +57,6 @@ def test_factory(): def test_random_multiple_ask(): suggestor = RandomStragegizer( suggestors=[(0.8, MockSuggestor([[1]])), (0.2, MockSuggestor([[2]]))], - n_objectives=1, rng=np.random.default_rng(1) ) assert all(suggestor.suggest([], [], n_asked=2) == [[1], [2]]) @@ -71,7 +70,6 @@ def test_default_suggestor(): (0.8, MockSuggestor([[1]])), (0.2, DefaultSuggestor(space=[], n_objectives=1, rng=None)) ], - n_objectives=1, rng=np.random.default_rng(1) ) @@ -81,7 +79,6 @@ def test_wrong_sum(): # Warning if the sum of usage ratios is not 1 or 100 RandomStragegizer( suggestors=[(0.8, MockSuggestor([[1]])), (0.3, MockSuggestor([[2]]))], - n_objectives=1, rng=np.random.default_rng(1) ) with warnings.catch_warnings(): @@ -89,13 +86,11 @@ def test_wrong_sum(): # No warnings if the sum of usage ratios is 1 RandomStragegizer( suggestors=[(0.8, MockSuggestor([[1]])), (0.2, MockSuggestor([[2]]))], - n_objectives=1, rng=np.random.default_rng(1) ) # No warnings if the sum of usage ratios is 100 RandomStragegizer( suggestors=[(80, MockSuggestor([[1]])), (20, MockSuggestor([[2]]))], - n_objectives=1, rng=np.random.default_rng(1) ) diff --git a/XpyriMentor/suggestors/random_strategizer.py b/XpyriMentor/suggestors/random_strategizer.py index 305e04ec..e77854ca 100644 --- a/XpyriMentor/suggestors/random_strategizer.py +++ b/XpyriMentor/suggestors/random_strategizer.py @@ -8,7 +8,7 @@ class RandomStragegizer(): def __init__( - self, suggestors: list[tuple[float, Suggestor]], n_objectives, rng: np.random.Generator + self, suggestors: list[tuple[float, Suggestor]], rng: np.random.Generator ): self.total = sum(item[0] for item in suggestors) if float(self.total) != 1.0 and float(self.total) != 100.0: diff --git a/XpyriMentor/suggestors/suggestor_factory.py b/XpyriMentor/suggestors/suggestor_factory.py index ad432b8e..57439a9b 100644 --- a/XpyriMentor/suggestors/suggestor_factory.py +++ b/XpyriMentor/suggestors/suggestor_factory.py @@ -99,9 +99,7 @@ def suggestor_factory( suggestors.append(( usage_ratio, suggestor_factory(space, suggestor, n_objectives, rng) )) - return RandomStragegizer( - suggestors=suggestors, n_objectives=n_objectives, rng=rng - ) + return RandomStragegizer(suggestors=suggestors, rng=rng) elif suggestor_type == "LHS": logger.debug("Creating a cached LHSSuggestor.") return LHSSuggestor( From 1dacbe89eb983df1f8617c95b152e46ef78d08cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Fri, 6 Dec 2024 08:56:03 +0100 Subject: [PATCH 17/30] fix: LHS sampling return an numpy.ndarray instead of a list of lists --- .../test_suggestors/test_lhs_suggestor.py | 12 +++++++++ XpyriMentor/suggestors/lhs_suggestor.py | 26 +++++++------------ 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py index ce975840..632befc9 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py +++ b/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py @@ -11,6 +11,7 @@ def test_initializaton(): ) assert isinstance(suggestor, Suggestor) assert suggestor.n_points == 5 + assert len(suggestor.cache) == 5 def test_factory(): @@ -44,7 +45,18 @@ def test_suggest_too_many(): suggestor = LHSSuggestor( space, rng=np.random.default_rng(1), n_points=5 ) + for n_told in range(1, 5): + told = suggestor.suggest([], [], n_asked=n_told) + suggestor.suggest(told, [0]*n_told, n_asked=1) with pytest.raises(IncompatibleNumberAsked): suggestor.suggest([], [], n_asked=6) with pytest.raises(IncompatibleNumberAsked): suggestor.suggest([[1, 0.0, "cat"]], [1], n_asked=5) + + +def test_n(): + space = space_factory([[0, 10], [0.0, 1.0], ["cat", "dog"]]) + for n in range(1, 10): + suggestor = LHSSuggestor(space, rng=np.random.default_rng(1), n_points=n) + assert suggestor.n_points == n + assert len(suggestor.cache) == n diff --git a/XpyriMentor/suggestors/lhs_suggestor.py b/XpyriMentor/suggestors/lhs_suggestor.py index e5713152..34636c66 100644 --- a/XpyriMentor/suggestors/lhs_suggestor.py +++ b/XpyriMentor/suggestors/lhs_suggestor.py @@ -19,23 +19,15 @@ def __init__( def find_lhs_points(self): # Create a list of evenly distributed points in the range [0, 1] to sample from sample_indices = (np.arange(self.n_points) + 0.5) / self.n_points - samples = [] - for i in range(self.space.n_dims): - # Sample the points in the ith dimension - lhs_aranged = self.space.dimensions[i].sample(sample_indices) - # Shuffle the points in the ith dimension - samples.append( - [lhs_aranged[p] for p in self.rng.permutation(self.n_points)] - ) - # Now we have a list of lists where each inner list is all the points in one - # dimension in random order. We need to transpose this so that we get a list of - # points in the space, where each point is a list of values from each - # dimension. - transposed_samples = [] - for i in range(self.n_points): - row = [samples[j][i] for j in range(self.space.n_dims)] - transposed_samples.append(row) - return transposed_samples + permuted_sample_indices = np.array( + [self.rng.permutation(sample_indices) for _ in range(self.space.n_dims)], + dtype=object, + ) + # Permuted sample indices are now a list of n_dims arrays, each containing the + # same n_points indices in a different order. We need to transpose this list to + # get an array of n_points arrays, each containing the indices for one point. + samples_indexed = permuted_sample_indices.T + return self.space.sample(samples_indexed) def suggest(self, Xi: list[list], Yi: list, n_asked: int = 1) -> np.ndarray: if n_asked + len(Xi) > self.n_points: From aa1139014f4c168e2fa30fcf01804fdd2878dc30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Fri, 6 Dec 2024 10:31:55 +0100 Subject: [PATCH 18/30] feat: Space.sample gives errors oon wrongly sized input --- ProcessOptimizer/space/space.py | 5 +++ ProcessOptimizer/tests/test_space.py | 52 ++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/ProcessOptimizer/space/space.py b/ProcessOptimizer/space/space.py index e6946867..85c8653e 100644 --- a/ProcessOptimizer/space/space.py +++ b/ProcessOptimizer/space/space.py @@ -833,6 +833,11 @@ def sample(self, points: Iterable[Union[float, Iterable[float]]]) -> np.ndarray: """ if not is_2Dlistlike(points): points = [points] + if any(len(point) != len(self) for point in points): + raise ValueError( + "One or more points does not have the same length as the space " + + str(({len(self)})) + ) sampled_points = [] for dim in self.dimensions: sampled_points.append(dim.sample([p[0] for p in points])) diff --git a/ProcessOptimizer/tests/test_space.py b/ProcessOptimizer/tests/test_space.py index bb77c9c3..f0cba461 100644 --- a/ProcessOptimizer/tests/test_space.py +++ b/ProcessOptimizer/tests/test_space.py @@ -744,3 +744,55 @@ def test_lhs(): # Asserting the the values are the same for both the lhs, even though the order is different for i in range(4): assert set([x[i] for x in lhs_one]) == set([x[i] for x in lhs_two]) + + +def test_sample(): + SPACE = Space( + [ + Integer(1, 6), + Real(1, 7), + Real(10**-3, 10**3, prior="log-uniform"), + Categorical(list("abc")), + ] + ) + # Getting one sample + samples = SPACE.sample([0.5]*len(SPACE)) + assert len(samples) == 1 + assert isinstance(samples, np.ndarray) + assert len(samples[0]) == 4 + values = samples[0] + assert isinstance(values, np.ndarray) + assert isinstance(values[0], int) + assert isinstance(values[1], float) + assert isinstance(values[2], float) + assert isinstance(values[3], str) + # Getting multiple samples + samples = SPACE.sample([[0.5]*len(SPACE)]*5) + assert len(samples) == 5 + assert len(samples[0]) == 4 + assert isinstance(samples, np.ndarray) + for sample in samples: + assert isinstance(sample, np.ndarray) + assert isinstance(sample[0], int) + assert isinstance(sample[1], float) + assert isinstance(sample[2], float) + assert isinstance(sample[3], str) + + +def test_sample_wrong_size(): + SPACE = Space( + [ + Integer(1, 6), + Real(1, 7), + Real(10**-3, 10**3, prior="log-uniform"), + Categorical(list("abc")), + ] + ) + with pytest.raises(ValueError): + SPACE.sample([0.5]*3) + with pytest.raises(ValueError): + SPACE.sample([0.5]*5) + with pytest.raises(ValueError): + SPACE.sample([[0.5]*3]*5) + with pytest.raises(ValueError): + SPACE.sample([[0.5]*5]*3) From 65bed889d03d27168f6e6254daf3a870aee58382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Fri, 6 Dec 2024 10:32:18 +0100 Subject: [PATCH 19/30] test: Fixed last test for RandomStrategizer --- ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py index 3cb30747..bffa94a5 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py +++ b/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py @@ -26,7 +26,6 @@ def suggest(self, Xi, Yi, n_asked=1): def test_random_strategizer(): suggestor = RandomStragegizer( suggestors=[(0.8, MockSuggestor([[1]])), (0.2, MockSuggestor([[2]]))], - n_objectives=1, rng=np.random.default_rng(1) ) assert isinstance(suggestor, Suggestor) From 2d25c62a498de10ea387161b096970418de11cdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Fri, 6 Dec 2024 10:39:55 +0100 Subject: [PATCH 20/30] chore: Better type hints fro XpyriMentor --- XpyriMentor/suggestors/initial_points_strategizer.py | 3 ++- XpyriMentor/suggestors/lhs_suggestor.py | 8 ++++++-- XpyriMentor/suggestors/po_suggestor.py | 5 ++++- XpyriMentor/suggestors/random_strategizer.py | 3 ++- XpyriMentor/suggestors/suggestor.py | 6 +++--- XpyriMentor/xpyrimentor.py | 8 ++++---- 6 files changed, 21 insertions(+), 12 deletions(-) diff --git a/XpyriMentor/suggestors/initial_points_strategizer.py b/XpyriMentor/suggestors/initial_points_strategizer.py index 7df8eb36..5201ff5c 100644 --- a/XpyriMentor/suggestors/initial_points_strategizer.py +++ b/XpyriMentor/suggestors/initial_points_strategizer.py @@ -1,4 +1,5 @@ import logging +from typing import Iterable import numpy as np @@ -64,7 +65,7 @@ def __init__( self.initial_suggestor = initial_suggestor self.ultimate_suggestor = ultimate_suggestor - def suggest(self, Xi: list[list], Yi: list, n_asked: int = 1) -> np.ndarray: + def suggest(self, Xi: Iterable[Iterable], Yi: Iterable, n_asked: int = 1) -> np.ndarray: initial_points_left = max(self.n_initial_points - len(Xi), 0) n_initial_points = min(n_asked, initial_points_left) n_ultimate_points = n_asked - n_initial_points diff --git a/XpyriMentor/suggestors/lhs_suggestor.py b/XpyriMentor/suggestors/lhs_suggestor.py index 34636c66..f702321c 100644 --- a/XpyriMentor/suggestors/lhs_suggestor.py +++ b/XpyriMentor/suggestors/lhs_suggestor.py @@ -1,3 +1,5 @@ +from typing import Iterable + import numpy as np from ProcessOptimizer.space import Space @@ -16,7 +18,7 @@ def __init__( self.n_points = n_points self.cache = self.find_lhs_points() - def find_lhs_points(self): + def find_lhs_points(self) -> np.ndarray: # Create a list of evenly distributed points in the range [0, 1] to sample from sample_indices = (np.arange(self.n_points) + 0.5) / self.n_points permuted_sample_indices = np.array( @@ -29,7 +31,9 @@ def find_lhs_points(self): samples_indexed = permuted_sample_indices.T return self.space.sample(samples_indexed) - def suggest(self, Xi: list[list], Yi: list, n_asked: int = 1) -> np.ndarray: + def suggest( + self, Xi: Iterable[Iterable], Yi: Iterable, n_asked: int = 1 + ) -> np.ndarray: if n_asked + len(Xi) > self.n_points: raise IncompatibleNumberAsked( "The number of points requested is greater than the number of points " diff --git a/XpyriMentor/suggestors/po_suggestor.py b/XpyriMentor/suggestors/po_suggestor.py index d5e078c1..ae5336b5 100644 --- a/XpyriMentor/suggestors/po_suggestor.py +++ b/XpyriMentor/suggestors/po_suggestor.py @@ -1,4 +1,5 @@ import logging +from typing import Iterable import numpy as np from ProcessOptimizer import Optimizer @@ -26,7 +27,9 @@ def __init__( **kwargs, ) - def suggest(self, Xi: list[list], yi: list, n_asked: int = 1) -> np.ndarray: + def suggest( + self, Xi: Iterable[Iterable], yi: Iterable, n_asked: int = 1 + ) -> np.ndarray: if Xi != self.optimizer.Xi or yi != self.optimizer.yi: self.optimizer.Xi = Xi.copy() self.optimizer.yi = yi.copy() diff --git a/XpyriMentor/suggestors/random_strategizer.py b/XpyriMentor/suggestors/random_strategizer.py index e77854ca..6fe501e8 100644 --- a/XpyriMentor/suggestors/random_strategizer.py +++ b/XpyriMentor/suggestors/random_strategizer.py @@ -1,4 +1,5 @@ import warnings +from typing import Iterable import numpy as np @@ -24,7 +25,7 @@ def __init__( self.suggestors = suggestors self.rng = rng - def suggest(self, Xi: list[list], Yi: list, n_asked: int = 1) -> np.ndarray: + def suggest(self, Xi: Iterable[Iterable], Yi: Iterable, n_asked: int = 1) -> np.ndarray: # Creating n_asked random indices in the range [0, total) selector_indices = [ relative_index*self.total for relative_index in self.rng.random(size=n_asked) diff --git a/XpyriMentor/suggestors/suggestor.py b/XpyriMentor/suggestors/suggestor.py index 0cf3fccc..4b280820 100644 --- a/XpyriMentor/suggestors/suggestor.py +++ b/XpyriMentor/suggestors/suggestor.py @@ -1,6 +1,6 @@ -import numpy as np -from typing import Protocol, runtime_checkable +from typing import Iterable, Protocol, runtime_checkable +import numpy as np @runtime_checkable # Need to be runtime checkable for the factory to work class Suggestor(Protocol): @@ -18,7 +18,7 @@ def __init__(self, **kwargs): """ pass - def suggest(self, Xi: list[list], Yi: list, n_asked: int) -> np.ndarray: + def suggest(self, Xi: Iterable[Iterable], Yi: Iterable, n_asked: int) -> np.ndarray: """ Suggest a new point to evaluate. The input is a list of already evaluated points and their corresponding scores. The output is a list of new points to evaluate. diff --git a/XpyriMentor/xpyrimentor.py b/XpyriMentor/xpyrimentor.py index 98704c02..4c98da0a 100644 --- a/XpyriMentor/xpyrimentor.py +++ b/XpyriMentor/xpyrimentor.py @@ -1,5 +1,5 @@ import logging -from typing import Any, Union +from typing import Any, Iterable, Union import warnings import numpy as np @@ -56,7 +56,7 @@ def __init__( "be set to 0." ) self.suggestor = suggestor - self.Xi: list[list] = [] + self.Xi: list[np.ndarray] = [] # This is a list of points in the search space. Each point is a list of values for # each dimension of the search space. self.yi: list = [] @@ -64,14 +64,14 @@ def __init__( # objective optimization or a list of floats for multiobjective optimization. pass - def ask(self, n: int = 1): + def ask(self, n: int = 1) -> np.ndarray: """ Ask the suggestor for new points to evaluate. The number of points to ask is specified by the argument n. The method returns a list of new points to evaluate. """ return self.suggestor.suggest(Xi=self.Xi, Yi=self.yi, n_asked=n) - def tell(self, x: list, y: Any): + def tell(self, x: Iterable, y: Any) -> None: if is_2Dlistlike(x): # If x is a list of points, we assume that y is a list of scores of the same # length, and we add the members of x and y to the lists Xi and yi. From ab90ab2d8e2154263345543d12a9e3579a9f8d28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Fri, 6 Dec 2024 10:54:01 +0100 Subject: [PATCH 21/30] doc: Better docstrings for Suggestor protocol --- XpyriMentor/suggestors/suggestor.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/XpyriMentor/suggestors/suggestor.py b/XpyriMentor/suggestors/suggestor.py index 4b280820..bca5b0d4 100644 --- a/XpyriMentor/suggestors/suggestor.py +++ b/XpyriMentor/suggestors/suggestor.py @@ -2,14 +2,14 @@ import numpy as np + @runtime_checkable # Need to be runtime checkable for the factory to work class Suggestor(Protocol): """ Protocol for suggestors. Suggestors are used to suggest new points to evaluate in the optimization process. Suggestors should be stateless and only depend on the search space and the already evaluated points. In particular, consecutive calls to the - suggest method with the same input should ideally return the same output, or at least - output the same number of points. + suggest method with the same input should ideally return the same output. """ def __init__(self, **kwargs): """ @@ -20,9 +20,21 @@ def __init__(self, **kwargs): def suggest(self, Xi: Iterable[Iterable], Yi: Iterable, n_asked: int) -> np.ndarray: """ - Suggest a new point to evaluate. The input is a list of already evaluated points - and their corresponding scores. The output is a list of new points to evaluate. - The list can have the length of 1 or more. + Suggest a new point to evaluate. + + Parameters + ---------- + * Xi [`Iterable[Iterable]`]: + The input is a list of already evaluated points. + * Yi [`Iterable`]: + The results of the evaulations of `Xi`. + * n_asked [`int`]: + The number of suggested points to return + + Returns + ---------- + A np.ndarray of size `n_asked` x `n_dim`, where `n_dim` is the number of + dimenstion in the search space. """ pass From 74fddd8a8faf33f4645e73252c4d3f698e8cc598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Mon, 9 Dec 2024 12:52:29 +0100 Subject: [PATCH 22/30] feat: Replaced InititalPointStrategizer with SequentialStrategizer. --- .../test_initial_point_suggestor.py | 85 ------------ ...uggestor.py => test_random_strategizer.py} | 0 .../test_sequential_strategizer.py | 124 ++++++++++++++++++ ProcessOptimizer/tests/test_xpyrimentor.py | 13 +- XpyriMentor/suggestors/__init__.py | 4 +- .../suggestors/initial_points_strategizer.py | 94 ------------- XpyriMentor/suggestors/po_suggestor.py | 2 +- .../suggestors/sequential_strategizer.py | 102 ++++++++++++++ XpyriMentor/suggestors/suggestor_factory.py | 34 ++--- XpyriMentor/xpyrimentor.py | 12 +- 10 files changed, 260 insertions(+), 210 deletions(-) delete mode 100644 ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py rename ProcessOptimizer/tests/test_suggestors/{test_random_suggestor.py => test_random_strategizer.py} (100%) create mode 100644 ProcessOptimizer/tests/test_suggestors/test_sequential_strategizer.py delete mode 100644 XpyriMentor/suggestors/initial_points_strategizer.py create mode 100644 XpyriMentor/suggestors/sequential_strategizer.py diff --git a/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py deleted file mode 100644 index e042d119..00000000 --- a/ProcessOptimizer/tests/test_suggestors/test_initial_point_suggestor.py +++ /dev/null @@ -1,85 +0,0 @@ -import numpy as np -from XpyriMentor.suggestors import ( - suggestor_factory, - InitialPointStrategizer, - DefaultSuggestor, - POSuggestor, - Suggestor, - LHSSuggestor, -) -from ProcessOptimizer.space import space_factory - - -class MockSuggestor: - def __init__(self, suggestions: list): - self.suggestions = suggestions - self.last_input = {} - - def suggest(self, Xi, Yi, n_asked=1): - self.last_input = {"Xi": Xi, "Yi": Yi} - return self.suggestions[:n_asked] - - -def test_initialization(): - space = space_factory([[0, 1], [0, 1]]) - suggestor = InitialPointStrategizer( - initial_suggestor=DefaultSuggestor(space, n_objectives=1, rng=np.random.default_rng(1)), - ultimate_suggestor=DefaultSuggestor(space, n_objectives=1, rng=np.random.default_rng(2)), - ) - assert isinstance(suggestor, Suggestor) - assert suggestor.n_initial_points == 5 - # Initial suggestor is a POSuggestor with more than 5 initial points. This means than - # we will only use the LHS part of this ProcessOptimizer. - assert isinstance(suggestor.initial_suggestor, LHSSuggestor) - assert suggestor.initial_suggestor.n_points == 5 - # Ultimate suggestor is a POSuggestor with no initial points, since - # InitialPointSuggestor handles the initial points. - assert isinstance(suggestor.ultimate_suggestor, POSuggestor) - assert suggestor.ultimate_suggestor.optimizer._n_initial_points == 0 - - -def test_factory(): - space = space_factory([[0, 1], [0, 1]]) - suggestor = suggestor_factory( - space=space, - definition={"name": "InitialPoint", "n_initial_points": 10}, - ) - assert isinstance(suggestor, InitialPointStrategizer) - assert suggestor.n_initial_points == 10 - assert isinstance(suggestor.initial_suggestor, LHSSuggestor) - assert suggestor.initial_suggestor.n_points == 10 - assert isinstance(suggestor.ultimate_suggestor, POSuggestor) - assert suggestor.ultimate_suggestor.optimizer._n_initial_points == 0 - - -def test_suggestor_switch(): - suggestor = InitialPointStrategizer( - initial_suggestor=MockSuggestor([[1]]), - ultimate_suggestor=MockSuggestor([[2]]), - n_initial_points=3, - ) - assert suggestor.suggest([], []) == [[1]] - assert suggestor.suggest([1], []) == [[1]] - assert suggestor.suggest([1, 2], []) == [[1]] - assert suggestor.suggest([1, 2, 3], []) == [[2]] - - -def test_bridging_the_switch(): - suggestor = InitialPointStrategizer( - initial_suggestor=MockSuggestor([[1], [1]]), - ultimate_suggestor=MockSuggestor([[2]]), - n_initial_points=2, - ) - assert all(suggestor.suggest([], [], n_asked=2) == [[1], [1]]) - assert all(suggestor.suggest([1], [], n_asked=2) == [[1], [2]]) - - -def test_multiple_initial(): - suggestor = InitialPointStrategizer( - initial_suggestor=MockSuggestor([[1], [1]]), - ultimate_suggestor=MockSuggestor([[2]]), - n_initial_points=2, - ) - assert suggestor.suggest([], []) == [[1]] - assert suggestor.suggest([1], []) == [[1]] - assert suggestor.suggest([1, 1], []) == [[2]] diff --git a/ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_random_strategizer.py similarity index 100% rename from ProcessOptimizer/tests/test_suggestors/test_random_suggestor.py rename to ProcessOptimizer/tests/test_suggestors/test_random_strategizer.py diff --git a/ProcessOptimizer/tests/test_suggestors/test_sequential_strategizer.py b/ProcessOptimizer/tests/test_suggestors/test_sequential_strategizer.py new file mode 100644 index 00000000..51292d06 --- /dev/null +++ b/ProcessOptimizer/tests/test_suggestors/test_sequential_strategizer.py @@ -0,0 +1,124 @@ +import numpy as np +import pytest +from ProcessOptimizer.space import space_factory +from XpyriMentor.suggestors import ( + DefaultSuggestor, + IncompatibleNumberAsked, + LHSSuggestor, + POSuggestor, + SequentialStrategizer, + Suggestor, + suggestor_factory +) + + +class MockSuggestor: + def __init__(self, suggestions: list): + self.suggestions = suggestions + self.last_input = {} + + def suggest(self, Xi, Yi, n_asked=1): + self.last_input = {"Xi": Xi, "Yi": Yi} + return self.suggestions*n_asked + + +def test_initialization(): + suggestor = SequentialStrategizer( + suggestors=[(5, MockSuggestor([[1]])), (-1, MockSuggestor([[2]]))], + ) + assert suggestor.suggestors[0][0] == 5 + assert suggestor.suggestors[1][0] == float("inf") + assert suggestor.suggestors[0][1].suggest(Xi=None, Yi=None, n_asked=1) == [[1]] + + +def test_protocol(): + suggestor = SequentialStrategizer( + suggestors=[(5, MockSuggestor([[1]])), (-1, MockSuggestor([[2]]))], + ) + assert isinstance(suggestor, Suggestor) + + +def test_factory(): + suggestor = { + "name": "Sequential", + "suggestors": [ + {"suggestor_budget": 5, "suggestor": MockSuggestor([[1]])}, + {"suggestor_budget": -1, "suggestor": MockSuggestor([[2]])}, + ], + } + suggestor = suggestor_factory( + space=None, + definition=suggestor, + ) + assert isinstance(suggestor, SequentialStrategizer) + + +def test_budget(): + suggestor = SequentialStrategizer( + suggestors=[(5, MockSuggestor([[1]])), (-1, MockSuggestor([[2]]))], + ) + assert suggestor.suggestors[0][0] == 5 + assert suggestor.suggestors[1][0] == float("inf") + suggestor = SequentialStrategizer( + suggestors=[(5, MockSuggestor([[1]])), (10, MockSuggestor([[2]]))], + ) + assert suggestor.suggestors[0][0] == 5 + assert suggestor.suggestors[1][0] == 10 + with pytest.raises(IncompatibleNumberAsked): + suggestor.suggest([[1]], [1], n_asked=16) + with pytest.raises(ValueError): + SequentialStrategizer( + suggestors=[(-1, MockSuggestor([[1]])), (5, MockSuggestor([[2]]))], + ) + + +def test_suggest(): + suggestor_1 = MockSuggestor([[1]]) + suggestor_2 = MockSuggestor([[2]]) + suggestor_3 = MockSuggestor([[3]]) + suggestor = SequentialStrategizer( + suggestors=[(3, suggestor_1), (2, suggestor_2), (-1, suggestor_3)], + ) + assert suggestor.suggest([], []) == [[1]] + assert all(suggestor.suggest([], [], n_asked=2) == [[1], [1]]) + assert all(suggestor.suggest([], [], n_asked=3) == [[1], [1], [1]]) + assert all(suggestor.suggest([], [], n_asked=4) == [[1], [1], [1], [2]]) + assert suggestor_1.last_input == {"Xi": [], "Yi": []} + assert suggestor_2.last_input == {"Xi": [], "Yi": []} + assert suggestor.suggest([[1]]*2, [1, 1]) == [[1]] + assert all(suggestor.suggest([[1]]*2, [1, 1], n_asked=2) == [[1], [2]]) + assert all(suggestor.suggest([[1]]*2, [1, 1], n_asked=4) == [[1], [2], [2], [3]]) + assert suggestor_1.last_input == {"Xi": [[1], [1]], "Yi": [1, 1]} + assert suggestor_2.last_input == {"Xi": [[1], [1]], "Yi": [1, 1]} + assert suggestor_3.last_input == {"Xi": [[1], [1]], "Yi": [1, 1]} + assert suggestor.suggest([[1]]*100, [1]*100) == [[3]] + assert suggestor_3.last_input == {"Xi": [[1]]*100, "Yi": [1]*100} + + +def test_default_suggestors(): + rng = np.random.default_rng(1) + space = space_factory([[0, 1], [0, 1]]) + suggestor = SequentialStrategizer( + suggestors=[ + (3, DefaultSuggestor(space=space, n_objectives=1, rng=rng)), + (2, DefaultSuggestor(space=space, n_objectives=1, rng=rng)), + (-1, DefaultSuggestor(space=space, n_objectives=2, rng=rng)) + ], + ) + assert isinstance(suggestor.suggestors[0][1], LHSSuggestor) + assert suggestor.suggestors[0][1].n_points == 3 + assert isinstance(suggestor.suggestors[1][1], POSuggestor) + assert suggestor.suggestors[1][1].optimizer.n_objectives == 1 + assert isinstance(suggestor.suggestors[2][1], POSuggestor) + assert suggestor.suggestors[2][1].optimizer.n_objectives == 2 + + +def test_incompatible_n_points(): + class NPointsSuggestor: + def __init__(self, n_points): + self.n_points = n_points + + def suggest(self, Xi, Yi, n_asked=1): + return [[self.n_points]]*n_asked + with pytest.warns(UserWarning): + SequentialStrategizer(suggestors=[(5, NPointsSuggestor(10))]) diff --git a/ProcessOptimizer/tests/test_xpyrimentor.py b/ProcessOptimizer/tests/test_xpyrimentor.py index 2643275a..20a6ebc0 100644 --- a/ProcessOptimizer/tests/test_xpyrimentor.py +++ b/ProcessOptimizer/tests/test_xpyrimentor.py @@ -1,6 +1,6 @@ import pytest from XpyriMentor.xpyrimentor import XpyriMentor -from XpyriMentor.suggestors import InitialPointStrategizer, POSuggestor, LHSSuggestor +from XpyriMentor.suggestors import POSuggestor, LHSSuggestor, SequentialStrategizer class MockSuggestor: @@ -18,12 +18,11 @@ def test_initialization(): exp = XpyriMentor(space) assert exp.Xi == [] assert exp.yi == [] - assert isinstance(exp.suggestor, InitialPointStrategizer) - assert exp.suggestor.n_initial_points == 5 - assert isinstance(exp.suggestor.initial_suggestor, LHSSuggestor) - assert exp.suggestor.initial_suggestor.n_points == 5 - assert isinstance(exp.suggestor.ultimate_suggestor, POSuggestor) - assert exp.suggestor.ultimate_suggestor.optimizer._n_initial_points == 0 + assert isinstance(exp.suggestor, SequentialStrategizer) + assert isinstance(exp.suggestor.suggestors[0][1], LHSSuggestor) + assert exp.suggestor.suggestors[0][0] == 5 + assert isinstance(exp.suggestor.suggestors[1][1], POSuggestor) + assert exp.suggestor.suggestors[1][0] == float("inf") def test_tell_single_objective(): diff --git a/XpyriMentor/suggestors/__init__.py b/XpyriMentor/suggestors/__init__.py index c1796645..af821f18 100644 --- a/XpyriMentor/suggestors/__init__.py +++ b/XpyriMentor/suggestors/__init__.py @@ -1,18 +1,18 @@ from .default_suggestor import DefaultSuggestor -from .initial_points_strategizer import InitialPointStrategizer from .lhs_suggestor import LHSSuggestor from .po_suggestor import POSuggestor from .random_strategizer import RandomStragegizer +from .sequential_strategizer import SequentialStrategizer from .suggestor import IncompatibleNumberAsked, Suggestor from .suggestor_factory import suggestor_factory __all__ = [ "DefaultSuggestor", "IncompatibleNumberAsked", - "InitialPointStrategizer", "LHSSuggestor", "POSuggestor", "RandomStragegizer", + "SequentialStrategizer", "Suggestor", "suggestor_factory", ] diff --git a/XpyriMentor/suggestors/initial_points_strategizer.py b/XpyriMentor/suggestors/initial_points_strategizer.py deleted file mode 100644 index 5201ff5c..00000000 --- a/XpyriMentor/suggestors/initial_points_strategizer.py +++ /dev/null @@ -1,94 +0,0 @@ -import logging -from typing import Iterable - -import numpy as np - -from .default_suggestor import DefaultSuggestor -from .lhs_suggestor import LHSSuggestor -from .po_suggestor import POSuggestor -from .suggestor import Suggestor - -logger = logging.getLogger(__name__) - - -class InitialPointStrategizer(): - """ - A strategizer that uses one suggestor for a fixed number of initial points and then - switches to another suggestor. - - Be careful when using a suggestor that suggest multiple points at once as the - inital suggestor. If it suggests more points than the number of remaining initial - points, its suggestions will be truncated so it only returns the remaining initial - points. This can be a problem with design of experiments suggestors, for example, as - all of their suggested points are needed for the analysis. - """ - def __init__( - self, - initial_suggestor: Suggestor, - ultimate_suggestor: Suggestor, - n_initial_points: int = 5, - ): - """ - Parameters - ---------- - initial_suggestor : Suggestor - The suggestor to use for the initial points. Default is a POSuggestor with - n_initial_points initial points. - ultimate_suggestor : Suggestor - The suggestor to use after the initial points have been suggested. Default - is a POSuggestor with no initial points. - n_initial_points : int - The number of initial points to suggest with the initial suggestor. Default - is 5. - """ - if isinstance(initial_suggestor, DefaultSuggestor): - # If the initial suggestor is the default suggestor, we replace it with a - # POSuggestor. It should be a much simpler suggestor, e.g. a Latin Hypercube - # Sampling suggestor. Replace the POSuggestor when available. - logger.debug( - "Initial suggestor is DefaultSuggestor, replacing with " - "cached LHSSuggestor." - ) - initial_suggestor = LHSSuggestor( - space=initial_suggestor.space, - rng=initial_suggestor.rng, - n_points=n_initial_points, - ) - if isinstance(ultimate_suggestor, DefaultSuggestor): - logger.debug( - "Ultimate suggestor is DefaultSuggestor, replacing with POSuggestor." - ) - ultimate_suggestor = POSuggestor( - ultimate_suggestor.space, n_initial_points=0, rng=ultimate_suggestor.rng - ) - self.n_initial_points = n_initial_points - self.initial_suggestor = initial_suggestor - self.ultimate_suggestor = ultimate_suggestor - - def suggest(self, Xi: Iterable[Iterable], Yi: Iterable, n_asked: int = 1) -> np.ndarray: - initial_points_left = max(self.n_initial_points - len(Xi), 0) - n_initial_points = min(n_asked, initial_points_left) - n_ultimate_points = n_asked - n_initial_points - suggestions = [] - if n_initial_points > 0: - suggestions.extend(self.initial_suggestor.suggest(Xi, Yi, n_initial_points)) - if n_ultimate_points > 0: - suggestions.extend(self.ultimate_suggestor.suggest( - Xi, Yi, n_ultimate_points) - ) - return np.array(suggestions, dtype=object) - - def __str__(self): - return ( - f"InitialPointStrategizer with a {self.initial_suggestor.__class__.__name__} " - f"as inital suggestor and a {self.ultimate_suggestor.__class__.__name__} as " - "ultimate suggestor" - ) - - def __repr__(self): - return ( - f"InitialPointStrategizer(" - f"initial_suggestor={self.initial_suggestor.__class__.__name__}(...), " - f"ultimate_suggestor={self.ultimate_suggestor.__class__.__name__}(...), " - f"n_initial_points={self.n_initial_points})" - ) diff --git a/XpyriMentor/suggestors/po_suggestor.py b/XpyriMentor/suggestors/po_suggestor.py index ae5336b5..25a548ff 100644 --- a/XpyriMentor/suggestors/po_suggestor.py +++ b/XpyriMentor/suggestors/po_suggestor.py @@ -42,7 +42,7 @@ def suggest( point, ) if n_asked == 1: - # PO returns a singe point as a list, so we wrap it in another list to + # PO returns a single point as a list, so we wrap it in another list to # maintain the same interface as the other suggestors. return [point] else: diff --git a/XpyriMentor/suggestors/sequential_strategizer.py b/XpyriMentor/suggestors/sequential_strategizer.py new file mode 100644 index 00000000..49d52ecb --- /dev/null +++ b/XpyriMentor/suggestors/sequential_strategizer.py @@ -0,0 +1,102 @@ +import math +import warnings +from typing import Iterable + +import numpy as np + +from .default_suggestor import DefaultSuggestor +from .lhs_suggestor import LHSSuggestor +from .po_suggestor import POSuggestor +from .suggestor import IncompatibleNumberAsked, Suggestor + + +class SequentialStrategizer(): + """ + Stratgizer that uses a sequence of suggestors, each with a budget of suggestions to + make. + + It uses the suggestors in order, skipping suggestors with a budget of suggestions + that have already been made. + """ + def __init__(self, suggestors: list[tuple[int, Suggestor]]): + """ + Initialize the strategizer with a list of suggestors and their budgets. + + Parameters + ---------- + * suggestors [`list[tuple[int, Suggestor]]`]: + A list of tuples where the first element is the number of suggestions the + suggestor can make and the second element is the suggestor. A negative number + of suggestions is interpreted as infinity, aand can only be used as the last + budget. + + If the first suggestor is a DefaultSuggestor, it will be replaced with a + LHSSuggestor with the same budget. If any other suggestor is a + DefaultSuggestor, it will be replaced with a POSuggestor. + """ + for n, (budget, suggestor) in enumerate(suggestors): + if isinstance(suggestor, DefaultSuggestor): + if n == 0: + suggestors[n] = ( + budget, + LHSSuggestor( + space=suggestor.space, rng=suggestor.rng, n_points=budget + ) + ) + else: + suggestors[n] = ( + budget, + POSuggestor( + space=suggestor.space, + rng=suggestor.rng, + n_objectives=suggestor.n_objectives, + )) + if budget <= 0: + # Interpret negative budgets as infinity + suggestors[n] = (float("inf"), suggestors[n][1]) + if hasattr(suggestors[n][1], "n_points"): + if suggestors[n][1].n_points != suggestors[n][0]: + warnings.warn( + f"Budget of {suggestors[n][0]} points does not match number of " + "points for suggestor of type " + f"{suggestors[n][1].__class__.__name__}." + ) + if math.isinf(suggestors[n][0]): + if n < len(suggestors) - 1: + raise ValueError("Infinite budget must be the last budget") + self.suggestors = suggestors + + def suggest(self, Xi: Iterable[Iterable], Yi: Iterable, n_asked: int = 1): + number_to_skip = len(Xi) + number_left_to_find = n_asked + suggestions = [] + for budget, suggestor in self.suggestors: + if number_left_to_find == 0: + break + if number_to_skip >= budget: + number_to_skip -= budget + continue + if number_to_skip + number_left_to_find >= budget: + suggestions.extend(suggestor.suggest(Xi, Yi, budget - number_to_skip)) + number_left_to_find -= budget - number_to_skip + number_to_skip = 0 + else: + suggestions.extend(suggestor.suggest(Xi, Yi, number_left_to_find)) + number_left_to_find = 0 + + if len(suggestions) < n_asked: + raise IncompatibleNumberAsked("Not enough suggestions") + return np.array(suggestions, dtype = object) + + def __str__(self): + return "Sequential Strategizer with suggestors: " + ", ".join( + suggestor.__class__.__name__ for _, suggestor in self.suggestors + ) + + def __repr__(self): + suggestor_list_str = ", ".join( + f"({budget}, {suggestor.__class__.__name__}(...))" for budget, suggestor in self.suggestors + ) + return ( + f"SequentialStrategizer(suggestors=[{suggestor_list_str}]" + ) diff --git a/XpyriMentor/suggestors/suggestor_factory.py b/XpyriMentor/suggestors/suggestor_factory.py index 57439a9b..6f74b7cd 100644 --- a/XpyriMentor/suggestors/suggestor_factory.py +++ b/XpyriMentor/suggestors/suggestor_factory.py @@ -5,10 +5,10 @@ from ProcessOptimizer.space import Space from .default_suggestor import DefaultSuggestor -from .initial_points_strategizer import InitialPointStrategizer from .lhs_suggestor import LHSSuggestor from .po_suggestor import POSuggestor from .random_strategizer import RandomStragegizer +from .sequential_strategizer import SequentialStrategizer from .suggestor import Suggestor logger = logging.getLogger(__name__) @@ -57,20 +57,6 @@ def suggestor_factory( if suggestor_type == "Default" or suggestor_type is None: logger.debug("Creating DefaultSuggestor") return DefaultSuggestor(space, n_objectives, rng) - elif suggestor_type == "InitialPoint": - logger.debug("Creating InitialPointSuggestor") - # If either of the necessary keys are missing, the default values are used. - initial_suggestor = definition.get("initial_suggestor", None) - ultimate_suggestor = definition.get("ultimate_suggestor", None) - return InitialPointStrategizer( - initial_suggestor=suggestor_factory( - space, initial_suggestor, n_objectives, rng - ), - ultimate_suggestor=suggestor_factory( - space, ultimate_suggestor, n_objectives, rng - ), - n_initial_points=definition["n_initial_points"], - ) elif suggestor_type == "PO": logger.debug("Creating POSuggestor") return POSuggestor( @@ -101,11 +87,27 @@ def suggestor_factory( )) return RandomStragegizer(suggestors=suggestors, rng=rng) elif suggestor_type == "LHS": - logger.debug("Creating a cached LHSSuggestor.") + logger.debug("Creating a LHSSuggestor.") return LHSSuggestor( space=space, rng=rng, **definition, ) + elif suggestor_type == "Sequential": + logger.debug("Creating SequentialStrategizer") + suggestors = [] + for suggestor in definition["suggestors"]: + n = suggestor.pop("suggestor_budget") + if "suggestor" in suggestor: + if len(suggestor) > 1: + raise ValueError( + "If a suggestor definition for a SequentialStrategizer has a " + "'suggestor' key, it should only have that key and " + "'suggestor_budget', but it has the keys `suggestor_budget`, " + f"{', '.join(suggestor.keys())}." + ) + suggestor = suggestor["suggestor"] + suggestors.append((n, suggestor_factory(space, suggestor, n_objectives, rng))) + return SequentialStrategizer(suggestors) else: raise ValueError(f"Unknown suggestor name: {suggestor_type}") diff --git a/XpyriMentor/xpyrimentor.py b/XpyriMentor/xpyrimentor.py index 4c98da0a..e42dae48 100644 --- a/XpyriMentor/xpyrimentor.py +++ b/XpyriMentor/xpyrimentor.py @@ -1,3 +1,4 @@ +import copy import logging from typing import Any, Iterable, Union import warnings @@ -12,10 +13,11 @@ logger = logging.getLogger(__name__) DEFAULT_SUGGESTOR = { - "name": "InitialPoint", - "initial_suggestor": {"name": "Default"}, - "ultimate_suggestor": {"name": "Default"}, - "n_initial_points": 5 + "name": "Sequential", + "suggestors": [ + {"suggestor_budget": 5, "name": "Default"}, + {"suggestor_budget": -1, "name": "Default"}, + ], } @@ -46,7 +48,7 @@ def __init__( if isinstance(suggestor, DefaultSuggestor): logger.debug("Replacing DefaultSuggestor with InitialPointSuggestor") suggestor = suggestor_factory( - space, DEFAULT_SUGGESTOR.copy(), n_objectives, rng=rng + space, copy.deepcopy(DEFAULT_SUGGESTOR), n_objectives, rng=rng ) if isinstance(suggestor, POSuggestor): warnings.warn( From 57bf808094b541654447c9e91d7861f56c768cdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Mon, 9 Dec 2024 12:53:25 +0100 Subject: [PATCH 23/30] chore: Renamed jupyter notebook --- examples/{expyrementor.ipynb => XpyriMentor.ipynb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/{expyrementor.ipynb => XpyriMentor.ipynb} (100%) diff --git a/examples/expyrementor.ipynb b/examples/XpyriMentor.ipynb similarity index 100% rename from examples/expyrementor.ipynb rename to examples/XpyriMentor.ipynb From ae58e2f05ffbc13127b52e4d9f6a30f27a5f4a08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Mon, 9 Dec 2024 13:15:34 +0100 Subject: [PATCH 24/30] doc: Updated Xpyrimentor notebook to use SequentialStrategizer --- .../test_random_strategizer.py | 12 +- XpyriMentor/suggestors/suggestor_factory.py | 8 +- examples/XpyriMentor.ipynb | 125 +++++++++--------- 3 files changed, 69 insertions(+), 76 deletions(-) diff --git a/ProcessOptimizer/tests/test_suggestors/test_random_strategizer.py b/ProcessOptimizer/tests/test_suggestors/test_random_strategizer.py index bffa94a5..bf92e720 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_random_strategizer.py +++ b/ProcessOptimizer/tests/test_suggestors/test_random_strategizer.py @@ -42,8 +42,8 @@ def test_factory(): suggestor = suggestor_factory( space=space, definition={"name": "Random", "suggestors": [ - {"usage_ratio": 0.8, "name": "PO"}, - {"usage_ratio": 0.2, "name": "LHS"},]}, + {"suggestor_usage_ratio": 0.8, "name": "PO"}, + {"suggestor_usage_ratio": 0.2, "name": "LHS"},]}, ) assert isinstance(suggestor, RandomStragegizer) assert len(suggestor.suggestors) == 2 @@ -99,8 +99,8 @@ def test_random_with_suggestor_given(): suggestor = suggestor_factory( space=space, definition={"name": "Random", "suggestors": [ - {"usage_ratio": 0.8, "suggestor": MockSuggestor([[1]])}, - {"usage_ratio": 0.2, "suggestor": MockSuggestor([[2]])},]}, + {"suggestor_usage_ratio": 0.8, "suggestor": MockSuggestor([[1]])}, + {"suggestor_usage_ratio": 0.2, "suggestor": MockSuggestor([[2]])},]}, ) assert isinstance(suggestor, RandomStragegizer) assert len(suggestor.suggestors) == 2 @@ -116,9 +116,9 @@ def test_random_with_suggestor_given_wrong_keys(): suggestor_factory( space=space, definition={"name": "Random", "suggestors": [ - {"usage_ratio": 0.8, "suggestor": MockSuggestor([[1]])}, + {"suggestor_usage_ratio": 0.8, "suggestor": MockSuggestor([[1]])}, { - "usage_ratio": 0.2, + "suggestor_usage_ratio": 0.2, "suggestor": MockSuggestor([[2]]), "additional_key": "Can't have this key", },]}, diff --git a/XpyriMentor/suggestors/suggestor_factory.py b/XpyriMentor/suggestors/suggestor_factory.py index 6f74b7cd..3e0c5b6f 100644 --- a/XpyriMentor/suggestors/suggestor_factory.py +++ b/XpyriMentor/suggestors/suggestor_factory.py @@ -37,9 +37,9 @@ def suggestor_factory( placeholder in strategizers, and should be replaced with a real suggestor before use. - The keys `name`, `usage_ratio`, and `suggestor` are reserved and should not be used - in __init__ of suggestors. They might be removed from the definition dict before - passing it to the suggestor. + The keys `name`, `suggestor` and any keyword starting with `suggestor_` are reserved + and should not be used in __init__ of suggestors. They might be removed from the + definition dict before passing it to the suggestor. """ if isinstance(definition, Suggestor): return definition @@ -69,7 +69,7 @@ def suggestor_factory( logger.debug("Creating RandomStrategizer") suggestors = [] for suggestor in definition["suggestors"]: - usage_ratio = suggestor.pop("usage_ratio") + usage_ratio = suggestor.pop("suggestor_usage_ratio") # Note that we are removing the key usage_ratio from the suggestor # definition. If any suggestor uses this key, it will have to be redefined in # the suggestor definition. diff --git a/examples/XpyriMentor.ipynb b/examples/XpyriMentor.ipynb index 3dbd9daa..bf1dcd02 100644 --- a/examples/XpyriMentor.ipynb +++ b/examples/XpyriMentor.ipynb @@ -18,11 +18,11 @@ "name": "stdout", "output_type": "stream", "text": [ - "XpyriMentor with a InitialPointStrategizer suggestor\n", - "[[90.10000000000001 170 'dog']]\n", - "[[50.5 110 'fish']]\n", - "[[70.3 130 'cat']\n", - " [30.7 150 'cat']]\n" + "XpyriMentor with a SequentialStrategizer suggestor\n", + "[[90.10000000000001 190 'fish']]\n", + "[[50.5 150 'dog']]\n", + "[[70.3 170 'fish']\n", + " [30.7 130 'cat']]\n" ] } ], @@ -52,10 +52,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "However, we can also make more complicated suggestors. As an example, we are going to make\n", - "a suggestor that starts with a 3 point Latin Hypercube Sampling, and after that\n", - "randomly chooses an exploring Optimizer (xi = 10) 20% of the time, ans an exploiting\n", - "Optimizer (xi = 0.00001) 80% of the time." + "However, we can also make more complicated suggestors. As an example, let's assume we have\n", + "a total experiment budget of 30 points. We would like to start with a 7 point Latin\n", + "Hypercube sampling, then split the next 20 between an exploring Optimizer (80% likelyhood,\n", + "`xi = 10`) and an exploiting Optimizer (20% likelyhood, `xi = 0.00001`), and use the last\n", + "experiments with confirming the result by using an exploiting Optimizer ( `xi = 0.00001`)" ] }, { @@ -67,37 +68,34 @@ "name": "stdout", "output_type": "stream", "text": [ - "[[95.05 135 'dog']\n", - " [15.85 105 'dog']\n", - " [85.14999999999999 115 'cat']\n", - " [33.99800434958006 104 'dog']\n", - " [3.429739342486463 120 'fish']\n", - " [83.55809954099502 105 'cat']\n", - " [12.47231893940161 170 'dog']\n", - " [49.818785128244784 114 'cat']\n", - " [91.57420742670749 149 'dog']\n", - " [21.859534469711146 136 'cat']]\n" + "[[78.78571428571428 179 'fish']\n", + " [8.071428571428571 107 'cat']\n", + " [2.819163230542185 101 'fish']\n", + " [72.5913762383257 114 'cat']\n", + " [15.47387860203124 172 'cat']\n", + " [86.40148481213906 143 'fish']\n", + " [43.333608793143426 186 'cat']\n", + " [56.15180835969205 105 'dog']\n", + " [65.42827321276842 152 'fish']\n", + " [12.56621714545117 176 'cat']]\n" ] } ], "source": [ "suggestor_definition = {\n", - " \"name\": \"InitialPoint\",\n", - " \"n_initial_points\": 10,\n", - " \"initial_suggestor\": {\n", - " \"name\": \"LHS\", \"n_points\": 10\n", - " },\n", - " \"ultimate_suggestor\": {\n", - " \"name\": \"Random\",\n", - " \"suggestors\": [\n", - " {\"usage_ratio\": 20, \"name\": \"PO\", \"acq_func_kwargs\": {\"xi\": 10}},\n", - " {\"usage_ratio\": 80, \"name\": \"PO\", \"acq_func_kwargs\": {\"xi\": 0.00001}},\n", - " ]\n", - " }\n", + " \"name\": \"Sequential\",\n", + " \"suggestors\": [\n", + " {\"suggestor_budget\": 7, \"name\": \"LHS\", \"n_points\": 7},\n", + " {\"suggestor_budget\": 20, \"name\": \"Random\", \"suggestors\": [\n", + " {\"suggestor_usage_ratio\": 80, \"name\": \"PO\", \"acq_func_kwargs\": {\"xi\": 10}},\n", + " {\"suggestor_usage_ratio\": 20, \"name\": \"PO\", \"acq_func_kwargs\": {\"xi\": 0.00001}},\n", + " ]},\n", + " {\"suggestor_budget\": 20, \"name\": \"PO\", \"acq_func_kwargs\": {\"xi\": 0.00001}},\n", + " ]\n", "}\n", "director = XpyriMentor(space, suggestor_definition)\n", - "intial_suggestions = director.ask(7)\n", - "director.tell(intial_suggestions, [0.5, 0.6, -0.7, 0.8, 0.9, 1.0, 1.1])\n", + "intial_suggestions = director.ask(5)\n", + "director.tell(intial_suggestions, [0.5, 0.6, -0.7, 0.8, 0.9])\n", "print(director.ask(10))" ] }, @@ -115,7 +113,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -123,9 +121,9 @@ "output_type": "stream", "text": [ "ConstantSuggestor is a Suggestor: True\n", - "[[83.5 116 'dog']\n", - " [50.5 184 'fish']\n", - " [17.5 150 'cat']\n", + "[[83.5 184 'fish']\n", + " [50.5 150 'dog']\n", + " [17.5 116 'cat']\n", " [50.5 150 'dog']\n", " [50.5 150 'dog']\n", " [90.10000000000001 190 'fish']\n", @@ -154,18 +152,14 @@ "space = space_factory([[1.0, 100.0], [100, 200], [\"cat\", \"dog\", \"fish\"]])\n", "\n", "suggestor_definition = {\n", - " \"name\": \"InitialPoint\",\n", - " \"n_initial_points\": 3,\n", - " \"initial_suggestor\": {\n", - " \"name\": \"LHS\", \"n_points\": 3\n", - " },\n", - " \"ultimate_suggestor\": {\n", - " \"name\": \"Random\",\n", - " \"suggestors\": [\n", - " {\"usage_ratio\": 30, \"suggestor\": ConstantSuggestor(space)},\n", - " {\"usage_ratio\": 70, \"suggestor\": ConstantSuggestor(space, constant=0.9)},\n", - " ]\n", - " }\n", + " \"name\": \"Sequential\",\n", + " \"suggestors\": [\n", + " {\"suggestor_budget\": 3, \"name\": \"LHS\", \"n_points\": 3},\n", + " {\"suggestor_budget\": 20, \"name\": \"Random\", \"suggestors\": [\n", + " {\"suggestor_usage_ratio\": 30, \"suggestor\": ConstantSuggestor(space)},\n", + " {\"suggestor_usage_ratio\": 70, \"suggestor\": ConstantSuggestor(space, constant=0.9)},\n", + " ]},\n", + " ]\n", "}\n", "\n", "director = XpyriMentor(space, suggestor_definition)\n", @@ -176,45 +170,44 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Initial Points Suggestor has the option of uding default suggestors for either the\n", - "suggestor used to suggest the initial points (initiali suggestor), or the suggestor used\n", - "after the intial points are used up (ultimate suggestor), or both.\n", + "Sequential Strategizer has the option of using default suggestors for any suggestor.\n", "\n", - "The default initial suggestor is a Lating Hypercube Sampling suggestor with as many points\n", - "as the Initial Point Suggestor will use it.\n", + "The default firts suggestor is a Latin Hypercube Sampling suggestor with as many points\n", + "as the budget for that suggestor.\n", "\n", - "The default ultimate suggestor is a ProcessOptimizer Optimizer suggestor with no initial\n", - "points (the Initial Point Suggestor takes care of the initial points).\n", + "The default suggestor for all other suggestors is a ProcessOptimizer Optimizer suggestor\n", + "with no initial points.\n", "\n" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Director is an XpyriMentor with a InitialPointStrategizer suggestor.\n", - "Its suggestor is a InitialPointStrategizer with a LHSSuggestor as inital suggestor and a POSuggestor as ultimate suggestor.\n", - "More specifically, it has a Latin Hypercube Suggestor with 11 points as intial suggestor and a ProcessOptimizer Suggestor as ultimate suggestor.\n" + "Director is an XpyriMentor with a SequentialStrategizer suggestor.\n", + "Its suggestor is a Sequential Strategizer with suggestors: LHSSuggestor, POSuggestor.\n", + "More specifically, it has a Latin Hypercube Suggestor with 3 points as intial suggestor and a ProcessOptimizer Suggestor as ultimate suggestor.\n" ] } ], "source": [ "space = space_factory([[1.0, 100.0], [100, 200], [\"cat\", \"dog\", \"fish\"]])\n", "suggestor_definition = {\n", - " \"name\": \"InitialPoint\",\n", - " \"n_initial_points\": 11,\n", - " \"initial_suggestor\": {\"name\": \"Default\"},\n", - " \"ultimate_suggestor\": {\"name\": \"Default\"},\n", - " }\n", + " \"name\": \"Sequential\",\n", + " \"suggestors\":[\n", + " {\"suggestor_budget\": 3, \"name\": \"Default\"},\n", + " {\"suggestor_budget\": 20, \"name\": \"Default\"},\n", + " ]\n", + "}\n", "director = XpyriMentor(space, suggestor_definition)\n", "print(f\"Director is an {director}.\")\n", "print(f\"Its suggestor is a {director.suggestor}.\")\n", - "print(f\"More specifically, it has a {director.suggestor.initial_suggestor} as intial suggestor and a {director.suggestor.ultimate_suggestor} as ultimate suggestor.\")" + "print(f\"More specifically, it has a {director.suggestor.suggestors[0][1]} as intial suggestor and a {director.suggestor.suggestors[1][1]} as ultimate suggestor.\")" ] } ], From dd68eefdae537b2f3613a0de4965b6ee54f76617 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Tue, 10 Dec 2024 09:58:45 +0100 Subject: [PATCH 25/30] feat: SequentialStrategizer passes budget to its suggestors, LHSSuggestor uses the budget as default n_points --- .../test_sequential_strategizer.py | 13 ++++++++++ XpyriMentor/suggestors/default_suggestor.py | 13 +++++++++- XpyriMentor/suggestors/suggestor.py | 4 ++++ XpyriMentor/suggestors/suggestor_factory.py | 12 ++++++---- examples/XpyriMentor.ipynb | 24 ++++++++++++------- 5 files changed, 51 insertions(+), 15 deletions(-) diff --git a/ProcessOptimizer/tests/test_suggestors/test_sequential_strategizer.py b/ProcessOptimizer/tests/test_suggestors/test_sequential_strategizer.py index 51292d06..218b8a84 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_sequential_strategizer.py +++ b/ProcessOptimizer/tests/test_suggestors/test_sequential_strategizer.py @@ -122,3 +122,16 @@ def suggest(self, Xi, Yi, n_asked=1): return [[self.n_points]]*n_asked with pytest.warns(UserWarning): SequentialStrategizer(suggestors=[(5, NPointsSuggestor(10))]) + + +def test_default_n_points(): + space = space_factory([[0, 1], [0, 1]]) + suggestor_definition = { + "name": "Sequential", + "suggestors": [{"suggestor_budget": 7, "name": "LHS"}], + } + suggestor = suggestor_factory( + space=space, + definition=suggestor_definition, + ) + assert suggestor.suggestors[0][1].n_points == 7 diff --git a/XpyriMentor/suggestors/default_suggestor.py b/XpyriMentor/suggestors/default_suggestor.py index 37e6e388..fe940185 100644 --- a/XpyriMentor/suggestors/default_suggestor.py +++ b/XpyriMentor/suggestors/default_suggestor.py @@ -1,3 +1,5 @@ +from typing import Iterable + import numpy as np from ProcessOptimizer.space import Space @@ -15,7 +17,16 @@ def __init__(self, space: Space, n_objectives: int, rng: np.random.Generator, ** self.n_objectives = n_objectives self.rng = rng - def suggest(self, Xi: list[list], Yi: list, n_asked: int = -1) -> np.ndarray: + def suggest( + self, Xi: Iterable[Iterable], Yi: Iterable, n_asked: int = -1 + ) -> np.ndarray: + """ + SHOULD NOT BE CALLED! + + DefaultSuggestor only exists to act as a placeholder to be replaced with the + appropriate Suggestor. If its suggest method is called, this has not happened, + which is an error. + """ raise NoDefaultSuggestorError("Default suggestor should not be used.") diff --git a/XpyriMentor/suggestors/suggestor.py b/XpyriMentor/suggestors/suggestor.py index bca5b0d4..8aa00b15 100644 --- a/XpyriMentor/suggestors/suggestor.py +++ b/XpyriMentor/suggestors/suggestor.py @@ -15,6 +15,10 @@ def __init__(self, **kwargs): """ Initialize the suggestor with the search space. Suggestors can take other input arguments as needed. + + The input keys `name`, `suggestor` and any keyword starting with `suggestor_` + are reserved and should not be used in __init__ of suggestors. They might be + removed from the definition dict before passing it to the suggestor. """ pass diff --git a/XpyriMentor/suggestors/suggestor_factory.py b/XpyriMentor/suggestors/suggestor_factory.py index 3e0c5b6f..4d9733fb 100644 --- a/XpyriMentor/suggestors/suggestor_factory.py +++ b/XpyriMentor/suggestors/suggestor_factory.py @@ -19,6 +19,7 @@ def suggestor_factory( definition: Union[Suggestor, dict[str, Any], None], n_objectives: int = 1, rng: Optional[np.random.Generator] = None, + n_points: Optional[int] = None ) -> Suggestor: """ Create a suggestor from a definition dictionary. @@ -36,10 +37,6 @@ def suggestor_factory( If definition is None, a DefaultSuggestor is created. This is useful as a placeholder in strategizers, and should be replaced with a real suggestor before use. - - The keys `name`, `suggestor` and any keyword starting with `suggestor_` are reserved - and should not be used in __init__ of suggestors. They might be removed from the - definition dict before passing it to the suggestor. """ if isinstance(definition, Suggestor): return definition @@ -87,6 +84,8 @@ def suggestor_factory( )) return RandomStragegizer(suggestors=suggestors, rng=rng) elif suggestor_type == "LHS": + if "n_points" not in definition and n_points is not None: + definition["n_points"] = n_points logger.debug("Creating a LHSSuggestor.") return LHSSuggestor( space=space, @@ -107,7 +106,10 @@ def suggestor_factory( f"{', '.join(suggestor.keys())}." ) suggestor = suggestor["suggestor"] - suggestors.append((n, suggestor_factory(space, suggestor, n_objectives, rng))) + suggestors.append(( + n, + suggestor_factory(space, suggestor, n_objectives, rng, n_points = n) + )) return SequentialStrategizer(suggestors) else: raise ValueError(f"Unknown suggestor name: {suggestor_type}") diff --git a/examples/XpyriMentor.ipynb b/examples/XpyriMentor.ipynb index bf1dcd02..34a56514 100644 --- a/examples/XpyriMentor.ipynb +++ b/examples/XpyriMentor.ipynb @@ -56,7 +56,9 @@ "a total experiment budget of 30 points. We would like to start with a 7 point Latin\n", "Hypercube sampling, then split the next 20 between an exploring Optimizer (80% likelyhood,\n", "`xi = 10`) and an exploiting Optimizer (20% likelyhood, `xi = 0.00001`), and use the last\n", - "experiments with confirming the result by using an exploiting Optimizer ( `xi = 0.00001`)" + "experiments with confirming the result by using an exploiting Optimizer ( `xi = 0.00001`).\n", + "We set the budget of that last suggestor to be infinite, so we wil keep using that even if\n", + "we move beyond our 30 point experiment budget." ] }, { @@ -85,12 +87,16 @@ "suggestor_definition = {\n", " \"name\": \"Sequential\",\n", " \"suggestors\": [\n", - " {\"suggestor_budget\": 7, \"name\": \"LHS\", \"n_points\": 7},\n", - " {\"suggestor_budget\": 20, \"name\": \"Random\", \"suggestors\": [\n", - " {\"suggestor_usage_ratio\": 80, \"name\": \"PO\", \"acq_func_kwargs\": {\"xi\": 10}},\n", - " {\"suggestor_usage_ratio\": 20, \"name\": \"PO\", \"acq_func_kwargs\": {\"xi\": 0.00001}},\n", - " ]},\n", - " {\"suggestor_budget\": 20, \"name\": \"PO\", \"acq_func_kwargs\": {\"xi\": 0.00001}},\n", + " {\"suggestor_budget\": 7, \"name\": \"LHS\"},\n", + " {\n", + " \"suggestor_budget\": 20,\n", + " \"name\": \"Random\",\n", + " \"suggestors\": [\n", + " {\"suggestor_usage_ratio\": 80, \"name\": \"PO\", \"acq_func_kwargs\": {\"xi\": 10}},\n", + " {\"suggestor_usage_ratio\": 20, \"name\": \"PO\", \"acq_func_kwargs\": {\"xi\": 0.00001}},\n", + " ],\n", + " },\n", + " {\"suggestor_budget\": float(\"inf\"), \"name\": \"PO\", \"acq_func_kwargs\": {\"xi\": 0.00001}},\n", " ]\n", "}\n", "director = XpyriMentor(space, suggestor_definition)\n", @@ -154,7 +160,7 @@ "suggestor_definition = {\n", " \"name\": \"Sequential\",\n", " \"suggestors\": [\n", - " {\"suggestor_budget\": 3, \"name\": \"LHS\", \"n_points\": 3},\n", + " {\"suggestor_budget\": 3, \"name\": \"LHS\"},\n", " {\"suggestor_budget\": 20, \"name\": \"Random\", \"suggestors\": [\n", " {\"suggestor_usage_ratio\": 30, \"suggestor\": ConstantSuggestor(space)},\n", " {\"suggestor_usage_ratio\": 70, \"suggestor\": ConstantSuggestor(space, constant=0.9)},\n", @@ -182,7 +188,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 4, "metadata": {}, "outputs": [ { From 3c59437b767c5303f503165ddd780bec805f0b30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Tue, 10 Dec 2024 10:03:45 +0100 Subject: [PATCH 26/30] doc: Comments in SequentialStrategizer suggest method describe why suggstors are handled they way they are --- XpyriMentor/suggestors/sequential_strategizer.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/XpyriMentor/suggestors/sequential_strategizer.py b/XpyriMentor/suggestors/sequential_strategizer.py index 49d52ecb..6754c6bb 100644 --- a/XpyriMentor/suggestors/sequential_strategizer.py +++ b/XpyriMentor/suggestors/sequential_strategizer.py @@ -72,15 +72,22 @@ def suggest(self, Xi: Iterable[Iterable], Yi: Iterable, n_asked: int = 1): suggestions = [] for budget, suggestor in self.suggestors: if number_left_to_find == 0: + # If we have already found all the points we need, we can stop. break if number_to_skip >= budget: + # If we have been told enough points, we have to skip this suggestor. number_to_skip -= budget continue if number_to_skip + number_left_to_find >= budget: - suggestions.extend(suggestor.suggest(Xi, Yi, budget - number_to_skip)) - number_left_to_find -= budget - number_to_skip + # If we need more points than the suggestor can give us, we take all the + # points the suggestor can give us and continue with the next suggestor. + number_from_this_suggestor = budget - number_to_skip + suggestions.extend(suggestor.suggest(Xi, Yi, number_from_this_suggestor)) + number_left_to_find -= number_from_this_suggestor number_to_skip = 0 else: + # If we need fewer points than the suggestor can give us, we take the + # points we need and stop. suggestions.extend(suggestor.suggest(Xi, Yi, number_left_to_find)) number_left_to_find = 0 From 0a94fc81cc8779551a04fd3bb6669e9934050023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Tue, 10 Dec 2024 10:26:00 +0100 Subject: [PATCH 27/30] fix: POSuggestor returns a numpy.ndarray --- XpyriMentor/suggestors/po_suggestor.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/XpyriMentor/suggestors/po_suggestor.py b/XpyriMentor/suggestors/po_suggestor.py index 25a548ff..6ca72afc 100644 --- a/XpyriMentor/suggestors/po_suggestor.py +++ b/XpyriMentor/suggestors/po_suggestor.py @@ -41,12 +41,7 @@ def suggest( yi, point, ) - if n_asked == 1: - # PO returns a single point as a list, so we wrap it in another list to - # maintain the same interface as the other suggestors. - return [point] - else: - return point + return np.array(point, dtype = object).reshape(n_asked,-1) def __str__(self): return "ProcessOptimizer Suggestor" From 47845e396b1f46bd811056a73241e681c1f915f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Wed, 11 Dec 2024 09:40:08 +0100 Subject: [PATCH 28/30] chore: "name" -> "suggestor_name" --- .../test_suggestors/test_lhs_suggestor.py | 2 +- .../test_suggestors/test_po_suggestor.py | 2 +- .../test_random_strategizer.py | 8 ++-- .../test_sequential_strategizer.py | 6 +-- ProcessOptimizer/tests/test_xpyrimentor.py | 2 +- XpyriMentor/suggestors/suggestor_factory.py | 4 +- XpyriMentor/xpyrimentor.py | 6 +-- examples/XpyriMentor.ipynb | 40 ++++++++++++------- 8 files changed, 41 insertions(+), 29 deletions(-) diff --git a/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py index 632befc9..35abd913 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py +++ b/ProcessOptimizer/tests/test_suggestors/test_lhs_suggestor.py @@ -18,7 +18,7 @@ def test_factory(): space = space_factory([[1, 2], [1, 2]]) suggestor = suggestor_factory( space=space, - definition={"name": "LHS", "n_points": 10}, + definition={"suggestor_name": "LHS", "n_points": 10}, ) assert isinstance(suggestor, LHSSuggestor) assert suggestor.n_points == 10 diff --git a/ProcessOptimizer/tests/test_suggestors/test_po_suggestor.py b/ProcessOptimizer/tests/test_suggestors/test_po_suggestor.py index 569b0c0b..3b1253a4 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_po_suggestor.py +++ b/ProcessOptimizer/tests/test_suggestors/test_po_suggestor.py @@ -18,7 +18,7 @@ def test_factory(): space = space_factory([[0, 1], [0, 1]]) suggestor = suggestor_factory( space=space, - definition={"name": "PO"}, + definition={"suggestor_name": "PO"}, ) assert isinstance(suggestor, POSuggestor) assert suggestor.optimizer._n_initial_points == 0 diff --git a/ProcessOptimizer/tests/test_suggestors/test_random_strategizer.py b/ProcessOptimizer/tests/test_suggestors/test_random_strategizer.py index bf92e720..16821bbd 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_random_strategizer.py +++ b/ProcessOptimizer/tests/test_suggestors/test_random_strategizer.py @@ -41,9 +41,9 @@ def test_factory(): space = space_factory([[0, 1], [0, 1]]) suggestor = suggestor_factory( space=space, - definition={"name": "Random", "suggestors": [ - {"suggestor_usage_ratio": 0.8, "name": "PO"}, - {"suggestor_usage_ratio": 0.2, "name": "LHS"},]}, + definition={"suggestor_name": "Random", "suggestors": [ + {"suggestor_usage_ratio": 0.8, "suggestor_name": "PO"}, + {"suggestor_usage_ratio": 0.2, "suggestor_name": "LHS"},]}, ) assert isinstance(suggestor, RandomStragegizer) assert len(suggestor.suggestors) == 2 @@ -98,7 +98,7 @@ def test_random_with_suggestor_given(): space = space_factory([[0, 1], [0, 1]]) suggestor = suggestor_factory( space=space, - definition={"name": "Random", "suggestors": [ + definition={"suggestor_name": "Random", "suggestors": [ {"suggestor_usage_ratio": 0.8, "suggestor": MockSuggestor([[1]])}, {"suggestor_usage_ratio": 0.2, "suggestor": MockSuggestor([[2]])},]}, ) diff --git a/ProcessOptimizer/tests/test_suggestors/test_sequential_strategizer.py b/ProcessOptimizer/tests/test_suggestors/test_sequential_strategizer.py index 218b8a84..aa383dde 100644 --- a/ProcessOptimizer/tests/test_suggestors/test_sequential_strategizer.py +++ b/ProcessOptimizer/tests/test_suggestors/test_sequential_strategizer.py @@ -40,7 +40,7 @@ def test_protocol(): def test_factory(): suggestor = { - "name": "Sequential", + "suggestor_name": "Sequential", "suggestors": [ {"suggestor_budget": 5, "suggestor": MockSuggestor([[1]])}, {"suggestor_budget": -1, "suggestor": MockSuggestor([[2]])}, @@ -127,8 +127,8 @@ def suggest(self, Xi, Yi, n_asked=1): def test_default_n_points(): space = space_factory([[0, 1], [0, 1]]) suggestor_definition = { - "name": "Sequential", - "suggestors": [{"suggestor_budget": 7, "name": "LHS"}], + "suggestor_name": "Sequential", + "suggestors": [{"suggestor_budget": 7, "suggestor_name": "LHS"}], } suggestor = suggestor_factory( space=space, diff --git a/ProcessOptimizer/tests/test_xpyrimentor.py b/ProcessOptimizer/tests/test_xpyrimentor.py index 20a6ebc0..19594374 100644 --- a/ProcessOptimizer/tests/test_xpyrimentor.py +++ b/ProcessOptimizer/tests/test_xpyrimentor.py @@ -88,4 +88,4 @@ def test_ask_multiple(): def test_warning_if_raw_POSuggestor(): with pytest.warns(UserWarning): - XpyriMentor([[0, 1], [0, 1]], suggestor={"name": "PO"}) + XpyriMentor([[0, 1], [0, 1]], suggestor={"suggestor_name": "PO"}) diff --git a/XpyriMentor/suggestors/suggestor_factory.py b/XpyriMentor/suggestors/suggestor_factory.py index 4d9733fb..ad13fe73 100644 --- a/XpyriMentor/suggestors/suggestor_factory.py +++ b/XpyriMentor/suggestors/suggestor_factory.py @@ -46,10 +46,10 @@ def suggestor_factory( logger.debug("Creating DefaultSuggestor") return DefaultSuggestor(space, n_objectives, rng) try: - suggestor_type = definition.pop("name") + suggestor_type = definition.pop("suggestor_name") except KeyError as e: raise ValueError( - f"Missing 'name' key in suggestor definition: {definition}" + f"Missing 'suggestor_name' key in suggestor definition: {definition}" ) from e if suggestor_type == "Default" or suggestor_type is None: logger.debug("Creating DefaultSuggestor") diff --git a/XpyriMentor/xpyrimentor.py b/XpyriMentor/xpyrimentor.py index e42dae48..70e91cfc 100644 --- a/XpyriMentor/xpyrimentor.py +++ b/XpyriMentor/xpyrimentor.py @@ -13,10 +13,10 @@ logger = logging.getLogger(__name__) DEFAULT_SUGGESTOR = { - "name": "Sequential", + "suggestor_name": "Sequential", "suggestors": [ - {"suggestor_budget": 5, "name": "Default"}, - {"suggestor_budget": -1, "name": "Default"}, + {"suggestor_budget": 5, "suggestor_name": "Default"}, + {"suggestor_budget": -1, "suggestor_name": "Default"}, ], } diff --git a/examples/XpyriMentor.ipynb b/examples/XpyriMentor.ipynb index 34a56514..40555f91 100644 --- a/examples/XpyriMentor.ipynb +++ b/examples/XpyriMentor.ipynb @@ -63,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -85,18 +85,30 @@ ], "source": [ "suggestor_definition = {\n", - " \"name\": \"Sequential\",\n", + " \"suggestor_name\": \"Sequential\",\n", " \"suggestors\": [\n", - " {\"suggestor_budget\": 7, \"name\": \"LHS\"},\n", + " {\"suggestor_budget\": 7, \"suggestor_name\": \"LHS\"},\n", " {\n", " \"suggestor_budget\": 20,\n", - " \"name\": \"Random\",\n", + " \"suggestor_name\": \"Random\",\n", " \"suggestors\": [\n", - " {\"suggestor_usage_ratio\": 80, \"name\": \"PO\", \"acq_func_kwargs\": {\"xi\": 10}},\n", - " {\"suggestor_usage_ratio\": 20, \"name\": \"PO\", \"acq_func_kwargs\": {\"xi\": 0.00001}},\n", + " {\n", + " \"suggestor_usage_ratio\": 80,\n", + " \"suggestor_name\": \"PO\",\n", + " \"acq_func_kwargs\": {\"xi\": 10},\n", + " },\n", + " {\n", + " \"suggestor_usage_ratio\": 20,\n", + " \"suggestor_name\": \"PO\",\n", + " \"acq_func_kwargs\": {\"xi\": 0.00001},\n", + " },\n", " ],\n", " },\n", - " {\"suggestor_budget\": float(\"inf\"), \"name\": \"PO\", \"acq_func_kwargs\": {\"xi\": 0.00001}},\n", + " {\n", + " \"suggestor_budget\": float(\"inf\"),\n", + " \"suggestor_name\": \"PO\",\n", + " \"acq_func_kwargs\": {\"xi\": 0.00001}\n", + " },\n", " ]\n", "}\n", "director = XpyriMentor(space, suggestor_definition)\n", @@ -119,7 +131,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -158,10 +170,10 @@ "space = space_factory([[1.0, 100.0], [100, 200], [\"cat\", \"dog\", \"fish\"]])\n", "\n", "suggestor_definition = {\n", - " \"name\": \"Sequential\",\n", + " \"suggestor_name\": \"Sequential\",\n", " \"suggestors\": [\n", - " {\"suggestor_budget\": 3, \"name\": \"LHS\"},\n", - " {\"suggestor_budget\": 20, \"name\": \"Random\", \"suggestors\": [\n", + " {\"suggestor_budget\": 3, \"suggestor_name\": \"LHS\"},\n", + " {\"suggestor_budget\": 20, \"suggestor_name\": \"Random\", \"suggestors\": [\n", " {\"suggestor_usage_ratio\": 30, \"suggestor\": ConstantSuggestor(space)},\n", " {\"suggestor_usage_ratio\": 70, \"suggestor\": ConstantSuggestor(space, constant=0.9)},\n", " ]},\n", @@ -204,10 +216,10 @@ "source": [ "space = space_factory([[1.0, 100.0], [100, 200], [\"cat\", \"dog\", \"fish\"]])\n", "suggestor_definition = {\n", - " \"name\": \"Sequential\",\n", + " \"suggestor_name\": \"Sequential\",\n", " \"suggestors\":[\n", - " {\"suggestor_budget\": 3, \"name\": \"Default\"},\n", - " {\"suggestor_budget\": 20, \"name\": \"Default\"},\n", + " {\"suggestor_budget\": 3, \"suggestor_name\": \"Default\"},\n", + " {\"suggestor_budget\": 20, \"suggestor_name\": \"Default\"},\n", " ]\n", "}\n", "director = XpyriMentor(space, suggestor_definition)\n", From b1fdd2a03e950e4f794e6582edf8217ee524bd0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Wed, 11 Dec 2024 09:44:00 +0100 Subject: [PATCH 29/30] doc: Removed mention of "name" as reserved keyword in suggestors --- XpyriMentor/suggestors/suggestor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/XpyriMentor/suggestors/suggestor.py b/XpyriMentor/suggestors/suggestor.py index 8aa00b15..e9086164 100644 --- a/XpyriMentor/suggestors/suggestor.py +++ b/XpyriMentor/suggestors/suggestor.py @@ -16,9 +16,9 @@ def __init__(self, **kwargs): Initialize the suggestor with the search space. Suggestors can take other input arguments as needed. - The input keys `name`, `suggestor` and any keyword starting with `suggestor_` - are reserved and should not be used in __init__ of suggestors. They might be - removed from the definition dict before passing it to the suggestor. + The input key `suggestor` and any keyword starting with `suggestor_` are + reserved and should not be used in __init__ of suggestors. They might be removed + from the definition dict by the factory before passing it to the suggestor. """ pass From 832063d5130120bd73dd4aa7810b3c414698c0cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Furbo?= Date: Thu, 12 Dec 2024 13:27:55 +0100 Subject: [PATCH 30/30] chore: Readability updates from code review. --- .../suggestors/sequential_strategizer.py | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/XpyriMentor/suggestors/sequential_strategizer.py b/XpyriMentor/suggestors/sequential_strategizer.py index 6754c6bb..661f2977 100644 --- a/XpyriMentor/suggestors/sequential_strategizer.py +++ b/XpyriMentor/suggestors/sequential_strategizer.py @@ -26,9 +26,9 @@ def __init__(self, suggestors: list[tuple[int, Suggestor]]): ---------- * suggestors [`list[tuple[int, Suggestor]]`]: A list of tuples where the first element is the number of suggestions the - suggestor can make and the second element is the suggestor. A negative number - of suggestions is interpreted as infinity, aand can only be used as the last - budget. + suggestor can make (its budget) and the second element is the suggestor. A + negative number of suggestions is interpreted as infinity, and can only be + used as budget for the last suggestor. If the first suggestor is a DefaultSuggestor, it will be replaced with a LHSSuggestor with the same budget. If any other suggestor is a @@ -51,7 +51,7 @@ def __init__(self, suggestors: list[tuple[int, Suggestor]]): rng=suggestor.rng, n_objectives=suggestor.n_objectives, )) - if budget <= 0: + if budget < 0: # Interpret negative budgets as infinity suggestors[n] = (float("inf"), suggestors[n][1]) if hasattr(suggestors[n][1], "n_points"): @@ -63,37 +63,39 @@ def __init__(self, suggestors: list[tuple[int, Suggestor]]): ) if math.isinf(suggestors[n][0]): if n < len(suggestors) - 1: - raise ValueError("Infinite budget must be the last budget") + raise ValueError( + "Only the last suggestor can have an infinite budget." + ) self.suggestors = suggestors def suggest(self, Xi: Iterable[Iterable], Yi: Iterable, n_asked: int = 1): - number_to_skip = len(Xi) - number_left_to_find = n_asked + # We will skip as many points as we have already been told about. + number_left_to_skip = len(Xi) # Running tally of points to skip. + number_left_to_find = n_asked # Running tally of points to find. + # Both of these will be decremented as we go through the suggestors. suggestions = [] for budget, suggestor in self.suggestors: - if number_left_to_find == 0: - # If we have already found all the points we need, we can stop. - break - if number_to_skip >= budget: + if number_left_to_skip >= budget: # If we have been told enough points, we have to skip this suggestor. - number_to_skip -= budget + number_left_to_skip -= budget continue - if number_to_skip + number_left_to_find >= budget: + elif number_left_to_skip + number_left_to_find >= budget: # If we need more points than the suggestor can give us, we take all the # points the suggestor can give us and continue with the next suggestor. - number_from_this_suggestor = budget - number_to_skip - suggestions.extend(suggestor.suggest(Xi, Yi, number_from_this_suggestor)) - number_left_to_find -= number_from_this_suggestor - number_to_skip = 0 + number_from_this_suggestor = budget - number_left_to_skip + number_left_to_skip = 0 else: # If we need fewer points than the suggestor can give us, we take the # points we need and stop. - suggestions.extend(suggestor.suggest(Xi, Yi, number_left_to_find)) - number_left_to_find = 0 - + number_from_this_suggestor = number_left_to_find + suggestions.extend(suggestor.suggest(Xi, Yi, number_from_this_suggestor)) + number_left_to_find -= number_from_this_suggestor + if number_left_to_find == 0: + # If we have already found all the points we need, we can stop. + break if len(suggestions) < n_asked: raise IncompatibleNumberAsked("Not enough suggestions") - return np.array(suggestions, dtype = object) + return np.array(suggestions, dtype=object) def __str__(self): return "Sequential Strategizer with suggestors: " + ", ".join(