diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 0000000..aff2c57 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,78 @@ +name: Run kurobako benchmark +on: + pull_request: + paths: + - '.github/workflows/benchmarks.yml' + - 'cmaes/**.py' + - 'benchmark/*.py' + - 'benchmark/runner.sh' + - 'requirements-bench.txt' +jobs: + benchmark-sphere: + name: Run kurobako benchmark + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.x' + architecture: 'x64' + - run: pip install -U pip setuptools + - run: pip install -e . + - run: pip install --progress-bar off -r requirements-bench.txt + - run: pip install --progress-bar off -U git+https://github.com/sile/kurobako-py + - run: pip install --progress-bar off -U git+https://github.com/optuna/optuna.git + - name: Cache kurobako CLI + id: cache-kurobako + uses: actions/cache@v3 + with: + path: ./kurobako + key: kurobako-0-2-10 + - name: Download kurobako CLI + if: steps.cache-kurobako.outputs.cache-hit != 'true' + run: | + curl -L https://github.com/optuna/kurobako/releases/download/0.2.10/kurobako-0.2.10.linux-amd64 -o kurobako + chmod +x kurobako + ./kurobako -h + - name: Run benchmark of Sphere function + env: + KUROBAKO: ./kurobako + DIM: 5 + REPEATS: 5 + BUDGET: 500 + run: ./benchmark/runner.sh sphere ./kurobako-report.json + - name: Plot kurobako result + uses: c-bata/github-actions-kurobako/plot@master + id: kurobako-plot + with: + ylogscale: true + report-json-path: './kurobako-report.json' + - name: Generate kurobako markdown report + run: cat ./kurobako-report.json | ./kurobako report > ./kurobako-report.md + + - id: 'auth' + uses: 'google-github-actions/auth@v2' + with: + credentials_json: '${{ secrets.GCP_CREDENTIALS }}' + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v2' + with: + version: '>= 363.0.0' + project_id: ${{ secrets.GCP_PROJECT_ID }} + - run: gcloud info + - run: gsutil cp ${{ steps.kurobako-plot.outputs.image-path }} gs://alg-cmaes-gh-benchmark/${{ github.repository }}/sphere-${{ github.sha }}.png + - name: Comment to a pull request + uses: c-bata/github-actions-kurobako@master + with: + report-md-path: './kurobako-report.md' + public-image-url: https://storage.googleapis.com/alg-cmaes-gh-benchmark/${{ github.repository }}/sphere-${{ github.sha }}.png + title: 'Benchmark of Sphere function' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - run: mv ./kurobako-report.json ./kurobako-report-sphere.json + - uses: actions/upload-artifact@v2 + with: + name: kurobako-report + path: kurobako-report-sphere.json diff --git a/benchmark/problem_sphere.py b/benchmark/problem_sphere.py new file mode 100644 index 0000000..6cb2abb --- /dev/null +++ b/benchmark/problem_sphere.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import sys +import numpy as np + +from kurobako import problem +from kurobako.problem import Problem + +from typing import Optional + + +class SphereEvaluator(problem.Evaluator): + def __init__(self, params: list[Optional[float]]): + self.n = len(params) + self.x = np.array(params, dtype=float) + self._current_step = 0 + + def evaluate(self, next_step: int) -> list[float]: + self._current_step = 1 + value = np.mean(self.x**2) + return [value] + + def current_step(self) -> int: + return self._current_step + + +class SphereProblem(problem.Problem): + def create_evaluator( + self, params: list[Optional[float]] + ) -> Optional[problem.Evaluator]: + return SphereEvaluator(params) + + +class SphereProblemFactory(problem.ProblemFactory): + def __init__(self, dim): + self.dim = dim + + def create_problem(self, seed: int) -> Problem: + return SphereProblem() + + def specification(self) -> problem.ProblemSpec: + params = [ + problem.Var(f"x{i + 1}", problem.ContinuousRange(-5.12, 5.12)) + for i in range(self.dim) + ] + return problem.ProblemSpec( + name=f"Sphere (dim={self.dim})", + params=params, + values=[problem.Var("Sphere")], + ) + + +if __name__ == "__main__": + dim = int(sys.argv[1]) if len(sys.argv) == 2 else 2 + runner = problem.ProblemRunner(SphereProblemFactory(dim)) + runner.run() diff --git a/benchmark/runner.sh b/benchmark/runner.sh index 96b1250..4ccd63b 100755 --- a/benchmark/runner.sh +++ b/benchmark/runner.sh @@ -24,6 +24,7 @@ Problem: himmelblau : https://en.wikipedia.org/wiki/Himmelblau%27s_function ackley : https://www.sfu.ca/~ssurjano/ackley.html rastrigin : https://www.sfu.ca/~ssurjano/rastr.html + sphere : https://www.sfu.ca/~ssurjano/spheref.html toxic-lightgbm : https://github.com/c-bata/benchmark-warm-starting-cmaes Options: @@ -52,6 +53,10 @@ case "$1" in # "kurobako problem sigopt --dim 8 rastrigin" only accepts 8-dim. PROBLEM=$($KUROBAKO problem command python $DIR/problem_rastrigin.py $DIM) ;; + sphere) + # "kurobako problem sigopt --dim 8 rastrigin" only accepts 8-dim. + PROBLEM=$($KUROBAKO problem command python $DIR/problem_sphere.py $DIM) + ;; toxic-lightgbm) PROBLEM=$($KUROBAKO problem warm-starting \ $($KUROBAKO problem surrogate $SURROGATE_ROOT/wscmaes-toxic-source/) \ @@ -76,22 +81,9 @@ IPOP_SEP_CMAES_SOLVER=$($KUROBAKO solver --name 'ipop-sep-cmaes' command -- pyth PYCMA_SOLVER=$($KUROBAKO solver --name 'pycma' command -- python $DIR/optuna_solver.py pycma) WS_CMAES_SOLVER=$($KUROBAKO solver --name 'ws-cmaes' command -- python $DIR/optuna_solver.py ws-cmaes --warm-starting-trials $WARM_START) -if [ $WARM_START -gt 0 ]; then - $KUROBAKO studies \ - --solvers $CMAES_SOLVER $WS_CMAES_SOLVER \ - --problems $PROBLEM \ - --seed $SEED --repeats $REPEATS --budget $BUDGET \ - | $KUROBAKO run --parallelism 6 > $2 -elif [ $BUDGET -le 500 ]; then - $KUROBAKO studies \ - --solvers $RANDOM_SOLVER $PYCMA_SOLVER $CMAES_SOLVER $SEP_CMAES_SOLVER \ - --problems $PROBLEM \ - --seed $SEED --repeats $REPEATS --budget $BUDGET \ - | $KUROBAKO run --parallelism 4 > $2 -else - $KUROBAKO studies \ - --solvers $RANDOM_SOLVER $CMAES_SOLVER $IPOP_SEP_CMAES_SOLVER $IPOP_CMAES_SOLVER $SEP_CMAES_SOLVER \ - --problems $PROBLEM \ - --seed $SEED --repeats $REPEATS --budget $BUDGET \ - | $KUROBAKO run --parallelism 6 > $2 -fi +$KUROBAKO studies \ + --solvers $RANDOM_SOLVER $CMAES_SOLVER $PYCMA_SOLVER \ + --problems $PROBLEM \ + --seed $SEED --repeats $REPEATS --budget $BUDGET \ + | $KUROBAKO run --parallelism 6 > $2 +