Run sphere benchmark on GitHub action

.github/workflows/benchmarks.yml

@@ -0,0 +1,78 @@
+name: Run kurobako benchmark
+on:
+  pull_request:
+    paths:
+    - '.github/workflows/benchmarks.yml'
+    - 'cmaes/**.py'
+    - 'benchmark/*.py'
+    - 'benchmark/runner.sh'
+    - 'requirements-bench.txt'
+jobs:
+  benchmark-sphere:
+    name: Run kurobako benchmark
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+          architecture: 'x64'
+      - run: pip install -U pip setuptools
+      - run: pip install -e .
+      - run: pip install --progress-bar off -r requirements-bench.txt
+      - run: pip install --progress-bar off -U git+https://github.com/sile/kurobako-py
+      - run: pip install --progress-bar off -U git+https://github.com/optuna/optuna.git
+      - name: Cache kurobako CLI
+        id: cache-kurobako
+        uses: actions/cache@v3
+        with:
+          path: ./kurobako
+          key: kurobako-0-2-10
+      - name: Download kurobako CLI
+        if: steps.cache-kurobako.outputs.cache-hit != 'true'
+        run: |
+          curl -L https://github.com/optuna/kurobako/releases/download/0.2.10/kurobako-0.2.10.linux-amd64 -o kurobako
+          chmod +x kurobako
+          ./kurobako -h
+      - name: Run benchmark of Sphere function
+        env:
+          KUROBAKO: ./kurobako
+          DIM: 40
+          REPEATS: 10
+          BUDGET: 3000
+        run: ./benchmark/runner.sh sphere ./kurobako-report.json
+      - name: Plot kurobako result
+        uses: c-bata/github-actions-kurobako/plot@master
+        id: kurobako-plot
+        with:
+          ylogscale: true
+          report-json-path: './kurobako-report.json'
+      - name: Generate kurobako markdown report
+        run: cat ./kurobako-report.json | ./kurobako report > ./kurobako-report.md
+
+      - id: 'auth'
+        uses: 'google-github-actions/auth@v2'
+        with:
+          credentials_json: '${{ secrets.GCP_CREDENTIALS }}'
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v2'
+        with:
+          version: '>= 363.0.0'
+          project_id: ${{ secrets.GCP_PROJECT_ID }}
+      - run: gcloud info
+      - run: gsutil cp ${{ steps.kurobako-plot.outputs.image-path }} gs://cmaes-gh-benchmark/${{ github.repository }}/sphere-${{ github.sha }}.png
+      - name: Comment to a pull request
+        uses: c-bata/github-actions-kurobako@master
+        with:
+          report-md-path: './kurobako-report.md'
+          public-image-url: https://storage.googleapis.com/cmaes-gh-benchmark/${{ github.repository }}/sphere-${{ github.sha }}.png
+          title: 'Benchmark of Sphere function'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - run: mv ./kurobako-report.json ./kurobako-report-sphere.json
+      - uses: actions/upload-artifact@v2
+        with:
+          name: kurobako-report
+          path: kurobako-report-sphere.json

benchmark/problem_sphere.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+import sys
+import numpy as np
+
+from kurobako import problem
+from kurobako.problem import Problem
+
+from typing import Optional
+
+
+class SphereEvaluator(problem.Evaluator):
+    def __init__(self, params: list[Optional[float]]):
+        self.n = len(params)
+        self.x = np.array(params, dtype=float)
+        self._current_step = 0
+
+    def evaluate(self, next_step: int) -> list[float]:
+        self._current_step = 1
+        value = self.x**2 / self.n
+        return [value]
+
+    def current_step(self) -> int:
+        return self._current_step
+
+
+class SphereProblem(problem.Problem):
+    def create_evaluator(
+        self, params: list[Optional[float]]
+    ) -> Optional[problem.Evaluator]:
+        return SphereEvaluator(params)
+
+
+class SphereProblemFactory(problem.ProblemFactory):
+    def __init__(self, dim):
+        self.dim = dim
+
+    def create_problem(self, seed: int) -> Problem:
+        return SphereProblem()
+
+    def specification(self) -> problem.ProblemSpec:
+        params = [
+            problem.Var(f"x{i + 1}", problem.ContinuousRange(-5.12, 5.12))
+            for i in range(self.dim)
+        ]
+        return problem.ProblemSpec(
+            name=f"Sphere (dim={self.dim})",
+            params=params,
+            values=[problem.Var("Sphere")],
+        )
+
+
+if __name__ == "__main__":
+    dim = int(sys.argv[1]) if len(sys.argv) == 2 else 2
+    runner = problem.ProblemRunner(SphereProblemFactory(dim))
+    runner.run()

benchmark/runner.sh

@@ -24,6 +24,7 @@ Problem:
     himmelblau     : https://en.wikipedia.org/wiki/Himmelblau%27s_function
     ackley         : https://www.sfu.ca/~ssurjano/ackley.html
     rastrigin      : https://www.sfu.ca/~ssurjano/rastr.html
+    sphere         : https://www.sfu.ca/~ssurjano/spheref.html
     toxic-lightgbm : https://github.com/c-bata/benchmark-warm-starting-cmaes
 
 Options:
@@ -52,6 +53,10 @@ case "$1" in
         # "kurobako problem sigopt --dim 8 rastrigin" only accepts 8-dim.
         PROBLEM=$($KUROBAKO problem command python $DIR/problem_rastrigin.py $DIM)
         ;;
+    sphere)
+        # "kurobako problem sigopt --dim 8 rastrigin" only accepts 8-dim.
+        PROBLEM=$($KUROBAKO problem command python $DIR/problem_sphere.py $DIM)
+        ;;
     toxic-lightgbm)
         PROBLEM=$($KUROBAKO problem warm-starting \
             $($KUROBAKO problem surrogate $SURROGATE_ROOT/wscmaes-toxic-source/) \