Add more complex benchmark (#3324)

* Add more complex benchmark

* Reduce number of items

* Just 100

* Reduce to 50

* Fix for older python version

* Fix name

* Xfail a flaky test

noxfile.py

@@ -18,6 +18,7 @@
     "--ignore=tests/mypy",
     "--ignore=tests/pyright",
     "--ignore=tests/cli",
+    "--ignore=tests/benchmarks",
     "--ignore=tests/experimental/pydantic",
 ]
 

tests/benchmarks/schema.py

@@ -0,0 +1,181 @@
+from __future__ import annotations
+
+from enum import Enum
+from typing import List, Union
+from typing_extensions import Annotated
+
+import strawberry
+
+
+@strawberry.enum
+class Role(Enum):
+    ADMIN = "ADMIN"
+    USER = "USER"
+    GUEST = "GUEST"
+
+
+@strawberry.interface
+class Node:
+    id: strawberry.ID
+
+
+@strawberry.type
+class PageInfo:
+    has_next_page: bool
+    has_previous_page: bool
+    start_cursor: str | None
+    end_cursor: str | None
+
+
+@strawberry.type
+class User(Node):
+    id: strawberry.ID
+    username: str
+    email: str
+    role: Role
+
+    @classmethod
+    def random(cls, seed: int) -> User:
+        return User(
+            id=strawberry.ID(str(int)),
+            username=f"username={seed}",
+            email=f"email={seed}",
+            role=Role.ADMIN,
+        )
+
+    @strawberry.field
+    def posts(self, first: int = 10, after: str | None = None) -> PostConnection | None:
+        return PostConnection(
+            edges=[
+                PostEdge(
+                    node=Post.random(i),
+                    cursor=str(i),
+                )
+                for i in range(first)
+            ],
+            page_info=PageInfo(
+                has_next_page=False,
+                has_previous_page=False,
+                start_cursor=None,
+                end_cursor=None,
+            ),
+        )
+
+
+@strawberry.type
+class Post(Node):
+    id: strawberry.ID
+    title: str
+    content: str
+    author: User
+
+    @classmethod
+    def random(cls, seed: int) -> Post:
+        return Post(
+            id=strawberry.ID(str(int)),
+            title=f"title={seed}",
+            content=f"content={seed}",
+            author=User.random(seed),
+        )
+
+    @strawberry.field
+    def comments(
+        self, first: int = 10, after: str | None = None
+    ) -> CommentConnection | None:
+        return CommentConnection(
+            edges=[
+                CommentEdge(
+                    node=Comment.random(i),
+                    cursor=str(i),
+                )
+                for i in range(first)
+            ],
+            page_info=PageInfo(
+                has_next_page=False,
+                has_previous_page=False,
+                start_cursor=None,
+                end_cursor=None,
+            ),
+        )
+
+
+@strawberry.type
+class Comment(Node):
+    id: strawberry.ID
+    text: str
+    author: User
+    post: Post
+
+    @classmethod
+    def random(cls, seed: int) -> Comment:
+        return Comment(
+            id=strawberry.ID(str(int)),
+            text=f"text={seed}",
+            author=User.random(seed),
+            post=Post.random(seed),
+        )
+
+
+@strawberry.type
+class UserConnection:
+    edges: List[UserEdge | None] | None
+    page_info: PageInfo
+
+
+@strawberry.type
+class UserEdge:
+    node: User | None
+    cursor: str
+
+
+@strawberry.type
+class PostConnection:
+    edges: List[PostEdge | None] | None
+    page_info: PageInfo
+
+
+@strawberry.type
+class PostEdge:
+    node: Post | None
+    cursor: str
+
+
+@strawberry.type
+class CommentConnection:
+    edges: List[CommentEdge | None] | None
+    page_info: PageInfo
+
+
+@strawberry.type
+class CommentEdge:
+    node: Comment | None
+    cursor: str
+
+
+SearchResult = Annotated[
+    Union[User, Post, Comment], strawberry.union(name="SearchResult")
+]
+
+
+@strawberry.type
+class Query:
+    users: UserConnection | None
+    posts: PostConnection | None
+    comments: CommentConnection | None
+
+    @strawberry.field
+    async def search(
+        self, query: str, first: int = 10, after: str | None = None
+    ) -> List[SearchResult | None] | None:
+        div = 3
+
+        chunks = [first // div + (1 if x < first % div else 0) for x in range(div)]
+
+        return [
+            *[User.random(i) for i in range(chunks[0])],
+            *[Post.random(i) for i in range(chunks[1])],
+            *[Comment.random(i) for i in range(chunks[2])],
+        ]
+
+
+schema = strawberry.Schema(query=Query)

tests/benchmarks/test_complex_schema.py

@@ -0,0 +1,96 @@
+import pytest
+from asgiref.sync import async_to_sync
+from pytest_codspeed.plugin import BenchmarkFixture
+
+from .schema import schema
+
+query = """
+fragment AuthorFragment on User {
+  id
+  username
+  email
+  role
+}
+
+fragment PostFragment on Post {
+  id
+  content
+  title
+
+  author {
+    ...AuthorFragment
+  }
+
+  comments {
+    edges {
+      node {
+        author {
+          id
+          username
+          email
+          role
+        }
+      }
+    }
+  }
+}
+
+query Query($query: String!, $first: Int!) {
+  search(query: $query, first: $first) {
+    ... on User {
+      id
+      username
+      email
+      role
+      posts {
+        pageInfo {
+          hasNextPage
+          hasPreviousPage
+          startCursor
+          endCursor
+        }
+        edges {
+          cursor
+          node {
+            author {
+              email
+              id
+              posts {
+                edges {
+                  node {
+                    ...PostFragment
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+    ... on Post {
+      ...PostFragment
+    }
+    ... on Comment {
+      id
+      text
+      author {
+        ...AuthorFragment
+      }
+      post {
+        ...PostFragment
+      }
+    }
+  }
+}
+"""
+
+
+@pytest.mark.parametrize("number", [50])
+def test_execute_complex_schema(benchmark: BenchmarkFixture, number: int):
+    result = benchmark(
+        async_to_sync(schema.execute),
+        query,
+        variable_values={"query": "test", "first": number},
+    )
+
+    assert not result.errors

tests/websockets/test_graphql_transport_ws.py

@@ -817,7 +817,7 @@ async def test_rejects_connection_params_with_wrong_type(
 
 # timings can sometimes fail currently.  Until this test is rewritten when
 # generator based subscriptions are implemented, mark it as flaky
-@pytest.mark.flaky
+@pytest.mark.xfail(reason="This test is flaky, see comment above")
 async def test_subsciption_cancel_finalization_delay(ws: WebSocketClient):
     # Test that when we cancel a subscription, the websocket isn't blocked
     # while some complex finalization takes place.