Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Restructure mirror responses #12

Merged
merged 6 commits into from
Jun 30, 2024
Merged

Restructure mirror responses #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6ed3993
Log beatmap requests in better quality
cmyui Jun 30, 2024
2e6dfa6
Better log detail
cmyui Jun 30, 2024
5067495
Remove redundant logs
cmyui Jun 30, 2024
0e78f27
type fixes
cmyui Jun 30, 2024
508874f
unused module
cmyui Jun 30, 2024
d6b05bf
set error message
cmyui Jun 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
201 changes: 71 additions & 130 deletions app/adapters/osu_mirrors/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,77 +71,50 @@ async def fetch_beatmap_zip_data(beatmapset_id: int) -> bytes | None:
return None

num_attempts += 1

started_at = datetime.now()
mirror_response = await mirror.fetch_beatmap_zip_data(beatmapset_id)
ended_at = datetime.now()

mirror_response: BeatmapMirrorResponse[bytes | None] | None = None
try:
mirror_response = await mirror.fetch_beatmap_zip_data(beatmapset_id)
if mirror_response.is_success:
if mirror_response.data is not None and not is_valid_zip_file(
mirror_response.data,
):
raise ValueError("Received bad osz data from mirror")
except Exception as exc:
ended_at = datetime.now()
await beatmap_mirror_requests.create(
request_url=(
mirror_response.request_url if mirror_response else "unavailable"
),
api_key_id=None,
mirror_name=mirror.name,
success=False,
started_at=started_at,
ended_at=ended_at,
response_status_code=(
mirror_response.status_code
if mirror_response and mirror_response.status_code
else None
),
response_size=(
len(mirror_response.data)
if mirror_response and mirror_response.data
else 0
),
response_error=str(exc),
resource=MirrorResource.OSZ_FILE,
)
await OSZ_FILE_MIRROR_SELECTOR.update_all_mirror_and_selector_weights()
logging.warning(
"Failed to fetch beatmapset osz from mirror",
exc_info=True,
extra={
"response": (
{
"url": mirror_response.request_url,
"status_code": mirror_response.status_code,
}
if mirror_response is not None
else None
),
"mirror_name": mirror.name,
"mirror_weight": mirror.weight,
"beatmapset_id": beatmapset_id,
},
)
prev_mirror = mirror
continue
else:
mirror_response.is_success = False
mirror_response.error_message = "Invalid .osz file"

await beatmap_mirror_requests.create(
request_url=mirror_response.request_url or "unavailable",
api_key_id=None,
mirror_name=mirror.name,
success=mirror_response.is_success,
started_at=started_at,
ended_at=ended_at,
response_status_code=mirror_response.status_code,
response_size=len(mirror_response.data) if mirror_response.data else 0,
response_error=mirror_response.error_message,
resource=MirrorResource.OSZ_FILE,
)
await OSZ_FILE_MIRROR_SELECTOR.update_all_mirror_and_selector_weights()

if mirror_response.is_success:
break

ended_at = datetime.now()

await beatmap_mirror_requests.create(
request_url=f"{mirror.base_url}/d/{beatmapset_id}",
api_key_id=None,
mirror_name=mirror.name,
success=True,
started_at=started_at,
ended_at=ended_at,
response_status_code=mirror_response.status_code,
response_size=len(mirror_response.data) if mirror_response.data else 0,
response_error=None,
resource=MirrorResource.OSZ_FILE,
)
await OSZ_FILE_MIRROR_SELECTOR.update_all_mirror_and_selector_weights()
logging.warning(
"Failed to fetch beatmapset osz from mirror",
exc_info=True,
extra={
"response": {
"url": mirror_response.request_url,
"status_code": mirror_response.status_code,
},
"mirror_name": mirror.name,
"mirror_weight": mirror.weight,
"beatmapset_id": beatmapset_id,
},
)
prev_mirror = mirror
continue

ms_elapsed = (ended_at.timestamp() - started_at.timestamp()) * 1000

Expand Down Expand Up @@ -186,75 +159,43 @@ async def fetch_beatmap_background_image(beatmap_id: int) -> bytes | None:
return None

num_attempts += 1
started_at = datetime.now()

mirror_response: BeatmapMirrorResponse[bytes | None] | None = None
try:
mirror_response = await mirror.fetch_beatmap_background_image(
beatmap_id,
)
except Exception as exc:
ended_at = datetime.now()
await beatmap_mirror_requests.create(
request_url=(
mirror_response.request_url if mirror_response else "unavailable"
),
api_key_id=None,
mirror_name=mirror.name,
success=False,
started_at=started_at,
ended_at=ended_at,
response_status_code=(
mirror_response.status_code
if mirror_response and mirror_response.status_code
else None
),
response_size=(
len(mirror_response.data)
if mirror_response and mirror_response.data
else 0
),
response_error=str(exc),
resource=MirrorResource.BACKGROUND_IMAGE,
)
await BACKGROUND_IMAGE_MIRROR_SELECTOR.update_all_mirror_and_selector_weights()
logging.warning(
"Failed to fetch beatmap background image from mirror",
exc_info=True,
extra={
"response": (
{
"url": mirror_response.request_url,
"status_code": mirror_response.status_code,
}
if mirror_response is not None
else None
),
"mirror_name": mirror.name,
"mirror_weight": mirror.weight,
"beatmap_id": beatmap_id,
},
)
prev_mirror = mirror
continue
else:
started_at = datetime.now()
mirror_response = await mirror.fetch_beatmap_background_image(beatmap_id)
ended_at = datetime.now()

await beatmap_mirror_requests.create(
request_url=mirror_response.request_url or "unavailable",
api_key_id=None,
mirror_name=mirror.name,
success=mirror_response.is_success,
started_at=started_at,
ended_at=ended_at,
response_status_code=mirror_response.status_code,
response_size=len(mirror_response.data) if mirror_response.data else 0,
response_error=mirror_response.error_message,
resource=MirrorResource.BACKGROUND_IMAGE,
)
await BACKGROUND_IMAGE_MIRROR_SELECTOR.update_all_mirror_and_selector_weights()

if mirror_response.is_success:
break

ended_at = datetime.now()

await beatmap_mirror_requests.create(
request_url=mirror_response.request_url,
api_key_id=None,
mirror_name=mirror.name,
success=True,
started_at=started_at,
ended_at=ended_at,
response_status_code=mirror_response.status_code,
response_size=len(mirror_response.data) if mirror_response.data else 0,
response_error=None,
resource=MirrorResource.BACKGROUND_IMAGE,
)
await BACKGROUND_IMAGE_MIRROR_SELECTOR.update_all_mirror_and_selector_weights()
logging.warning(
"Failed to fetch beatmap background image from mirror",
exc_info=True,
extra={
"response": {
"url": mirror_response.request_url,
"status_code": mirror_response.status_code,
},
"mirror_name": mirror.name,
"mirror_weight": mirror.weight,
"beatmap_id": beatmap_id,
},
)
prev_mirror = mirror
continue

ms_elapsed = (ended_at.timestamp() - started_at.timestamp()) * 1000

Expand Down
10 changes: 4 additions & 6 deletions app/adapters/osu_mirrors/backends/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,13 @@
T = TypeVar("T", covariant=True)


class MirrorRequestError(Exception):
pass


@dataclass
class BeatmapMirrorResponse(Generic[T]):
data: T
request_url: str
status_code: int
is_success: bool
request_url: str | None
status_code: int | None
error_message: str | None = None


class AbstractBeatmapMirror(ABC):
Expand Down
16 changes: 10 additions & 6 deletions app/adapters/osu_mirrors/backends/gatari.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import logging

import httpx
from typing_extensions import override

from app.adapters.osu_mirrors.backends import AbstractBeatmapMirror
from app.adapters.osu_mirrors.backends import BeatmapMirrorResponse
from app.adapters.osu_mirrors.backends import MirrorRequestError
from app.repositories.beatmap_mirror_requests import MirrorResource


Expand All @@ -18,27 +18,31 @@ async def fetch_beatmap_zip_data(
self,
beatmapset_id: int,
) -> BeatmapMirrorResponse[bytes | None]:
response: httpx.Response | None = None
try:
logging.info(f"Fetching beatmapset osz from gatari: {beatmapset_id}")
response = await self.http_client.get(
f"{self.base_url}/d/{beatmapset_id}",
follow_redirects=True,
)
if response.status_code == 404:
return BeatmapMirrorResponse(
data=None,
is_success=True,
request_url=str(response.request.url),
status_code=response.status_code,
)
response.raise_for_status()
return BeatmapMirrorResponse(
data=response.read(),
is_success=True,
request_url=str(response.request.url),
status_code=response.status_code,
)
except Exception as exc:
logging.warning(
"Failed to fetch beatmap from gatari.pw",
exc_info=True,
return BeatmapMirrorResponse(
data=None,
is_success=False,
request_url=str(response.request.url) if response else None,
status_code=response.status_code if response else None,
error_message=str(exc),
)
raise MirrorRequestError() from exc
26 changes: 19 additions & 7 deletions app/adapters/osu_mirrors/backends/mino.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@

from app.adapters.osu_mirrors.backends import AbstractBeatmapMirror
from app.adapters.osu_mirrors.backends import BeatmapMirrorResponse
from app.adapters.osu_mirrors.backends import MirrorRequestError
from app.repositories.beatmap_mirror_requests import MirrorResource


Expand All @@ -19,36 +18,41 @@ async def fetch_beatmap_zip_data(
self,
beatmapset_id: int,
) -> BeatmapMirrorResponse[bytes | None]:
response: httpx.Response | None = None
try:
logging.info(f"Fetching beatmapset osz from mino: {beatmapset_id}")
response = await self.http_client.get(
f"{self.base_url}/d/{beatmapset_id}",
timeout=httpx.Timeout(None, connect=2),
)
if response.status_code == 404:
return BeatmapMirrorResponse(
data=None,
is_success=True,
request_url=str(response.request.url),
status_code=response.status_code,
)
response.raise_for_status()
return BeatmapMirrorResponse(
data=response.read(),
is_success=True,
request_url=str(response.request.url),
status_code=response.status_code,
)
except Exception as exc:
logging.warning(
"Failed to fetch beatmap from catboy.best",
exc_info=True,
return BeatmapMirrorResponse(
data=None,
is_success=False,
request_url=str(response.request.url) if response else None,
status_code=response.status_code if response else None,
error_message=str(exc),
)
raise MirrorRequestError() from exc

@override
async def fetch_beatmap_background_image(
self,
beatmap_id: int,
) -> BeatmapMirrorResponse[bytes | None]:
response: httpx.Response | None = None
try:
logging.info(f"Fetching beatmap background from mino: {beatmap_id}")
response = await self.http_client.get(
Expand All @@ -58,12 +62,14 @@ async def fetch_beatmap_background_image(
if response.status_code == 404:
return BeatmapMirrorResponse(
data=None,
is_success=True,
request_url=str(response.request.url),
status_code=response.status_code,
)
response.raise_for_status()
return BeatmapMirrorResponse(
data=response.read(),
is_success=True,
request_url=str(response.request.url),
status_code=response.status_code,
)
Expand All @@ -72,4 +78,10 @@ async def fetch_beatmap_background_image(
"Failed to fetch beatmap background from catboy.best",
exc_info=True,
)
raise MirrorRequestError() from exc
return BeatmapMirrorResponse(
data=None,
is_success=False,
request_url=str(response.request.url) if response else None,
status_code=response.status_code if response else None,
error_message=str(exc),
)
Loading