More tweaking + BUILD

indexer/scraper.py

@@ -18,7 +18,8 @@ async def thread(id: int) -> dict[str, str] | None:
                 cookies=f95zone.cookies,
             ) as req:
                 if req.status == 429 and retries > 1:
-                    await asyncio.sleep(0.5)
+                    logger.warning("Hit a ratelimit, sleeping 2 seconds")
+                    await asyncio.sleep(2)
                     retries -= 1;
                     continue
                 res = await req.read()

modules/api.py

@@ -70,7 +70,7 @@
 updating = False
 session: aiohttp.ClientSession = None
 webpage_prefix = "F95Checker-Temp-"
-xenforo_ratelimit = aiolimiter.AsyncLimiter(max_rate=1, time_period=1)
+xenforo_ratelimit = aiolimiter.AsyncLimiter(max_rate=1, time_period=2)
 fast_checks_sem: asyncio.Semaphore = None
 full_checks_sem: asyncio.Semaphore = None
 fast_checks_counter = 0
@@ -117,6 +117,8 @@ async def request(method: str, url: str, read=True, no_cookies=False, **kwargs):
     ddos_guard_cookies = {}
     ddos_guard_first_challenge = False
     is_xenforo_request = url.startswith(f95_host)
+    if is_xenforo_request:
+        retries += 5
     while retries:
         try:
             # Only ratelimit when connecting to F95zone
@@ -129,7 +131,7 @@ async def request(method: str, url: str, read=True, no_cookies=False, **kwargs):
                 **kwargs
             ) as req:
                 if is_xenforo_request and req.status == 429 and retries > 1:
-                    await asyncio.sleep(1)
+                    await asyncio.sleep(2)
                     retries -= 1
                     continue
                 if not read: