Add a robots.txt file

release-notes.md

@@ -1,6 +1,12 @@
 # Release notes
 
 
+## 3.2.6
+
+* Allow user to GET their own user data
+* Add a robots.txt file and allow indexing only of documentation
+
+
 ## 3.2.5
 
 * Set default access token expiration time to 30 minutes

src/api.py

@@ -5,6 +5,7 @@
 from datetime import timedelta
 import json
 import os
+from textwrap import dedent
 from typing import Annotated, Awaitable, Callable, Union
 
 from fastapi import Depends, FastAPI, HTTPException, status, Header, Request, Response
@@ -118,9 +119,7 @@ async def check_user_agent(
 async def docs_redirect(
     accept: Union[str, None] = Header(default="text/html")
 ) -> RedirectResponse:
-    """
-    Redireciona para a documentação da API.
-    """
+    """Redireciona para a documentação da API."""
 
     if accept == "application/json":
         location = "/openapi.json"
@@ -132,6 +131,24 @@ async def docs_redirect(
     )
 
 
+@app.get("/robots.txt", include_in_schema=False)
+async def robots_txt() -> Response:
+    """Retorna um arquivo robots.txt para orientar crawlers e permitir
+    indexação somente dos caminhos de documentação.
+    """
+    return Response(
+        dedent(
+            """
+        User-agent: *
+        Allow: /docs$
+        Allow: /redoc$
+        Disallow: /
+        """
+        ).lstrip(),
+        media_type="text/plain",
+    )
+
+
 # ## AUTH --------------------------------------------------
 
 

tests/endpoints_test.py

@@ -85,3 +85,18 @@ def test_docs_csp_header(client: Client, path: str):
 
     assert response.status_code == status.HTTP_200_OK
     assert response.headers.get("Content-Security-Policy", None) is not None
+
+
+def test_robots_txt(client: Client):
+    """Testa a presença do arquivo robots.txt
+
+    Args:
+        client (Client): fixture do cliente http.
+    """
+    response = client.get("/robots.txt", headers={"Accept": "text/plain"})
+    content = response.text
+
+    assert response.status_code == status.HTTP_200_OK
+    assert "Allow: /docs$" in content
+    assert "Allow: /redoc$" in content
+    assert "Disallow: /\n" in content