[DOP-13779] Add robots.txt for documentation

docs/conf.py

@@ -57,6 +57,7 @@
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
+html_extra_path = ["robots.txt"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.

docs/robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /*/stable/
+Allow: /en/stable/   # Fallback for bots that don't understand wildcards
+Disallow: /