Merge pull request #183 from plone/issue-178-robots_txt

Implement a specific robots.txt for Volto sites

news/178.bugfix

@@ -0,0 +1,3 @@
+Implement a specific robots.txt for Volto sites.
+There is an upgrade step which will update the existing plone.robots_txt registry setting unless it has been customized.
+@ericof

src/plone/volto/profiles/default/metadata.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <metadata>
-  <version>1018</version>
+  <version>1019</version>
   <dependencies>
     <dependency>profile-plone.restapi:blocks</dependency>
   </dependencies>

src/plone/volto/profiles/default/registry/plone.base.interfaces.controlpanel.IImagingSchema.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="utf-8"?>
+<registry>
+
+  <!-- Allowed sizes -->
+  <record field="allowed_sizes"
+          interface="plone.base.interfaces.controlpanel.IImagingSchema"
+          name="plone.allowed_sizes"
+          purge="True"
+  >
+    <!-- Purge the existing ones! -->
+    <value>
+      <element>icon 32:32</element>
+      <element>tile 64:64</element>
+      <element>thumb 128:128</element>
+      <element>mini 200:65536</element>
+      <element>preview 400:65536</element>
+      <element>teaser 600:65536</element>
+      <element>large 800:65536</element>
+      <element>larger 1000:65536</element>
+      <element>great 1200:65536</element>
+      <element>huge 1600:65536</element>
+    </value>
+  </record>
+</registry>

src/plone/volto/profiles/default/registry/plone.base.interfaces.controlpanel.INavigationSchema.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="utf-8"?>
+<registry>
+
+  <record field="show_excluded_items"
+          interface="plone.base.interfaces.controlpanel.INavigationSchema"
+          name="plone.show_excluded_items"
+  >
+    <value>False</value>
+  </record>
+
+</registry>

src/plone/volto/profiles/default/registry/plone.base.interfaces.controlpanel.ISearchSchema.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8"?>
+<registry>
+
+  <record field="types_not_searched"
+          interface="plone.base.interfaces.controlpanel.ISearchSchema"
+          name="plone.types_not_searched"
+          purge="true"
+  >
+    <value />
+  </record>
+
+</registry>

src/plone/volto/profiles/default/registry/plone.base.interfaces.controlpanel.ISiteSchema.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="utf-8"?>
+<registry>
+  <record field="robots_txt"
+          interface="plone.base.interfaces.controlpanel.ISiteSchema"
+          name="plone.robots_txt"
+          purge="true"
+  >
+    <value>Sitemap: {portal_url}/sitemap-index.xml
+
+# Define access-restrictions for robots/spiders
+# http://www.robotstxt.org/wc/norobots.html
+
+User-agent: *
+Disallow: /search
+Disallow: /login
+
+# Add Googlebot-specific syntax extension to exclude forms
+# that are repeated for each piece of content in the site
+# the wildcard is only supported by Googlebot
+# http://www.google.com/support/webmasters/bin/answer.py?answer=40367&amp;ctx=sibling
+
+User-Agent: Googlebot
+Disallow: /*login
+Disallow: /*search
+Disallow: /*edit
+</value>
+  </record>
+</registry>

src/plone/volto/profiles/default/registry/plone.base.interfaces.controlpanel.ITypesSchema.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<registry>
+  <record field="types_use_view_action_in_listings"
+          interface="plone.base.interfaces.controlpanel.ITypesSchema"
+          name="plone.types_use_view_action_in_listings"
+          purge="true"
+  >
+    <value />
+  </record>
+</registry>

src/plone/volto/profiles/default/registry/plone.volto.interfaces.IVoltoSettings.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8"?>
+<registry>
+
+  <records interface="plone.volto.interfaces.IVoltoSettings"
+           prefix="volto"
+  />
+
+</registry>

src/plone/volto/profiles/default/registry/registry.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="utf-8"?>
+<registry>
+
+  <record name="plone.icon.volto-settings">
+    <field type="plone.registry.field.TextLine">
+      <title>Plone Icon Volto Control Panel</title>
+    </field>
+    <value key="resource">++plone++plone.volto/volto.svg</value>
+  </record>
+
+</registry>

src/plone/volto/upgrades.py

@@ -156,3 +156,37 @@ def rename_distribution(context):
             report.name = "volto"
         if report.answers.get("distribution") == "default":
             report.answers["distribution"] = "volto"
+
+
+ROBOTS_TXT = """Sitemap: {portal_url}/sitemap-index.xml
+
+# Define access-restrictions for robots/spiders
+# http://www.robotstxt.org/wc/norobots.html
+
+User-agent: *
+Disallow: /search
+Disallow: /login
+
+# Add Googlebot-specific syntax extension to exclude forms
+# that are repeated for each piece of content in the site
+# the wildcard is only supported by Googlebot
+# http://www.google.com/support/webmasters/bin/answer.py?answer=40367&ctx=sibling
+
+User-Agent: Googlebot
+Disallow: /*login
+Disallow: /*search
+Disallow: /*edit
+"""
+
+
+def update_robots_txt(context):
+    from plone.base.interfaces.controlpanel import ROBOTS_TXT as CLASSIC_ROBOTS
+
+    current_value = api.portal.get_registry_record("plone.robots_txt")
+    if current_value == CLASSIC_ROBOTS:
+        api.portal.set_registry_record("plone.robots_txt", ROBOTS_TXT)
+        logger.info("Updated plone.robots_txt registry with sane value.")
+    else:
+        logger.info(
+            "Ignoring plone.robots_txt registry as it was modified in this portal."
+        )

src/plone/volto/upgrades.zcml

@@ -86,4 +86,14 @@
         />
   </genericsetup:upgradeSteps>
 
+  <genericsetup:upgradeSteps
+      profile="plone.volto:default"
+      source="1018"
+      destination="1019"
+      >
+    <genericsetup:upgradeStep
+        title="Update robots_txt setting for Volto"
+        handler=".upgrades.update_robots_txt"
+        />
+  </genericsetup:upgradeSteps>
 </configure>

tests/setup/test_setup_install.py

@@ -16,7 +16,7 @@ def test_browserlayer(self, browser_layers):
 
     def test_latest_version(self, profile_last_version):
         """Test latest version of default profile."""
-        assert profile_last_version(f"{PACKAGE_NAME}:default") == "1018"
+        assert profile_last_version(f"{PACKAGE_NAME}:default") == "1019"
 
     @pytest.mark.parametrize(
         "portal_type,behavior",