Performance issues caused by the SpammedPages feature #115 (#114)

* changed Spammed pages HQL query to solr
* created a new solrMetadataEntityExtractor to store the number of comments in a document 
* made admintools-default module root
* removed wiki filtering for spammed pages livedata 
* fixed tests
* added tests
* code refactoring

application-admintools-default/pom.xml

@@ -32,6 +32,7 @@
   <description>Admin Tools Application (Pro) - Default</description>
   <properties>
     <xwiki.jacoco.instructionRatio>0.76</xwiki.jacoco.instructionRatio>
+    <xwiki.extension.namespaces>{root}</xwiki.extension.namespaces>
   </properties>
   <dependencies>
     <dependency>
@@ -85,5 +86,10 @@
       <artifactId>application-licensing-licensor-api</artifactId>
       <version>${licensing.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.xwiki.platform</groupId>
+      <artifactId>xwiki-platform-search-solr-api</artifactId>
+      <version>${platform.version}</version>
+    </dependency>
   </dependencies>
 </project>

application-admintools-default/src/main/java/com/xwiki/admintools/internal/AdminToolsManager.java

@@ -27,6 +27,7 @@
 import javax.inject.Provider;
 import javax.inject.Singleton;
 
+import org.apache.solr.common.SolrDocumentList;
 import org.xwiki.component.annotation.Component;
 import org.xwiki.component.manager.ComponentLookupException;
 import org.xwiki.component.manager.ComponentManager;
@@ -146,14 +147,13 @@ public String getInstanceSizeTemplate()
      *
      * @param maxComments maximum number of comments below which the page is ignored.
      * @param filters {@link Map} of filters to be applied on the gathered list.
-     * @param sortColumn target column to apply the sort on.
      * @param order the order of the sort.
-     * @return a {@link List} with the documents that have more than the given number of comments.
+     * @return a {@link SolrDocumentList} with the needed fields set.
      */
-    public List<DocumentReference> getPagesOverGivenNumberOfComments(long maxComments, Map<String, String> filters,
-        String sortColumn, String order)
+    public SolrDocumentList getPagesOverGivenNumberOfComments(long maxComments, Map<String, String> filters,
+        String order)
     {
-        return instanceUsageManager.getSpammedPages(maxComments, filters, sortColumn, order);
+        return instanceUsageManager.getSpammedPages(maxComments, filters, order);
     }
 
     /**

application-admintools-default/src/main/java/com/xwiki/admintools/internal/usage/InstanceUsageManager.java

@@ -29,6 +29,7 @@
 import javax.script.ScriptContext;
 
 import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.solr.common.SolrDocumentList;
 import org.slf4j.Logger;
 import org.xwiki.component.annotation.Component;
 import org.xwiki.model.reference.DocumentReference;
@@ -59,8 +60,6 @@ public class InstanceUsageManager
 
     private static final String TEMPLATE_NAME = "wikiSizeTemplate.vm";
 
-    private static final String WIKI_NAME_KEY = "wikiName";
-
     @Inject
     protected Provider<XWikiContext> xcontextProvider;
 
@@ -158,15 +157,13 @@ public List<WikiSizeResult> getWikisSize(Map<String, String> filters, String sor
      *
      * @param maxComments maximum number of comments below which the document is ignored.
      * @param filters {@link Map} of filters to be applied on the gathered list.
-     * @param sortColumn target column to apply the sort on.
      * @param order the order of the sort.
-     * @return a {@link List} with the documents that have more than the given number of comments.
+     * @return a {@link SolrDocumentList} with the needed fields set.
      */
-    public List<DocumentReference> getSpammedPages(long maxComments, Map<String, String> filters, String sortColumn,
-        String order)
+    public SolrDocumentList getSpammedPages(long maxComments, Map<String, String> filters, String order)
     {
         try {
-            return spamPagesProvider.getDocumentsOverGivenNumberOfComments(maxComments, filters, sortColumn, order);
+            return spamPagesProvider.getDocumentsOverGivenNumberOfComments(maxComments, filters, order);
         } catch (Exception e) {
             logger.warn("There have been issues while gathering wikis spammed pages. Root cause is: [{}]",
                 ExceptionUtils.getRootCauseMessage(e));

application-admintools-default/src/main/java/com/xwiki/admintools/internal/usage/SpamPagesProvider.java

@@ -20,22 +20,22 @@
 package com.xwiki.admintools.internal.usage;
 
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import javax.inject.Inject;
 import javax.inject.Named;
 import javax.inject.Singleton;
 
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocumentList;
 import org.xwiki.component.annotation.Component;
-import org.xwiki.model.reference.DocumentReference;
 import org.xwiki.query.Query;
 import org.xwiki.query.QueryException;
-import org.xwiki.query.QueryFilter;
 import org.xwiki.query.QueryManager;
-import org.xwiki.wiki.descriptor.WikiDescriptor;
-import org.xwiki.wiki.manager.WikiManagerException;
+import org.xwiki.query.SecureQuery;
+import org.xwiki.search.solr.SolrUtils;
 
 /**
  * Provide data for the documents that are spammed.
@@ -44,78 +44,51 @@
  */
 @Component(roles = SpamPagesProvider.class)
 @Singleton
-public class SpamPagesProvider extends AbstractInstanceUsageProvider
+public class SpamPagesProvider
 {
-    @Inject
-    private QueryManager queryManager;
-
-    @Inject
-    @Named("currentlanguage")
-    private QueryFilter currentLanguageFilter;
+    private static final String DESC = "desc";
 
-    @Inject
-    @Named("hidden/document")
-    private QueryFilter hiddenDocumentFilter;
+    private static final Set<String> VALID_SORT_ORDERS = Set.of(DESC, "asc");
 
     @Inject
-    @Named("document")
-    private QueryFilter documentFilter;
+    @Named("secure")
+    private QueryManager secureQueryManager;
 
     @Inject
-    @Named("viewable")
-    private QueryFilter viewableFilter;
+    private SolrUtils solrUtils;
 
     /**
-     * Retrieves the documents that have more than a given number of comments.
+     * Get a list of solr documents in wiki with comments above a given limit.
      *
      * @param maxComments maximum number of comments below which the document is ignored.
-     * @param filters {@link Map} of filters to be applied on the gathered list.
-     * @param sortColumn target column to apply the sort on.
+     * @param filters {@link Map} of filters to be applied on the results list.
      * @param order the order of the sort.
-     * @return a {@link List} with the documents that have more than the given number of comments.
-     */
-    public List<DocumentReference> getDocumentsOverGivenNumberOfComments(long maxComments, Map<String, String> filters,
-        String sortColumn, String order) throws WikiManagerException
-    {
-        Collection<WikiDescriptor> searchedWikis = getRequestedWikis(filters);
-        List<DocumentReference> spammedDocuments = new ArrayList<>();
-        searchedWikis.forEach(wikiDescriptor -> {
-            try {
-                List<DocumentReference> queryResults =
-                    getCommentsForWiki(maxComments, filters.get("docName"), wikiDescriptor.getId());
-                spammedDocuments.addAll(queryResults);
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-        });
-        applyDocumentsSort(spammedDocuments, sortColumn, order);
-        return spammedDocuments;
-    }
-
-    /**
-     * Get the references of documents in wiki with comments above a given limit.
-     *
-     * @param maxComments maximum number of comments below which the document is ignored.
-     * @param searchedDocument document hint to be searched.
-     * @param wikiId the wiki for which the data will be retrieved.
-     * @return a {@link List} with the {@link DocumentReference} of the documents with comments above a given limit.
+     * @return a {@link SolrDocumentList} with the needed fields set.
      * @throws QueryException if there are any exceptions while running the queries for data retrieval.
      */
-    public List<DocumentReference> getCommentsForWiki(long maxComments, String searchedDocument, String wikiId)
-        throws QueryException
+    public SolrDocumentList getDocumentsOverGivenNumberOfComments(long maxComments, Map<String, String> filters,
+        String order) throws Exception
     {
-        String searchString;
-        if (searchedDocument == null || searchedDocument.isEmpty()) {
-            searchString = "%";
-        } else {
-            searchString = String.format("%%%s%%", searchedDocument);
+        String searchedDocument = filters.get("docName");
+        String queryStatement = "*";
+        if (searchedDocument != null && !searchedDocument.isEmpty()) {
+            queryStatement = String.format("title:%s", solrUtils.toCompleteFilterQueryString(searchedDocument));
         }
-        return this.queryManager.createQuery("select obj.name from XWikiDocument as doc, BaseObject as obj "
-                + "where doc.fullName = obj.name and obj.className = 'XWiki.XWikiComments' "
-                + "and lower(doc.title) like lower(:searchString) "
-                + "group by obj.name having count(*) > :maxComments order by count(*) desc", Query.HQL).setWiki(wikiId)
-            .bindValue("maxComments", maxComments).bindValue("searchString", searchString)
-            .addFilter(currentLanguageFilter).addFilter(hiddenDocumentFilter).addFilter(documentFilter)
-            .addFilter(viewableFilter).execute();
+        List<String> filterStatements = new ArrayList<>();
+        filterStatements.add("type:DOCUMENT");
+        filterStatements.add(String.format("AdminTools.NumberOfComments_sortInt:[%d TO *]", maxComments));
+
+        Query query = this.secureQueryManager.createQuery(queryStatement, "solr");
+        if (query instanceof SecureQuery) {
+            ((SecureQuery) query).checkCurrentAuthor(true);
+            ((SecureQuery) query).checkCurrentUser(true);
+        }
+
+        query.bindValue("fl", "title_, reference, wiki, AdminTools.NumberOfComments_sortInt, name, spaces");
+        query.bindValue("fq", filterStatements);
+        query.bindValue("sort",
+            String.format("AdminTools.NumberOfComments_sortInt %s", VALID_SORT_ORDERS.contains(order) ? order : DESC));
+        query.setLimit(100);
+        return ((QueryResponse) query.execute().get(0)).getResults();
     }
 }

application-admintools-default/src/main/java/com/xwiki/admintools/internal/usage/SpamSolrEntityMetadataExtractor.java

@@ -0,0 +1,67 @@
+/*
+ * See the NOTICE file distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this software; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+ */
+package com.xwiki.admintools.internal.usage;
+
+import java.util.List;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+import javax.inject.Singleton;
+
+import org.apache.solr.common.SolrInputDocument;
+import org.slf4j.Logger;
+import org.xwiki.component.annotation.Component;
+import org.xwiki.model.reference.EntityReference;
+import org.xwiki.model.reference.LocalDocumentReference;
+import org.xwiki.search.solr.SolrEntityMetadataExtractor;
+
+import com.xpn.xwiki.doc.XWikiDocument;
+import com.xpn.xwiki.objects.BaseObject;
+
+/**
+ * This extractor retrieves all comment objects associated with the given XWiki document and stores their count in the
+ * Solr index under the field "AdminTools.NumberOfComments_sortInt".
+ *
+ * @version $Id$
+ * @since 1.0.2
+ */
+@Component
+@Named("spammed-doc")
+@Singleton
+public class SpamSolrEntityMetadataExtractor implements SolrEntityMetadataExtractor<XWikiDocument>
+{
+    private static final EntityReference COMMENTSCLASS_REFERENCE = new LocalDocumentReference("XWiki", "XWikiComments");
+
+    @Inject
+    private Logger logger;
+
+    @Override
+    public boolean extract(XWikiDocument entity, SolrInputDocument solrDocument)
+    {
+        try {
+            List<BaseObject> results = entity.getXObjects(COMMENTSCLASS_REFERENCE);
+            solrDocument.setField("AdminTools.NumberOfComments_sortInt", results.size());
+        } catch (Exception e) {
+            this.logger.error("Failed to index the right for document [{}]", entity.getDocumentReference(), e);
+        }
+
+        return true;
+    }
+}

application-admintools-default/src/main/java/com/xwiki/admintools/script/AdminToolsScriptService.java

@@ -26,6 +26,7 @@
 import javax.inject.Named;
 import javax.inject.Singleton;
 
+import org.apache.solr.common.SolrDocumentList;
 import org.xwiki.component.annotation.Component;
 import org.xwiki.component.manager.ComponentLookupException;
 import org.xwiki.job.Job;
@@ -182,17 +183,16 @@ public String getInstanceSizeSection() throws AccessDeniedException
      *
      * @param maxComments maximum number of comments below which the page is ignored.
      * @param filters {@link Map} of filters to be applied on the gathered list.
-     * @param sortColumn target column to apply the sort on.
      * @param order the order of the sort.
-     * @return a {@link List} with the documents that have more than the given number of comments.
+     * @return a {@link SolrDocumentList} with the needed fields set.
      * @since 1.0
      */
     @Unstable
-    public List<DocumentReference> getPagesOverGivenNumberOfComments(long maxComments, Map<String, String> filters,
-        String sortColumn, String order) throws AccessDeniedException
+    public SolrDocumentList getPagesOverGivenNumberOfComments(long maxComments, Map<String, String> filters,
+        String order) throws AccessDeniedException
     {
         this.contextualAuthorizationManager.checkAccess(Right.ADMIN);
-        return adminToolsManager.getPagesOverGivenNumberOfComments(maxComments, filters, sortColumn, order);
+        return adminToolsManager.getPagesOverGivenNumberOfComments(maxComments, filters, order);
     }
 
     /**

application-admintools-default/src/main/resources/META-INF/components.txt

@@ -34,3 +34,4 @@ com.xwiki.admintools.internal.usage.RecycleBinsProvider
 com.xwiki.admintools.internal.usage.UsageDataProvider
 com.xwiki.admintools.internal.usage.SpamPagesProvider
 com.xwiki.admintools.internal.usage.EmptyDocumentsProvider
+com.xwiki.admintools.internal.usage.SpamSolrEntityMetadataExtractor

application-admintools-default/src/test/java/com/xwiki/admintools/internal/usage/InstanceUsageManagerTest.java

@@ -27,6 +27,8 @@
 import javax.inject.Provider;
 import javax.script.ScriptContext;
 
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
@@ -131,6 +133,9 @@ class InstanceUsageManagerTest
     @Mock
     private Licensor licensor;
 
+    @Mock
+    private SolrDocument solrDocument;
+
     @Mock
     private DocumentReference document;
 
@@ -244,23 +249,24 @@ void getWikisSizeError() throws WikiManagerException
     }
 
     @Test
-    void getSpammedPages() throws WikiManagerException
+    void getSpammedPages() throws Exception
     {
-        List<DocumentReference> docs = List.of(document);
-        when(spamPagesProvider.getDocumentsOverGivenNumberOfComments(2, filters, SORT_COLUMN, SORT_ORDER)).thenReturn(
+        SolrDocumentList docs = new SolrDocumentList();
+        docs.add(solrDocument);
+        when(spamPagesProvider.getDocumentsOverGivenNumberOfComments(2, filters, SORT_ORDER)).thenReturn(
             docs);
 
-        assertArrayEquals(docs.toArray(),
-            instanceUsageManager.getSpammedPages(2, filters, SORT_COLUMN, SORT_ORDER).toArray());
+        assertEquals(docs.get(0),
+            instanceUsageManager.getSpammedPages(2, filters, SORT_ORDER).get(0));
     }
 
     @Test
-    void getPagesOverGivenNumberOfCommentsError() throws WikiManagerException
+    void getPagesOverGivenNumberOfCommentsError() throws Exception
     {
-        when(spamPagesProvider.getDocumentsOverGivenNumberOfComments(2, filters, SORT_COLUMN, SORT_ORDER)).thenThrow(
+        when(spamPagesProvider.getDocumentsOverGivenNumberOfComments(2, filters, SORT_ORDER)).thenThrow(
             new RuntimeException("Runtime error"));
         Exception exception = assertThrows(RuntimeException.class,
-            () -> instanceUsageManager.getSpammedPages(2, filters, SORT_COLUMN, SORT_ORDER));
+            () -> instanceUsageManager.getSpammedPages(2, filters, SORT_ORDER));
         assertEquals("java.lang.RuntimeException: Runtime error", exception.getMessage());
         assertEquals(
             "There have been issues while gathering wikis spammed pages. Root cause is: [RuntimeException: Runtime error]",