Merge pull request #916 from algolia/develop

Develop
algolia · Sep 15, 2017 · 2201ddd · 2201ddd
2 parents 79fc7a6 + b18bb5d
commit 2201ddd
Show file tree

Hide file tree

Showing 16 changed files with 269 additions and 74 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,11 @@
 ## CHANGE LOG
 
+### 1.11.1
+
+- Query rules are preserved during reindex with indexing queue enabled (#913)
+- Information about the indexing queue is displayed in admin-wide notifications (#905)
+- Information about queue processing are logged to `algoliasearch_queue_log` DB table (#907)
+
 ### 1.11.0
 
 ## FEATURES

diff --git a/README.md b/README.md
@@ -1,7 +1,7 @@
 Algolia Search for Magento 1.6+
 ==================
 
-![Latest version](https://img.shields.io/badge/latest-1.11.0-green.svg)
+![Latest version](https://img.shields.io/badge/latest-1.11.1-green.svg)
 
 [![Build Status](https://travis-ci.org/algolia/algoliasearch-magento.svg?branch=master)](https://travis-ci.org/algolia/algoliasearch-magento)
 ![PHP >= 5.3](https://img.shields.io/badge/php-%3E=5.3-green.svg)

diff --git a/app/code/community/Algolia/Algoliasearch/Block/Adminhtml/Notifications.php b/app/code/community/Algolia/Algoliasearch/Block/Adminhtml/Notifications.php
@@ -0,0 +1,42 @@
+<?php
+
+class Algolia_Algoliasearch_Block_Adminhtml_Notifications extends Mage_Adminhtml_Block_Template
+{
+    public function getConfigurationUrl()
+    {
+        return $this->getUrl('adminhtml/system_config/edit/section/algoliasearch');
+    }
+
+    public function getQueueInfo()
+    {
+        /** @var Algolia_Algoliasearch_Helper_Config $config */
+        $config = Mage::helper('algoliasearch/config');
+
+        /** @var Mage_Core_Model_Resource $resource */
+        $resource = Mage::getSingleton('core/resource');
+        $tableName = $resource->getTableName('algoliasearch/queue');
+
+        $readConnection = $resource->getConnection('core_read');
+
+        $size = (int)$readConnection->query('SELECT COUNT(*) as total_count FROM '.$tableName)->fetchColumn(0);
+        $maxJobsPerSingleRun = $config->getNumberOfJobToRun();
+
+        $etaMinutes = ceil($size / $maxJobsPerSingleRun) * 5; // 5 - assuming the queue runner runs every 5 minutes
+
+        $eta = $etaMinutes.' minutes';
+        if ($etaMinutes > 60) {
+            $hours = floor($etaMinutes / 60);
+            $restMinutes = $etaMinutes % 60;
+
+            $eta = $hours.' hours '.$restMinutes.' minutes';
+        }
+
+        $queueInfo = array(
+            'isEnabled' => $config->isQueueActive(),
+            'currentSize' => $size,
+            'eta' => $eta,
+        );
+
+        return $queueInfo;
+    }
+}
diff --git a/app/code/community/Algolia/Algoliasearch/Helper/Algoliahelper.php b/app/code/community/Algolia/Algoliasearch/Helper/Algoliahelper.php
@@ -217,6 +217,40 @@ public function copySynonyms($fromIndexName, $toIndexName)
         $this->lastTaskId = $res['taskID'];
     }
 
+    public function copyQueryRules($fromIndexName, $toIndexName)
+    {
+        $fromIndex = $this->getIndex($fromIndexName);
+        $toIndex = $this->getIndex($toIndexName);
+
+        $queryRulesToSet = array();
+
+        $hitsPerPage = 100;
+        $page = 0;
+        do {
+            $fetchedQueryRules = $fromIndex->searchRules(array(
+                'page' => $page,
+                'hitsPerPage' => $hitsPerPage,
+            ));
+
+            foreach ($fetchedQueryRules['hits'] as $hit) {
+                unset($hit['_highlightResult']);
+
+                $queryRulesToSet[] = $hit;
+            }
+
+            $page++;
+        } while (($page * $hitsPerPage) < $fetchedQueryRules['nbHits']);
+
+        if (empty($queryRulesToSet)) {
+            $res = $toIndex->clearRules(true);
+        } else {
+            $res = $toIndex->batchRules($queryRulesToSet, true, true);
+        }
+
+        $this->lastUsedIndexName = $toIndex;
+        $this->lastTaskId = $res['taskID'];
+    }
+
     public function waitLastTask()
     {
         if (!isset($this->lastUsedIndexName) || !isset($this->lastTaskId)) {

diff --git a/app/code/community/Algolia/Algoliasearch/Helper/Entity/Producthelper.php b/app/code/community/Algolia/Algoliasearch/Helper/Entity/Producthelper.php
@@ -406,6 +406,10 @@ public function setSettings($storeId, $saveToTmpIndicesToo = false)
         } elseif ($saveToTmpIndicesToo === true) {
             $this->algolia_helper->copySynonyms($this->getIndexName($storeId), $this->getIndexName($storeId, $saveToTmpIndicesToo));
         }
+
+        if ($saveToTmpIndicesToo === true) {
+            $this->algolia_helper->copyQueryRules($this->getIndexName($storeId), $this->getIndexName($storeId, $saveToTmpIndicesToo));
+        }
     }
 
     protected function getFields($store)

diff --git a/app/code/community/Algolia/Algoliasearch/Model/Queue.php b/app/code/community/Algolia/Algoliasearch/Model/Queue.php
@@ -6,6 +6,7 @@ class Algolia_Algoliasearch_Model_Queue
     const ERROR_LOG = 'algoliasearch_queue_errors.log';
 
     protected $table;
+    protected $logTable;
 
     /** @var Magento_Db_Adapter_Pdo_Mysql */
     protected $db;
@@ -29,12 +30,16 @@ class Algolia_Algoliasearch_Model_Queue
 
     private $noOfFailedJobs = 0;
 
+    private $logRecord = array();
+
     public function __construct()
     {
         /** @var Mage_Core_Model_Resource $coreResource */
         $coreResource = Mage::getSingleton('core/resource');
 
         $this->table = $coreResource->getTableName('algoliasearch/queue');
+        $this->logTable = $this->table.'_log';
+
         $this->db = $coreResource->getConnection('core_write');
 
         $this->config = Mage::helper('algoliasearch/config');
@@ -47,6 +52,7 @@ public function add($class, $method, $data, $data_size)
     {
         // Insert a row for the new job
         $this->db->insert($this->table, array(
+            'created'   => date('Y-m-d H:i:s'),
             'class'     => $class,
             'method'    => $method,
             'data'      => json_encode($data),
@@ -55,19 +61,38 @@ public function add($class, $method, $data, $data_size)
         ));
     }
 
-    public function runCron()
+    public function runCron($nbJobs = null, $force = false)
     {
-        if (!$this->config->isQueueActive()) {
+        if (!$this->config->isQueueActive() && $force === false) {
             return;
         }
 
-        $nbJobs = $this->config->getNumberOfJobToRun();
+        $this->clearOldLogRecords();
+
+        $this->logRecord = array(
+            'started' => date('Y-m-d H:i:s'),
+            'processed_jobs' => 0,
+            'with_empty_queue' => 0,
+        );
+
+        $started = time();
+
+        if ($nbJobs === null) {
+            $nbJobs = $this->config->getNumberOfJobToRun();
+            if (getenv('EMPTY_QUEUE') && getenv('EMPTY_QUEUE') == '1') {
+                $nbJobs = -1;
 
-        if (getenv('EMPTY_QUEUE') && getenv('EMPTY_QUEUE') == '1') {
-            $nbJobs = -1;
+                $this->logRecord['with_empty_queue'] = 1;
+            }
         }
 
         $this->run($nbJobs);
+
+        $this->logRecord['duration'] = time() - $started;
+
+        $this->db->insert($this->logTable, $this->logRecord);
+
+        $this->db->closeConnection();
     }
 
     public function run($maxJobs)
@@ -77,7 +102,7 @@ public function run($maxJobs)
         $jobs = $this->getJobs($maxJobs, $pid);
 
         if (empty($jobs)) {
-            $this->db->closeConnection();
+            return;
         }
 
         // Run all reserved jobs
@@ -96,6 +121,8 @@ public function run($maxJobs)
                 $model = Mage::getSingleton($job['class']);
                 $method = $job['method'];
                 $model->{$method}(new Varien_Object($job['data']));
+
+                $this->logRecord['processed_jobs'] += count($job['merged_ids']);
             } catch (\Exception $e) {
                 $this->noOfFailedJobs++;
 
@@ -119,8 +146,6 @@ public function run($maxJobs)
 
             return;
         }
-
-        $this->db->closeConnection();
     }
 
     private function getJobs($maxJobs, $pid)
@@ -381,4 +406,14 @@ private function maxValueInArray($array, $keyToSearch)
 
         return $currentMax;
     }
+
+    private function clearOldLogRecords()
+    {
+        $idsToDelete = $this->db->query("SELECT id FROM {$this->logTable} ORDER BY started DESC, id DESC LIMIT 25000, ".PHP_INT_MAX)
+                        ->fetchAll(\PDO::FETCH_COLUMN, 0);
+
+        if ($idsToDelete) {
+            $this->db->query("DELETE FROM {$this->logTable} WHERE id IN (" . implode(", ", $idsToDelete) . ")");
+        }
+    }
 }
diff --git a/app/code/community/Algolia/Algoliasearch/etc/config.xml b/app/code/community/Algolia/Algoliasearch/etc/config.xml
@@ -2,7 +2,7 @@
 <config>
     <modules>
         <Algolia_Algoliasearch>
-            <version>1.11.0</version>
+            <version>1.11.1</version>
         </Algolia_Algoliasearch>
     </modules>
     <frontend>

diff --git a/app/code/community/Algolia/Algoliasearch/etc/system.xml b/app/code/community/Algolia/Algoliasearch/etc/system.xml
@@ -4,7 +4,7 @@
         <algoliasearch translate="label" module="algoliasearch">
             <label>
                 <![CDATA[
-                Algolia Search 1.11.0
+                Algolia Search 1.11.1
                 <style>
                     .algoliasearch-admin-menu span {
                         padding-left: 38px !important;

diff --git a/...e/community/Algolia/Algoliasearch/sql/algoliasearch_setup/mysql4-upgrade-1.7.1-1.11.1.php b/...e/community/Algolia/Algoliasearch/sql/algoliasearch_setup/mysql4-upgrade-1.7.1-1.11.1.php
@@ -0,0 +1,21 @@
+<?php
+
+/** @var Mage_Core_Model_Resource_Setup $installer */
+$installer = $this;
+$installer->startSetup();
+
+$tableName = $installer->getTable('algoliasearch/queue');
+$installer->run("ALTER TABLE `{$tableName}` ADD `created` DATETIME AFTER `job_id`;");
+
+$installer->run("
+CREATE TABLE IF NOT EXISTS `{$tableName}_log` (
+  `id` INT(20) NOT NULL auto_increment,
+  `started` DATETIME NOT NULL,
+  `duration` INT(20) NOT NULL,
+  `processed_jobs` INT NOT NULL,
+  `with_empty_queue` INT(1) NOT NULL, 
+  PRIMARY KEY `id` (`id`)
+) ENGINE=InnoDB DEFAULT CHARACTER SET=utf8 AUTO_INCREMENT=1;
+");
+
+$installer->endSetup();
diff --git a/app/design/adminhtml/default/default/layout/algoliasearch.xml b/app/design/adminhtml/default/default/layout/algoliasearch.xml
@@ -1,5 +1,15 @@
 <?xml version="1.0"?>
 <layout>
+    <default>
+        <reference name="head">
+            <action method="addCss"><stylesheet>algoliasearch/algoliasearch.css</stylesheet></action>
+        </reference>
+
+        <reference name="notifications">
+            <block type="algoliasearch/adminhtml_notifications" name="algoliasearch_notifications" template="algoliasearch/notifications.phtml"/>
+        </reference>
+    </default>
+
     <algolia_bundle_handle>
         <reference name="head">
             <action method="addJs"><script>algoliasearch/internals/adminhtml/algoliaAdminBundle.min.js</script></action>

diff --git a/app/design/adminhtml/default/default/template/algoliasearch/notifications.phtml b/app/design/adminhtml/default/default/template/algoliasearch/notifications.phtml
@@ -0,0 +1,30 @@
+<?php
+
+/** @var Algolia_Algoliasearch_Block_Adminhtml_Notifications $this */
+
+$queueInfo = $this->getQueueInfo();
+
+?>
+
+<div class="notification-global algoliasearch-queue-notification"> <!-- TODO FIX the SKIN URL -->
+    <strong>Algolia Search</strong> -
+    <?php if($queueInfo['isEnabled'] === true):?>
+        <strong>
+            <a href="<?php echo $this->getConfigurationUrl(); ?>">Indexing queue</a> information:
+        </strong>
+
+        Number of queued jobs: <strong><?php echo $queueInfo['currentSize']; ?></strong>,
+
+        <?php if ($queueInfo['currentSize'] > 0): ?>
+            all queued jobs will be processed in appr. <strong><?php echo $queueInfo['eta']; ?></strong>,
+        <?php endif; ?>
+
+        more information about how the indexing queue works you can find in the documentation: <a href="https://community.algolia.com/magento/doc/m1/indexing/?utm_source=magento&utm_medium=extension&utm_campaign=magento_1&utm_term=shop-owner&utm_content=doc-link#general-information" target="_blank">Indexing queue</a>
+    <?php else: ?>
+        <strong>
+            <a href="<?php echo $this->getConfigurationUrl(); ?>">Indexing queue</a> is not enabled.
+        </strong>
+
+        It's highly recommended to enable it, especially if you are on production environment. You can learn how to enable the index queue in the documentation: <a href="https://community.algolia.com/magento/doc/m1/indexing/?utm_source=magento&utm_medium=extension&utm_campaign=magento_1&utm_term=shop-owner&utm_content=doc-link#general-information" target="_blank">Indexing queue</a>
+    <?php endif; ?>
+</div>
diff --git a/app/etc/modules/Algolia_Algoliasearch.xml b/app/etc/modules/Algolia_Algoliasearch.xml
@@ -4,7 +4,7 @@
         <Algolia_Algoliasearch>
             <active>true</active>
             <codePool>community</codePool>
-            <version>1.11.0</version>
+            <version>1.11.1</version>
         </Algolia_Algoliasearch>
     </modules>
 </config>
diff --git a/dev/bin/start.sh b/dev/bin/start.sh
@@ -22,12 +22,8 @@ n98-magerun --skip-root-check --root-dir=/var/www/htdocs config:set algoliasearc
 n98-magerun --skip-root-check --root-dir=/var/www/htdocs config:set web/unsecure/base_url $BASE_URL
 n98-magerun --skip-root-check --root-dir=/var/www/htdocs config:set web/secure/base_url $BASE_URL
 
-chmod -R 777 /var/www/htdocs/media
-chown -R www-data:www-data /var/www/htdocs/media
-
 if [ $INSTALL_ALGOLIA == Yes ]; then
-  /root/bin/modman repair --force algoliasearch-magento
-  /root/bin/modman repair --force algoliasearch-magento-extend-module-skeleton
+  /root/bin/modman deploy-all
 
   # reindex whole index
   n98-magerun --skip-root-check --root-dir=/var/www/htdocs index:reindex algolia_search_indexer
@@ -39,7 +35,6 @@ else
   /root/bin/modman undeploy algoliasearch-magento-extend-module-skeleton
 fi
 
-# Again in case root created some folder with root:root
 chmod -R 777 /var/www/htdocs/media
 chown -R www-data:www-data /var/www/htdocs/media
 

diff --git a/modman b/modman
@@ -14,5 +14,6 @@ app/locale/en_US/Algolia_Algoliasearch.csv                          app/locale/e
 app/locale/sv_SE/Algolia_Algoliasearch.csv                          app/locale/sv_SE/Algolia_Algoliasearch.csv
 app/locale/nl_NL/Algolia_Algoliasearch.csv                          app/locale/nl_NL/Algolia_Algoliasearch.csv
 skin/frontend/base/default/algoliasearch/                           skin/frontend/base/default/algoliasearch/
+skin/adminhtml/base/default/algoliasearch/                          skin/adminhtml/base/default/algoliasearch/
 js/algoliasearch/                                                   js/algoliasearch/
 lib/AlgoliaSearch/                                                  lib/AlgoliaSearch/
diff --git a/skin/adminhtml/base/default/algoliasearch/algoliasearch.css b/skin/adminhtml/base/default/algoliasearch/algoliasearch.css
@@ -0,0 +1,5 @@
+.algoliasearch-queue-notification {
+	background: #e9f3ff url(/skin/frontend/base/default/algoliasearch/algolia-admin-menu.svg) no-repeat 27px 5px;
+	background-size: 16px 16px;
+	border-color: #bee1ee;
+}