From 7fdfe626cb95146ff169c68c14a82890169b25d4 Mon Sep 17 00:00:00 2001 From: Jan Petr Date: Thu, 31 Aug 2017 18:22:19 +0200 Subject: [PATCH 1/7] Show queue information in notifications --- .../Block/Adminhtml/Notifications.php | 42 +++++++++++++++++++ .../default/default/layout/algoliasearch.xml | 10 +++++ .../algoliasearch/notifications.phtml | 30 +++++++++++++ modman | 1 + .../default/algoliasearch/algoliasearch.css | 5 +++ 5 files changed, 88 insertions(+) create mode 100644 app/code/community/Algolia/Algoliasearch/Block/Adminhtml/Notifications.php create mode 100644 app/design/adminhtml/default/default/template/algoliasearch/notifications.phtml create mode 100644 skin/adminhtml/base/default/algoliasearch/algoliasearch.css diff --git a/app/code/community/Algolia/Algoliasearch/Block/Adminhtml/Notifications.php b/app/code/community/Algolia/Algoliasearch/Block/Adminhtml/Notifications.php new file mode 100644 index 00000000..b635372b --- /dev/null +++ b/app/code/community/Algolia/Algoliasearch/Block/Adminhtml/Notifications.php @@ -0,0 +1,42 @@ +getUrl('adminhtml/system_config/edit/section/algoliasearch'); + } + + public function getQueueInfo() + { + /** @var Algolia_Algoliasearch_Helper_Config $config */ + $config = Mage::helper('algoliasearch/config'); + + /** @var Mage_Core_Model_Resource $resource */ + $resource = Mage::getSingleton('core/resource'); + $tableName = $resource->getTableName('algoliasearch/queue'); + + $readConnection = $resource->getConnection('core_read'); + + $size = (int)$readConnection->query('SELECT COUNT(*) as total_count FROM '.$tableName)->fetchColumn(0); + $maxJobsPerSingleRun = $config->getNumberOfJobToRun(); + + $etaMinutes = ceil($size / $maxJobsPerSingleRun) * 5; // 5 - assuming the queue runner runs every 5 minutes + + $eta = $etaMinutes.' minutes'; + if ($etaMinutes > 60) { + $hours = floor($etaMinutes / 60); + $restMinutes = $etaMinutes % 60; + + $eta = $hours.' hours '.$restMinutes.' minutes'; + } + + $queueInfo = array( + 'isEnabled' => $config->isQueueActive(), + 'currentSize' => $size, + 'eta' => $eta, + ); + + return $queueInfo; + } +} diff --git a/app/design/adminhtml/default/default/layout/algoliasearch.xml b/app/design/adminhtml/default/default/layout/algoliasearch.xml index 62d2b4b3..9397aee3 100644 --- a/app/design/adminhtml/default/default/layout/algoliasearch.xml +++ b/app/design/adminhtml/default/default/layout/algoliasearch.xml @@ -1,5 +1,15 @@ + + + algoliasearch/algoliasearch.css + + + + + + + diff --git a/app/design/adminhtml/default/default/template/algoliasearch/notifications.phtml b/app/design/adminhtml/default/default/template/algoliasearch/notifications.phtml new file mode 100644 index 00000000..a757f6ca --- /dev/null +++ b/app/design/adminhtml/default/default/template/algoliasearch/notifications.phtml @@ -0,0 +1,30 @@ +getQueueInfo(); + +?> + +
+ Algolia Search - + + + Indexing queue information: + + + Number of queued jobs: , + + 0): ?> + all queued jobs will be processed in appr. , + + + more information about how the indexing queue works you can find in the documentation: Indexing queue + + + Indexing queue is not enabled. + + + It's highly recommended to enable it, especially if you are on production environment. You can learn how to enable the index queue in the documentation: Indexing queue + +
diff --git a/modman b/modman index 6eeb3784..93875e83 100644 --- a/modman +++ b/modman @@ -14,5 +14,6 @@ app/locale/en_US/Algolia_Algoliasearch.csv app/locale/e app/locale/sv_SE/Algolia_Algoliasearch.csv app/locale/sv_SE/Algolia_Algoliasearch.csv app/locale/nl_NL/Algolia_Algoliasearch.csv app/locale/nl_NL/Algolia_Algoliasearch.csv skin/frontend/base/default/algoliasearch/ skin/frontend/base/default/algoliasearch/ +skin/adminhtml/base/default/algoliasearch/ skin/adminhtml/base/default/algoliasearch/ js/algoliasearch/ js/algoliasearch/ lib/AlgoliaSearch/ lib/AlgoliaSearch/ \ No newline at end of file diff --git a/skin/adminhtml/base/default/algoliasearch/algoliasearch.css b/skin/adminhtml/base/default/algoliasearch/algoliasearch.css new file mode 100644 index 00000000..2d7145ea --- /dev/null +++ b/skin/adminhtml/base/default/algoliasearch/algoliasearch.css @@ -0,0 +1,5 @@ +.algoliasearch-queue-notification { + background: #e9f3ff url(/skin/frontend/base/default/algoliasearch/algolia-admin-menu.svg) no-repeat 27px 5px; + background-size: 16px 16px; + border-color: #bee1ee; +} From f005f63e1afedcbc979e37f9d33dfb7c90f972ae Mon Sep 17 00:00:00 2001 From: Jan Petr Date: Fri, 1 Sep 2017 12:37:10 +0200 Subject: [PATCH 2/7] Optimize start.sh --- dev/bin/start.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/dev/bin/start.sh b/dev/bin/start.sh index a5104e5f..c162fce7 100644 --- a/dev/bin/start.sh +++ b/dev/bin/start.sh @@ -22,12 +22,8 @@ n98-magerun --skip-root-check --root-dir=/var/www/htdocs config:set algoliasearc n98-magerun --skip-root-check --root-dir=/var/www/htdocs config:set web/unsecure/base_url $BASE_URL n98-magerun --skip-root-check --root-dir=/var/www/htdocs config:set web/secure/base_url $BASE_URL -chmod -R 777 /var/www/htdocs/media -chown -R www-data:www-data /var/www/htdocs/media - if [ $INSTALL_ALGOLIA == Yes ]; then - /root/bin/modman repair --force algoliasearch-magento - /root/bin/modman repair --force algoliasearch-magento-extend-module-skeleton + /root/bin/modman deploy-all # reindex whole index n98-magerun --skip-root-check --root-dir=/var/www/htdocs index:reindex algolia_search_indexer @@ -39,7 +35,6 @@ else /root/bin/modman undeploy algoliasearch-magento-extend-module-skeleton fi -# Again in case root created some folder with root:root chmod -R 777 /var/www/htdocs/media chown -R www-data:www-data /var/www/htdocs/media From 46acbd317c2b00b37c525effb22e7acbd455d840 Mon Sep 17 00:00:00 2001 From: Jan Petr Date: Tue, 5 Sep 2017 17:59:30 +0200 Subject: [PATCH 3/7] Queue log table --- .../Algolia/Algoliasearch/Model/Queue.php | 50 +++++-- .../mysql4-upgrade-1.7.1-1.11.0.php | 21 +++ tests/QueueTest.php | 124 ++++++++++-------- 3 files changed, 131 insertions(+), 64 deletions(-) create mode 100644 app/code/community/Algolia/Algoliasearch/sql/algoliasearch_setup/mysql4-upgrade-1.7.1-1.11.0.php diff --git a/app/code/community/Algolia/Algoliasearch/Model/Queue.php b/app/code/community/Algolia/Algoliasearch/Model/Queue.php index 1c6d880f..955ad11d 100644 --- a/app/code/community/Algolia/Algoliasearch/Model/Queue.php +++ b/app/code/community/Algolia/Algoliasearch/Model/Queue.php @@ -6,6 +6,7 @@ class Algolia_Algoliasearch_Model_Queue const ERROR_LOG = 'algoliasearch_queue_errors.log'; protected $table; + protected $logTable; /** @var Magento_Db_Adapter_Pdo_Mysql */ protected $db; @@ -29,12 +30,16 @@ class Algolia_Algoliasearch_Model_Queue private $noOfFailedJobs = 0; + private $logRecord = array(); + public function __construct() { /** @var Mage_Core_Model_Resource $coreResource */ $coreResource = Mage::getSingleton('core/resource'); $this->table = $coreResource->getTableName('algoliasearch/queue'); + $this->logTable = $this->table.'_log'; + $this->db = $coreResource->getConnection('core_write'); $this->config = Mage::helper('algoliasearch/config'); @@ -55,19 +60,38 @@ public function add($class, $method, $data, $data_size) )); } - public function runCron() + public function runCron($nbJobs = null, $force = false) { - if (!$this->config->isQueueActive()) { + if (!$this->config->isQueueActive() && $force === false) { return; } - $nbJobs = $this->config->getNumberOfJobToRun(); + $this->clearOldLogRecords(); + + $this->logRecord = array( + 'started' => date('Y-m-d H:i:s'), + 'processed_jobs' => 0, + 'with_empty_queue' => 0, + ); + + $started = time(); + + if ($nbJobs === null) { + $nbJobs = $this->config->getNumberOfJobToRun(); + if (getenv('EMPTY_QUEUE') && getenv('EMPTY_QUEUE') == '1') { + $nbJobs = -1; - if (getenv('EMPTY_QUEUE') && getenv('EMPTY_QUEUE') == '1') { - $nbJobs = -1; + $this->logRecord['with_empty_queue'] = 1; + } } $this->run($nbJobs); + + $this->logRecord['duration'] = time() - $started; + + $this->db->insert($this->logTable, $this->logRecord); + + $this->db->closeConnection(); } public function run($maxJobs) @@ -77,7 +101,7 @@ public function run($maxJobs) $jobs = $this->getJobs($maxJobs, $pid); if (empty($jobs)) { - $this->db->closeConnection(); + return; } // Run all reserved jobs @@ -96,6 +120,8 @@ public function run($maxJobs) $model = Mage::getSingleton($job['class']); $method = $job['method']; $model->{$method}(new Varien_Object($job['data'])); + + $this->logRecord['processed_jobs'] += count($job['merged_ids']); } catch (\Exception $e) { $this->noOfFailedJobs++; @@ -119,8 +145,6 @@ public function run($maxJobs) return; } - - $this->db->closeConnection(); } private function getJobs($maxJobs, $pid) @@ -381,4 +405,14 @@ private function maxValueInArray($array, $keyToSearch) return $currentMax; } + + private function clearOldLogRecords() + { + $idsToDelete = $this->db->query("SELECT id FROM {$this->logTable} ORDER BY started DESC, id DESC LIMIT 25000, ".PHP_INT_MAX) + ->fetchAll(\PDO::FETCH_COLUMN, 0); + + if ($idsToDelete) { + $this->db->query("DELETE FROM {$this->logTable} WHERE id IN (" . implode(", ", $idsToDelete) . ")"); + } + } } diff --git a/app/code/community/Algolia/Algoliasearch/sql/algoliasearch_setup/mysql4-upgrade-1.7.1-1.11.0.php b/app/code/community/Algolia/Algoliasearch/sql/algoliasearch_setup/mysql4-upgrade-1.7.1-1.11.0.php new file mode 100644 index 00000000..41e9c3e5 --- /dev/null +++ b/app/code/community/Algolia/Algoliasearch/sql/algoliasearch_setup/mysql4-upgrade-1.7.1-1.11.0.php @@ -0,0 +1,21 @@ +startSetup(); + +$tableName = $installer->getTable('algoliasearch/queue'); +$installer->run("ALTER TABLE `{$tableName}` ADD `created` DATETIME AFTER `job_id`;"); + +$installer->run(" +CREATE TABLE IF NOT EXISTS `{$tableName}_log` ( + `id` INT(20) NOT NULL auto_increment, + `started` DATETIME NOT NULL, + `duration` INT(20) NOT NULL, + `processed_jobs` INT NOT NULL, + `with_empty_queue` INT(1) NOT NULL, + PRIMARY KEY `id` (`id`) +) ENGINE=InnoDB DEFAULT CHARACTER SET=utf8 AUTO_INCREMENT=1; +"); + +$installer->endSetup(); diff --git a/tests/QueueTest.php b/tests/QueueTest.php index 15f5eb34..fb91f634 100644 --- a/tests/QueueTest.php +++ b/tests/QueueTest.php @@ -61,10 +61,12 @@ public function testExecute() /** @var Algolia_Algoliasearch_Helper_Algoliahelper $algoliaHelper */ $algoliaHelper = Mage::helper('algoliasearch/algoliahelper'); + $this->writeConnection->query('TRUNCATE TABLE algoliasearch_queue_log'); + $queue = new Algolia_Algoliasearch_Model_Queue(); // Run the first four jobs - saveSettings, batch, move, batch - $queue->run(4); + $queue->runCron(4, true); $algoliaHelper->waitLastTask(); @@ -92,7 +94,7 @@ public function testExecute() $this->assertTrue($existsFrenchTmpIndex, 'French products TMP index does not exists and it should'); // Wasn't moved // Run the second three jobs - move, batch, move - $queue->run(3); + $queue->runCron(3, true); $algoliaHelper->waitLastTask(); @@ -137,6 +139,12 @@ public function testExecute() $this->assertTrue($existsDefaultProdIndex, 'Default product production index does not exists and it should'); $this->assertTrue($existsFrenchProdIndex, 'French products production index does not exists and it should'); $this->assertTrue($existsGermanProdIndex, 'German products production index does not exists and it should'); + + $logRecords = $this->readConnection->query('SELECT * FROM algoliasearch_queue_log')->fetchAll(); + + $this->assertSame(2, count($logRecords)); + $this->assertEquals(4, $logRecords[0]['processed_jobs']); + $this->assertEquals(3, $logRecords[1]['processed_jobs']); } public function testSettings() @@ -188,19 +196,19 @@ public function testSettings() public function testMerging() { $this->writeConnection->query('TRUNCATE TABLE algoliasearch_queue'); - $this->writeConnection->query('INSERT INTO `algoliasearch_queue` (`job_id`, `pid`, `class`, `method`, `data`, `max_retries`, `retries`, `error_log`, `data_size`) VALUES - (1, NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"1","category_ids":["9","22"]}\', 3, 0, \'\', 2), - (2, NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"2","category_ids":["9","22"]}\', 3, 0, \'\', 2), - (3, NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"3","category_ids":["9","22"]}\', 3, 0, \'\', 2), - (4, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"1","product_ids":["448"]}\', 3, 0, \'\', 1), - (5, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"2","product_ids":["448"]}\', 3, 0, \'\', 1), - (6, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"3","product_ids":["448"]}\', 3, 0, \'\', 1), - (7, NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"1","category_ids":["40"]}\', 3, 0, \'\', 1), - (8, NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"2","category_ids":["40"]}\', 3, 0, \'\', 1), - (9, NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"3","category_ids":["40"]}\', 3, 0, \'\', 1), - (10, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"1","product_ids":["405"]}\', 3, 0, \'\', 1), - (11, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"2","product_ids":["405"]}\', 3, 0, \'\', 1), - (12, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"3","product_ids":["405"]}\', 3, 0, \'\', 1);'); + $this->writeConnection->query('INSERT INTO `algoliasearch_queue` (`job_id`, `created`, `pid`, `class`, `method`, `data`, `max_retries`, `retries`, `error_log`, `data_size`) VALUES + (1, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"1","category_ids":["9","22"]}\', 3, 0, \'\', 2), + (2, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"2","category_ids":["9","22"]}\', 3, 0, \'\', 2), + (3, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"3","category_ids":["9","22"]}\', 3, 0, \'\', 2), + (4, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"1","product_ids":["448"]}\', 3, 0, \'\', 1), + (5, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"2","product_ids":["448"]}\', 3, 0, \'\', 1), + (6, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"3","product_ids":["448"]}\', 3, 0, \'\', 1), + (7, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"1","category_ids":["40"]}\', 3, 0, \'\', 1), + (8, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"2","category_ids":["40"]}\', 3, 0, \'\', 1), + (9, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"3","category_ids":["40"]}\', 3, 0, \'\', 1), + (10, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"1","product_ids":["405"]}\', 3, 0, \'\', 1), + (11, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"2","product_ids":["405"]}\', 3, 0, \'\', 1), + (12, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"3","product_ids":["405"]}\', 3, 0, \'\', 1);'); $queue = new Algolia_Algoliasearch_Model_Queue(); @@ -212,6 +220,7 @@ public function testMerging() $expectedCategoryJob = array( 'job_id' => 7, + 'created' => '2017-09-01 12:00:00', 'pid' => NULL, 'class' => 'algoliasearch/observer', 'method' => 'rebuildCategoryIndex', @@ -235,6 +244,7 @@ public function testMerging() $expectedProductJob = array( 'job_id' => 10, + 'created' => '2017-09-01 12:00:00', 'pid' => NULL, 'class' => 'algoliasearch/observer', 'method' => 'rebuildProductIndex', @@ -259,19 +269,19 @@ public function testMerging() public function testMergingWithStaticMethods() { $this->writeConnection->query('TRUNCATE TABLE algoliasearch_queue'); - $this->writeConnection->query('INSERT INTO `algoliasearch_queue` (`job_id`, `pid`, `class`, `method`, `data`, `max_retries`, `retries`, `error_log`, `data_size`) VALUES - (1, NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"1","category_ids":["9","22"]}\', 3, 0, \'\', 2), - (2, NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"2","category_ids":["9","22"]}\', 3, 0, \'\', 2), - (3, NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"3","category_ids":["9","22"]}\', 3, 0, \'\', 2), - (4, NULL, \'algoliasearch/observer\', \'removeCategories\', \'{"store_id":"1","product_ids":["448"]}\', 3, 0, \'\', 1), - (5, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"2","product_ids":["448"]}\', 3, 0, \'\', 1), - (6, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"3","product_ids":["448"]}\', 3, 0, \'\', 1), - (7, NULL, \'algoliasearch/observer\', \'saveSettings\', \'{"store_id":"1","category_ids":["40"]}\', 3, 0, \'\', 1), - (8, NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"2","category_ids":["40"]}\', 3, 0, \'\', 1), - (9, NULL, \'algoliasearch/observer\', \'moveProductsTmpIndex\', \'{"store_id":"3","category_ids":["40"]}\', 3, 0, \'\', 1), - (10, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"1","product_ids":["405"]}\', 3, 0, \'\', 1), - (11, NULL, \'algoliasearch/observer\', \'moveStoreSuggestionIndex\', \'{"store_id":"2","product_ids":["405"]}\', 3, 0, \'\', 1), - (12, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"3","product_ids":["405"]}\', 3, 0, \'\', 1);'); + $this->writeConnection->query('INSERT INTO `algoliasearch_queue` (`job_id`, `created`, `pid`, `class`, `method`, `data`, `max_retries`, `retries`, `error_log`, `data_size`) VALUES + (1, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"1","category_ids":["9","22"]}\', 3, 0, \'\', 2), + (2, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"2","category_ids":["9","22"]}\', 3, 0, \'\', 2), + (3, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"3","category_ids":["9","22"]}\', 3, 0, \'\', 2), + (4, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'removeCategories\', \'{"store_id":"1","product_ids":["448"]}\', 3, 0, \'\', 1), + (5, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"2","product_ids":["448"]}\', 3, 0, \'\', 1), + (6, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"3","product_ids":["448"]}\', 3, 0, \'\', 1), + (7, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'saveSettings\', \'{"store_id":"1","category_ids":["40"]}\', 3, 0, \'\', 1), + (8, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"2","category_ids":["40"]}\', 3, 0, \'\', 1), + (9, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'moveProductsTmpIndex\', \'{"store_id":"3","category_ids":["40"]}\', 3, 0, \'\', 1), + (10, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"1","product_ids":["405"]}\', 3, 0, \'\', 1), + (11, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'moveStoreSuggestionIndex\', \'{"store_id":"2","product_ids":["405"]}\', 3, 0, \'\', 1), + (12, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"3","product_ids":["405"]}\', 3, 0, \'\', 1);'); $queue = new Algolia_Algoliasearch_Model_Queue(); @@ -298,19 +308,19 @@ public function testMergingWithStaticMethods() public function testGetJobs() { $this->writeConnection->query('TRUNCATE TABLE algoliasearch_queue'); - $this->writeConnection->query('INSERT INTO `algoliasearch_queue` (`job_id`, `pid`, `class`, `method`, `data`, `max_retries`, `retries`, `error_log`, `data_size`) VALUES - (1, NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"1","category_ids":["9","22"]}\', 3, 0, \'\', 2), - (2, NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"2","category_ids":["9","22"]}\', 3, 0, \'\', 2), - (3, NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"3","category_ids":["9","22"]}\', 3, 0, \'\', 2), - (4, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"1","product_ids":["448"]}\', 3, 0, \'\', 1), - (5, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"2","product_ids":["448"]}\', 3, 0, \'\', 1), - (6, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"3","product_ids":["448"]}\', 3, 0, \'\', 1), - (7, NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"1","category_ids":["40"]}\', 3, 0, \'\', 1), - (8, NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"2","category_ids":["40"]}\', 3, 0, \'\', 1), - (9, NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"3","category_ids":["40"]}\', 3, 0, \'\', 1), - (10, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"1","product_ids":["405"]}\', 3, 0, \'\', 1), - (11, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"2","product_ids":["405"]}\', 3, 0, \'\', 1), - (12, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"3","product_ids":["405"]}\', 3, 0, \'\', 1);'); + $this->writeConnection->query('INSERT INTO `algoliasearch_queue` (`job_id`, `created`, `pid`, `class`, `method`, `data`, `max_retries`, `retries`, `error_log`, `data_size`) VALUES + (1, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"1","category_ids":["9","22"]}\', 3, 0, \'\', 2), + (2, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"2","category_ids":["9","22"]}\', 3, 0, \'\', 2), + (3, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"3","category_ids":["9","22"]}\', 3, 0, \'\', 2), + (4, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"1","product_ids":["448"]}\', 3, 0, \'\', 1), + (5, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"2","product_ids":["448"]}\', 3, 0, \'\', 1), + (6, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"3","product_ids":["448"]}\', 3, 0, \'\', 1), + (7, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"1","category_ids":["40"]}\', 3, 0, \'\', 1), + (8, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"2","category_ids":["40"]}\', 3, 0, \'\', 1), + (9, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"3","category_ids":["40"]}\', 3, 0, \'\', 1), + (10, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"1","product_ids":["405"]}\', 3, 0, \'\', 1), + (11, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"2","product_ids":["405"]}\', 3, 0, \'\', 1), + (12, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"3","product_ids":["405"]}\', 3, 0, \'\', 1);'); $queue = new Algolia_Algoliasearch_Model_Queue(); @@ -320,6 +330,7 @@ public function testGetJobs() $expectedFirstJob = array( 'job_id' => 7, + 'created' => '2017-09-01 12:00:00', 'pid' => NULL, 'class' => 'algoliasearch/observer', 'method' => 'rebuildCategoryIndex', @@ -341,6 +352,7 @@ public function testGetJobs() $expectedLastJob = array( 'job_id' => 12, + 'created' => '2017-09-01 12:00:00', 'pid' => NULL, 'class' => 'algoliasearch/observer', 'method' => 'rebuildProductIndex', @@ -381,9 +393,9 @@ public function testHugeJob() $jsonProductIds = json_encode($productIds); $this->writeConnection->query('TRUNCATE TABLE algoliasearch_queue'); - $this->writeConnection->query('INSERT INTO `algoliasearch_queue` (`job_id`, `pid`, `class`, `method`, `data`, `max_retries`, `retries`, `error_log`, `data_size`) VALUES - (1, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"1","product_ids":'.$jsonProductIds.'}\', 3, 0, \'\', 5000), - (2, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"2","product_ids":["9","22"]}\', 3, 0, \'\', 2);'); + $this->writeConnection->query('INSERT INTO `algoliasearch_queue` (`job_id`, `created`, `pid`, `class`, `method`, `data`, `max_retries`, `retries`, `error_log`, `data_size`) VALUES + (1, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"1","product_ids":'.$jsonProductIds.'}\', 3, 0, \'\', 5000), + (2, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"2","product_ids":["9","22"]}\', 3, 0, \'\', 2);'); $queue = new Algolia_Algoliasearch_Model_Queue(); @@ -417,9 +429,9 @@ public function testMaxSingleJobSize() $jsonProductIds = json_encode($productIds); $this->writeConnection->query('TRUNCATE TABLE algoliasearch_queue'); - $this->writeConnection->query('INSERT INTO `algoliasearch_queue` (`job_id`, `pid`, `class`, `method`, `data`, `max_retries`, `retries`, `error_log`, `data_size`) VALUES - (1, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"1","product_ids":'.$jsonProductIds.'}\', 3, 0, \'\', 99), - (2, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"2","product_ids":["9","22"]}\', 3, 0, \'\', 2);'); + $this->writeConnection->query('INSERT INTO `algoliasearch_queue` (`job_id`, `created`, `pid`, `class`, `method`, `data`, `max_retries`, `retries`, `error_log`, `data_size`) VALUES + (1, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"1","product_ids":'.$jsonProductIds.'}\', 3, 0, \'\', 99), + (2, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"2","product_ids":["9","22"]}\', 3, 0, \'\', 2);'); $queue = new Algolia_Algoliasearch_Model_Queue(); @@ -455,15 +467,15 @@ public function testFaildedJob() $this->writeConnection->query('TRUNCATE TABLE algoliasearch_queue'); // Setting "not-existing-store" as store_id throws exception during processing the job - $this->writeConnection->query('INSERT INTO `algoliasearch_queue` (`job_id`, `pid`, `class`, `method`, `data`, `max_retries`, `retries`, `error_log`, `data_size`) VALUES - (1, NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"not-existing-store","category_ids":["9","22"]}\', 3, 0, \'\', 2), - (2, NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"2","category_ids":["9","22"]}\', 3, 0, \'\', 2), - (3, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"not-existing-store","product_ids":["448"]}\', 3, 0, \'\', 1), - (4, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"2","product_ids":["448"]}\', 3, 0, \'\', 1), - (5, NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"not-existing-store","category_ids":["40"]}\', 3, 0, \'\', 1), - (6, NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"2","category_ids":["40"]}\', 3, 0, \'\', 1), - (7, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"not-existing-store","product_ids":["405"]}\', 3, 0, \'\', 1), - (8, NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"2","product_ids":["405"]}\', 3, 0, \'\', 1)'); + $this->writeConnection->query('INSERT INTO `algoliasearch_queue` (`job_id`, `created`, `pid`, `class`, `method`, `data`, `max_retries`, `retries`, `error_log`, `data_size`) VALUES + (1, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"not-existing-store","category_ids":["9","22"]}\', 3, 0, \'\', 2), + (2, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"2","category_ids":["9","22"]}\', 3, 0, \'\', 2), + (3, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"not-existing-store","product_ids":["448"]}\', 3, 0, \'\', 1), + (4, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"2","product_ids":["448"]}\', 3, 0, \'\', 1), + (5, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"not-existing-store","category_ids":["40"]}\', 3, 0, \'\', 1), + (6, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildCategoryIndex\', \'{"store_id":"2","category_ids":["40"]}\', 3, 0, \'\', 1), + (7, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"not-existing-store","product_ids":["405"]}\', 3, 0, \'\', 1), + (8, \'2017-09-01 12:00:00\', NULL, \'algoliasearch/observer\', \'rebuildProductIndex\', \'{"store_id":"2","product_ids":["405"]}\', 3, 0, \'\', 1)'); $queue = new Algolia_Algoliasearch_Model_Queue(); From 23e4d10cffde1afe6e3fe4bb5b9d34ba35d36784 Mon Sep 17 00:00:00 2001 From: Jan Petr Date: Thu, 7 Sep 2017 16:17:51 +0200 Subject: [PATCH 4/7] Log date of creation for queued jobs --- app/code/community/Algolia/Algoliasearch/Model/Queue.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/code/community/Algolia/Algoliasearch/Model/Queue.php b/app/code/community/Algolia/Algoliasearch/Model/Queue.php index 955ad11d..abe524fa 100644 --- a/app/code/community/Algolia/Algoliasearch/Model/Queue.php +++ b/app/code/community/Algolia/Algoliasearch/Model/Queue.php @@ -52,6 +52,7 @@ public function add($class, $method, $data, $data_size) { // Insert a row for the new job $this->db->insert($this->table, array( + 'created' => date('Y-m-d H:i:s'), 'class' => $class, 'method' => $method, 'data' => json_encode($data), From 46fcb33bf13e49e97ccb2152a6073971045b5e1a Mon Sep 17 00:00:00 2001 From: Jan Petr Date: Thu, 14 Sep 2017 11:50:16 +0200 Subject: [PATCH 5/7] Copy query rules to TMP index so it's not erased --- .../Algoliasearch/Helper/Algoliahelper.php | 34 +++++++++++++++++++ .../Helper/Entity/Producthelper.php | 4 +++ 2 files changed, 38 insertions(+) diff --git a/app/code/community/Algolia/Algoliasearch/Helper/Algoliahelper.php b/app/code/community/Algolia/Algoliasearch/Helper/Algoliahelper.php index 369d3aaf..81862825 100644 --- a/app/code/community/Algolia/Algoliasearch/Helper/Algoliahelper.php +++ b/app/code/community/Algolia/Algoliasearch/Helper/Algoliahelper.php @@ -217,6 +217,40 @@ public function copySynonyms($fromIndexName, $toIndexName) $this->lastTaskId = $res['taskID']; } + public function copyQueryRules($fromIndexName, $toIndexName) + { + $fromIndex = $this->getIndex($fromIndexName); + $toIndex = $this->getIndex($toIndexName); + + $queryRulesToSet = array(); + + $hitsPerPage = 100; + $page = 0; + do { + $fetchedQueryRules = $fromIndex->searchRules(array( + 'page' => $page, + 'hitsPerPage' => $hitsPerPage, + )); + + foreach ($fetchedQueryRules['hits'] as $hit) { + unset($hit['_highlightResult']); + + $queryRulesToSet[] = $hit; + } + + $page++; + } while (($page * $hitsPerPage) < $fetchedQueryRules['nbHits']); + + if (empty($queryRulesToSet)) { + $res = $toIndex->clearRules(true); + } else { + $res = $toIndex->batchRules($queryRulesToSet, true, true); + } + + $this->lastUsedIndexName = $toIndex; + $this->lastTaskId = $res['taskID']; + } + public function waitLastTask() { if (!isset($this->lastUsedIndexName) || !isset($this->lastTaskId)) { diff --git a/app/code/community/Algolia/Algoliasearch/Helper/Entity/Producthelper.php b/app/code/community/Algolia/Algoliasearch/Helper/Entity/Producthelper.php index 60dd5294..7cb31bca 100644 --- a/app/code/community/Algolia/Algoliasearch/Helper/Entity/Producthelper.php +++ b/app/code/community/Algolia/Algoliasearch/Helper/Entity/Producthelper.php @@ -406,6 +406,10 @@ public function setSettings($storeId, $saveToTmpIndicesToo = false) } elseif ($saveToTmpIndicesToo === true) { $this->algolia_helper->copySynonyms($this->getIndexName($storeId), $this->getIndexName($storeId, $saveToTmpIndicesToo)); } + + if ($saveToTmpIndicesToo === true) { + $this->algolia_helper->copyQueryRules($this->getIndexName($storeId), $this->getIndexName($storeId, $saveToTmpIndicesToo)); + } } protected function getFields($store) From 0f138971b357c73a843dfa030f7eee66246e93e0 Mon Sep 17 00:00:00 2001 From: Jan Petr Date: Thu, 14 Sep 2017 12:38:23 +0200 Subject: [PATCH 6/7] Rename migration to the right version --- ...4-upgrade-1.7.1-1.11.0.php => mysql4-upgrade-1.7.1-1.11.1.php} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename app/code/community/Algolia/Algoliasearch/sql/algoliasearch_setup/{mysql4-upgrade-1.7.1-1.11.0.php => mysql4-upgrade-1.7.1-1.11.1.php} (100%) diff --git a/app/code/community/Algolia/Algoliasearch/sql/algoliasearch_setup/mysql4-upgrade-1.7.1-1.11.0.php b/app/code/community/Algolia/Algoliasearch/sql/algoliasearch_setup/mysql4-upgrade-1.7.1-1.11.1.php similarity index 100% rename from app/code/community/Algolia/Algoliasearch/sql/algoliasearch_setup/mysql4-upgrade-1.7.1-1.11.0.php rename to app/code/community/Algolia/Algoliasearch/sql/algoliasearch_setup/mysql4-upgrade-1.7.1-1.11.1.php From 2f59862497beda46e748500747958f1432654258 Mon Sep 17 00:00:00 2001 From: Jan Petr Date: Thu, 14 Sep 2017 16:41:04 +0200 Subject: [PATCH 7/7] Bump to 1.11.1 --- CHANGELOG.md | 6 ++++++ README.md | 2 +- app/code/community/Algolia/Algoliasearch/etc/config.xml | 2 +- app/code/community/Algolia/Algoliasearch/etc/system.xml | 2 +- app/etc/modules/Algolia_Algoliasearch.xml | 2 +- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9aa69aa2..e8e6e02c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ ## CHANGE LOG +### 1.11.1 + +- Query rules are preserved during reindex with indexing queue enabled (#913) +- Information about the indexing queue is displayed in admin-wide notifications (#905) +- Information about queue processing are logged to `algoliasearch_queue_log` DB table (#907) + ### 1.11.0 ## FEATURES diff --git a/README.md b/README.md index facd5cb8..d1e25c0c 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Algolia Search for Magento 1.6+ ================== -![Latest version](https://img.shields.io/badge/latest-1.11.0-green.svg) +![Latest version](https://img.shields.io/badge/latest-1.11.1-green.svg) [![Build Status](https://travis-ci.org/algolia/algoliasearch-magento.svg?branch=master)](https://travis-ci.org/algolia/algoliasearch-magento) ![PHP >= 5.3](https://img.shields.io/badge/php-%3E=5.3-green.svg) diff --git a/app/code/community/Algolia/Algoliasearch/etc/config.xml b/app/code/community/Algolia/Algoliasearch/etc/config.xml index 8770511b..ad193e48 100644 --- a/app/code/community/Algolia/Algoliasearch/etc/config.xml +++ b/app/code/community/Algolia/Algoliasearch/etc/config.xml @@ -2,7 +2,7 @@ - 1.11.0 + 1.11.1 diff --git a/app/code/community/Algolia/Algoliasearch/etc/system.xml b/app/code/community/Algolia/Algoliasearch/etc/system.xml index 5525a6d1..fdeec604 100644 --- a/app/code/community/Algolia/Algoliasearch/etc/system.xml +++ b/app/code/community/Algolia/Algoliasearch/etc/system.xml @@ -4,7 +4,7 @@