Skip to content
This repository was archived by the owner on Apr 29, 2019. It is now read-only.

Commit 31c3293

Browse files
duhonduhon
authored
Merge pull request #2882 from magento-performance/MAGETWO-92447
[performance] MAGETWO-92447: [Indexer optimizations] Tables sharding / segmentation for price indexer - part 3
2 parents 9f006e0 + 80268b4 commit 31c3293

17 files changed

Lines changed: 494 additions & 177 deletions

File tree

‎app/code/Magento/Bundle/Model/ResourceModel/Indexer/Price.php‎

Lines changed: 309 additions & 122 deletions
Large diffs are not rendered by default.

‎app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php‎

Lines changed: 34 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ protected function _syncData(array $processIds = [])
160160
{
161161
// for backward compatibility split data from old idx table on dimension tables
162162
foreach ($this->dimensionCollectionFactory->create() as $dimensions) {
163-
$insertSelect = $this->_connection->select()->from(
163+
$insertSelect = $this->getConnection()->select()->from(
164164
['ip_tmp' => $this->_defaultIndexerResource->getIdxTable()]
165165
);
166166

@@ -174,7 +174,7 @@ protected function _syncData(array $processIds = [])
174174
}
175175

176176
$query = $insertSelect->insertFromSelect($this->tableMaintainer->getMainTable($dimensions));
177-
$this->_connection->query($query);
177+
$this->getConnection()->query($query);
178178
}
179179
return $this;
180180
}
@@ -191,7 +191,7 @@ protected function _prepareWebsiteDateTable()
191191
{
192192
$baseCurrency = $this->_config->getValue(\Magento\Directory\Model\Currency::XML_PATH_CURRENCY_BASE);
193193

194-
$select = $this->_connection->select()->from(
194+
$select = $this->getConnection()->select()->from(
195195
['cw' => $this->_defaultIndexerResource->getTable('store_website')],
196196
['website_id']
197197
)->join(
@@ -203,7 +203,7 @@ protected function _prepareWebsiteDateTable()
203203
);
204204

205205
$data = [];
206-
foreach ($this->_connection->fetchAll($select) as $item) {
206+
foreach ($this->getConnection()->fetchAll($select) as $item) {
207207
/** @var $website \Magento\Store\Model\Website */
208208
$website = $this->_storeManager->getWebsite($item['website_id']);
209209

@@ -237,7 +237,7 @@ protected function _prepareWebsiteDateTable()
237237
$this->_emptyTable($table);
238238
if ($data) {
239239
foreach ($data as $row) {
240-
$this->_connection->insertOnDuplicate($table, $row, array_keys($row));
240+
$this->getConnection()->insertOnDuplicate($table, $row, array_keys($row));
241241
}
242242
}
243243

@@ -325,19 +325,19 @@ protected function _getIndexer($productTypeId)
325325
*/
326326
protected function _insertFromTable($sourceTable, $destTable, $where = null)
327327
{
328-
$sourceColumns = array_keys($this->_connection->describeTable($sourceTable));
329-
$targetColumns = array_keys($this->_connection->describeTable($destTable));
330-
$select = $this->_connection->select()->from($sourceTable, $sourceColumns);
328+
$sourceColumns = array_keys($this->getConnection()->describeTable($sourceTable));
329+
$targetColumns = array_keys($this->getConnection()->describeTable($destTable));
330+
$select = $this->getConnection()->select()->from($sourceTable, $sourceColumns);
331331
if ($where) {
332332
$select->where($where);
333333
}
334-
$query = $this->_connection->insertFromSelect(
334+
$query = $this->getConnection()->insertFromSelect(
335335
$select,
336336
$destTable,
337337
$targetColumns,
338338
\Magento\Framework\DB\Adapter\AdapterInterface::INSERT_ON_DUPLICATE
339339
);
340-
$this->_connection->query($query);
340+
$this->getConnection()->query($query);
341341
}
342342

343343
/**
@@ -348,7 +348,7 @@ protected function _insertFromTable($sourceTable, $destTable, $where = null)
348348
*/
349349
protected function _emptyTable($table)
350350
{
351-
$this->_connection->delete($table);
351+
$this->getConnection()->delete($table);
352352
}
353353

354354
/**
@@ -381,7 +381,7 @@ protected function _reindexRows($changedIds = [])
381381
$this->tableMaintainer->createMainTmpTable($dimensions);
382382
$temporaryTable = $this->tableMaintainer->getMainTmpTable($dimensions);
383383
$this->_emptyTable($temporaryTable);
384-
$indexer->executeByDimension($dimensions, \SplFixedArray::fromArray($entityIds, false));
384+
$indexer->executeByDimensions($dimensions, \SplFixedArray::fromArray($entityIds, false));
385385
// copy to index
386386
$this->_insertFromTable(
387387
$temporaryTable,
@@ -407,12 +407,12 @@ protected function _reindexRows($changedIds = [])
407407
private function deleteIndexData(array $entityIds)
408408
{
409409
foreach ($this->dimensionCollectionFactory->create() as $dimensions) {
410-
$select = $this->_connection->select()->from(
410+
$select = $this->getConnection()->select()->from(
411411
['index_price' => $this->tableMaintainer->getMainTable($dimensions)],
412412
null
413413
)->where('index_price.entity_id IN (?)', $entityIds);
414414
$query = $select->deleteFromSelect('index_price');
415-
$this->_connection->query($query);
415+
$this->getConnection()->query($query);
416416
}
417417
}
418418

@@ -430,7 +430,7 @@ private function deleteIndexData(array $entityIds)
430430
protected function _copyRelationIndexData($parentIds, $excludeIds = null)
431431
{
432432
$linkField = $this->getProductIdFieldName();
433-
$select = $this->_connection->select()->from(
433+
$select = $this->getConnection()->select()->from(
434434
$this->_defaultIndexerResource->getTable('catalog_product_relation'),
435435
['child_id']
436436
)->join(
@@ -445,18 +445,18 @@ protected function _copyRelationIndexData($parentIds, $excludeIds = null)
445445
$select->where('child_id NOT IN(?)', $excludeIds);
446446
}
447447

448-
$children = $this->_connection->fetchCol($select);
448+
$children = $this->getConnection()->fetchCol($select);
449449

450450
if ($children) {
451451
foreach ($this->dimensionCollectionFactory->create() as $dimensions) {
452-
$select = $this->_connection->select()->from(
452+
$select = $this->getConnection()->select()->from(
453453
$this->getIndexTargetTableByDimension($dimensions)
454454
)->where(
455455
'entity_id IN(?)',
456456
$children
457457
);
458458
$query = $select->insertFromSelect($this->_defaultIndexerResource->getIdxTable(), [], false);
459-
$this->_connection->query($query);
459+
$this->getConnection()->query($query);
460460
}
461461
}
462462

@@ -502,8 +502,8 @@ protected function getIndexTargetTable()
502502
protected function getProductIdFieldName()
503503
{
504504
$table = $this->_defaultIndexerResource->getTable('catalog_product_entity');
505-
$indexList = $this->_connection->getIndexList($table);
506-
return $indexList[$this->_connection->getPrimaryKeyName($table)]['COLUMNS_LIST'][0];
505+
$indexList = $this->getConnection()->getIndexList($table);
506+
return $indexList[$this->getConnection()->getPrimaryKeyName($table)]['COLUMNS_LIST'][0];
507507
}
508508

509509
/**
@@ -514,14 +514,14 @@ protected function getProductIdFieldName()
514514
*/
515515
private function getProductsTypes(array $changedIds = [])
516516
{
517-
$select = $this->_connection->select()->from(
517+
$select = $this->getConnection()->select()->from(
518518
$this->_defaultIndexerResource->getTable('catalog_product_entity'),
519519
['entity_id', 'type_id']
520520
);
521521
if ($changedIds) {
522522
$select->where('entity_id IN (?)', $changedIds);
523523
}
524-
$pairs = $this->_connection->fetchPairs($select);
524+
$pairs = $this->getConnection()->fetchPairs($select);
525525

526526
$byType = [];
527527
foreach ($pairs as $productId => $productType) {
@@ -540,7 +540,7 @@ private function getProductsTypes(array $changedIds = [])
540540
*/
541541
private function getParentProductsTypes(array $productsIds)
542542
{
543-
$select = $this->_connection->select()->from(
543+
$select = $this->getConnection()->select()->from(
544544
['l' => $this->_defaultIndexerResource->getTable('catalog_product_relation')],
545545
''
546546
)->join(
@@ -551,7 +551,7 @@ private function getParentProductsTypes(array $productsIds)
551551
'l.child_id IN(?)',
552552
$productsIds
553553
);
554-
$pairs = $this->_connection->fetchPairs($select);
554+
$pairs = $this->getConnection()->fetchPairs($select);
555555

556556
$byType = [];
557557
foreach ($pairs as $productId => $productType) {
@@ -560,4 +560,14 @@ private function getParentProductsTypes(array $productsIds)
560560

561561
return $byType;
562562
}
563+
564+
/**
565+
* Get connection
566+
*
567+
* @return \Magento\Framework\DB\Adapter\AdapterInterface
568+
*/
569+
private function getConnection()
570+
{
571+
return $this->_defaultIndexerResource->getConnection();
572+
}
563573
}

‎app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,11 @@ class Full extends \Magento\Catalog\Model\Indexer\Product\Price\AbstractAction
5656
*/
5757
private $dimensionTableMaintainer;
5858

59+
/**
60+
* @var \Magento\Indexer\Model\ProcessManager
61+
*/
62+
private $processManager;
63+
5964
/**
6065
* @param \Magento\Framework\App\Config\ScopeConfigInterface $config
6166
* @param \Magento\Store\Model\StoreManagerInterface $storeManager
@@ -71,7 +76,7 @@ class Full extends \Magento\Catalog\Model\Indexer\Product\Price\AbstractAction
7176
* @param \Magento\Catalog\Model\ResourceModel\Indexer\ActiveTableSwitcher|null $activeTableSwitcher
7277
* @param \Magento\Catalog\Model\Indexer\Product\Price\DimensionCollectionFactory|null $dimensionCollectionFactory
7378
* @param \Magento\Catalog\Model\Indexer\Product\Price\TableMaintainer|null $dimensionTableMaintainer
74-
*
79+
* @param \Magento\Indexer\Model\ProcessManager $processManager
7580
* @SuppressWarnings(PHPMD.ExcessiveParameterList)
7681
*/
7782
public function __construct(
@@ -88,7 +93,8 @@ public function __construct(
8893
\Magento\Framework\Indexer\BatchProviderInterface $batchProvider = null,
8994
\Magento\Catalog\Model\ResourceModel\Indexer\ActiveTableSwitcher $activeTableSwitcher = null,
9095
\Magento\Catalog\Model\Indexer\Product\Price\DimensionCollectionFactory $dimensionCollectionFactory = null,
91-
\Magento\Catalog\Model\Indexer\Product\Price\TableMaintainer $dimensionTableMaintainer = null
96+
\Magento\Catalog\Model\Indexer\Product\Price\TableMaintainer $dimensionTableMaintainer = null,
97+
\Magento\Indexer\Model\ProcessManager $processManager = null
9298
) {
9399
parent::__construct(
94100
$config,
@@ -118,6 +124,9 @@ public function __construct(
118124
$this->dimensionTableMaintainer = $dimensionTableMaintainer ?: ObjectManager::getInstance()->get(
119125
\Magento\Catalog\Model\Indexer\Product\Price\TableMaintainer::class
120126
);
127+
$this->processManager = $processManager ?: ObjectManager::getInstance()->get(
128+
\Magento\Indexer\Model\ProcessManager::class
129+
);
121130
}
122131

123132
/**
@@ -195,9 +204,13 @@ private function truncateReplicaTables()
195204
*/
196205
private function reindexProductTypeWithDimensions(DimensionalIndexerInterface $priceIndexer, string $typeId)
197206
{
207+
$userFunctions = [];
198208
foreach ($this->dimensionCollectionFactory->create() as $dimensions) {
199-
$this->reindexByBatches($priceIndexer, $dimensions, $typeId);
209+
$userFunctions[] = function () use ($priceIndexer, $dimensions, $typeId) {
210+
return $this->reindexByBatches($priceIndexer, $dimensions, $typeId);
211+
};
200212
}
213+
$this->processManager->execute($userFunctions);
201214
}
202215

203216
/**
@@ -263,7 +276,7 @@ private function reindexByBatchWithDimensions(
263276
$temporaryTable = $this->dimensionTableMaintainer->getMainTmpTable($dimensions);
264277
$this->_emptyTable($temporaryTable);
265278

266-
$priceIndexer->executeByDimension($dimensions, \SplFixedArray::fromArray($entityIds, false));
279+
$priceIndexer->executeByDimensions($dimensions, \SplFixedArray::fromArray($entityIds, false));
267280

268281
// Sync data from temp table to index table
269282
$this->_insertFromTable(
@@ -415,11 +428,17 @@ private function moveDataFromReplicaTableToReplicaTables(array $dimensions)
415428
if (!$dimensions) {
416429
return;
417430
}
418-
//TODO: need to update logic for run this move only when replica table is not empty
419431
$select = $this->dimensionTableMaintainer->getConnection()->select()->from(
420432
$this->dimensionTableMaintainer->getMainReplicaTable([])
421433
);
422434

435+
$check = clone $select;
436+
$check->reset('columns')->columns('count(*)');
437+
438+
if (!$this->dimensionTableMaintainer->getConnection()->query($check)->fetchColumn()) {
439+
return;
440+
}
441+
423442
$replicaTablesByDimension = $this->dimensionTableMaintainer->getMainReplicaTable($dimensions);
424443

425444
foreach ($dimensions as $dimension) {

‎app/code/Magento/Catalog/Model/Indexer/Product/Price/ModeSwitcher.php‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ public function __construct(
5555
* @param string $currentMode
5656
*
5757
* @return void
58+
* @throws \Zend_Db_Exception
5859
*/
5960
public function createTables(string $currentMode)
6061
{
@@ -154,7 +155,12 @@ private function insertFromOldTablesToNew(string $newTable, string $oldTable, ar
154155
}
155156
}
156157
$this->tableMaintainer->getConnection()->query(
157-
$this->tableMaintainer->getConnection()->insertFromSelect($select, $newTable)
158+
$this->tableMaintainer->getConnection()->insertFromSelect(
159+
$select,
160+
$newTable,
161+
[],
162+
\Magento\Framework\DB\Adapter\AdapterInterface::INSERT_ON_DUPLICATE
163+
)
158164
);
159165
}
160166
}

‎app/code/Magento/Catalog/Model/ResourceModel/Layer/Filter/Price.php‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -399,10 +399,19 @@ public function applyPriceRange(\Magento\Catalog\Model\Layer\Filter\FilterInterf
399399

400400
/**
401401
* Initialize connection and define main table name
402-
* @deprecated
402+
*
403403
* @return void
404404
*/
405405
protected function _construct()
406+
{
407+
$this->_init('catalog_product_index_price', 'entity_id');
408+
}
409+
410+
/**
411+
* {@inheritdoc}
412+
* @return string
413+
*/
414+
public function getMainTable()
406415
{
407416
$storeKey = $this->httpContext->getValue(StoreManagerInterface::CONTEXT_STORE);
408417
$priceTableName = $this->priceTableResolver->resolve(
@@ -418,7 +427,8 @@ protected function _construct()
418427
)
419428
]
420429
);
421-
$this->_init($priceTableName, 'entity_id');
430+
431+
return $this->getTable($priceTableName);
422432
}
423433

424434
/**

‎app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/SimpleProductPrice.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ public function __construct(
6464
/**
6565
* {@inheritdoc}
6666
*/
67-
public function executeByDimension(array $dimensions, \Traversable $entityIds = null)
67+
public function executeByDimensions(array $dimensions, \Traversable $entityIds)
6868
{
6969
$this->tableMaintainer->createMainTmpTable($dimensions);
7070

0 commit comments

Comments
 (0)