Skip to content

Commit a53347e

Browse files
kandykandy
authored
Merge pull request #2514 from magento-performance/MAGETWO-90563
[performance] MAGETWO-90563: [Forwardport] Implement segmentation for Category Product Indexer
2 parents b2eb795 + 52971ba commit a53347e

File tree

27 files changed

+1054
-192
lines changed

27 files changed

+1054
-192
lines changed

‎app/code/Magento/AdvancedSearch/Model/ResourceModel/Index.php

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@
1010
use Magento\Framework\Model\ResourceModel\Db\Context;
1111
use Magento\Framework\EntityManager\MetadataPool;
1212
use Magento\Catalog\Api\Data\CategoryInterface;
13+
use Magento\Framework\App\ObjectManager;
14+
use Magento\Framework\Indexer\ScopeResolver\IndexScopeResolver as TableResolver;
15+
use Magento\Framework\Search\Request\Dimension;
16+
use Magento\Catalog\Model\Indexer\Category\Product\AbstractAction;
1317

1418
/**
1519
* @api
@@ -29,22 +33,30 @@ class Index extends AbstractDb
2933
*/
3034
protected $metadataPool;
3135

36+
/**
37+
* @var TableResolver
38+
*/
39+
private $tableResolver;
40+
3241
/**
3342
* Index constructor.
3443
* @param Context $context
3544
* @param StoreManagerInterface $storeManager
3645
* @param MetadataPool $metadataPool
3746
* @param null $connectionName
47+
* @param TableResolver|null $tableResolver
3848
*/
3949
public function __construct(
4050
Context $context,
4151
StoreManagerInterface $storeManager,
4252
MetadataPool $metadataPool,
43-
$connectionName = null
53+
$connectionName = null,
54+
TableResolver $tableResolver = null
4455
) {
4556
parent::__construct($context, $connectionName);
4657
$this->storeManager = $storeManager;
4758
$this->metadataPool = $metadataPool;
59+
$this->tableResolver = $tableResolver ?: ObjectManager::getInstance()->get(TableResolver::class);
4860
}
4961

5062
/**
@@ -116,8 +128,17 @@ public function getCategoryProductIndexData($storeId = null, $productIds = null)
116128
{
117129
$connection = $this->getConnection();
118130

131+
$catalogCategoryProductDimension = new Dimension(\Magento\Store\Model\Store::ENTITY, $storeId);
132+
133+
$catalogCategoryProductTableName = $this->tableResolver->resolve(
134+
AbstractAction::MAIN_INDEX_TABLE,
135+
[
136+
$catalogCategoryProductDimension
137+
]
138+
);
139+
119140
$select = $connection->select()->from(
120-
[$this->getTable('catalog_category_product_index')],
141+
[$catalogCategoryProductTableName],
121142
['category_id', 'product_id', 'position', 'store_id']
122143
)->where(
123144
'store_id = ?',

‎app/code/Magento/Catalog/Model/Indexer/Category/Product/AbstractAction.php

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@
99
use Magento\Catalog\Api\Data\ProductInterface;
1010
use Magento\Catalog\Model\Product;
1111
use Magento\Framework\App\ObjectManager;
12-
use Magento\Framework\DB\Query\Generator as QueryGenerator;
1312
use Magento\Framework\App\ResourceConnection;
13+
use Magento\Framework\DB\Query\Generator as QueryGenerator;
1414
use Magento\Framework\DB\Select;
1515
use Magento\Framework\EntityManager\MetadataPool;
1616
use Magento\Store\Model\Store;
@@ -42,6 +42,7 @@ abstract class AbstractAction
4242

4343
/**
4444
* Suffix for table to show it is temporary
45+
* @deprecated
4546
*/
4647
const TEMPORARY_TABLE_SUFFIX = '_tmp';
4748

@@ -106,6 +107,11 @@ abstract class AbstractAction
106107
*/
107108
protected $metadataPool;
108109

110+
/**
111+
* @var TableMaintainer
112+
*/
113+
protected $tableMaintainer;
114+
109115
/**
110116
* @var string
111117
* @since 101.0.0
@@ -121,22 +127,25 @@ abstract class AbstractAction
121127
* @param ResourceConnection $resource
122128
* @param \Magento\Store\Model\StoreManagerInterface $storeManager
123129
* @param \Magento\Catalog\Model\Config $config
124-
* @param QueryGenerator|null $queryGenerator
130+
* @param QueryGenerator $queryGenerator
125131
* @param MetadataPool|null $metadataPool
132+
* @param TableMaintainer|null $tableMaintainer
126133
*/
127134
public function __construct(
128135
\Magento\Framework\App\ResourceConnection $resource,
129136
\Magento\Store\Model\StoreManagerInterface $storeManager,
130137
\Magento\Catalog\Model\Config $config,
131138
QueryGenerator $queryGenerator = null,
132-
MetadataPool $metadataPool = null
139+
MetadataPool $metadataPool = null,
140+
TableMaintainer $tableMaintainer = null
133141
) {
134142
$this->resource = $resource;
135143
$this->connection = $resource->getConnection();
136144
$this->storeManager = $storeManager;
137145
$this->config = $config;
138146
$this->queryGenerator = $queryGenerator ?: ObjectManager::getInstance()->get(QueryGenerator::class);
139147
$this->metadataPool = $metadataPool ?: ObjectManager::getInstance()->get(MetadataPool::class);
148+
$this->tableMaintainer = $tableMaintainer ?: ObjectManager::getInstance()->get(TableMaintainer::class);
140149
}
141150

142151
/**
@@ -180,6 +189,7 @@ protected function getTable($table)
180189
* The name is switched between 'catalog_category_product_index' and 'catalog_category_product_index_replica'
181190
*
182191
* @return string
192+
* @deprecated
183193
*/
184194
protected function getMainTable()
185195
{
@@ -190,6 +200,7 @@ protected function getMainTable()
190200
* Return temporary index table name
191201
*
192202
* @return string
203+
* @deprecated
193204
*/
194205
protected function getMainTmpTable()
195206
{
@@ -198,6 +209,19 @@ protected function getMainTmpTable()
198209
: $this->getMainTable();
199210
}
200211

212+
/**
213+
* Return index table name
214+
*
215+
* @param int $storeId
216+
* @return string
217+
*/
218+
protected function getIndexTable($storeId)
219+
{
220+
return $this->useTempTable
221+
? $this->tableMaintainer->getMainReplicaTable($storeId)
222+
: $this->tableMaintainer->getMainTable($storeId);
223+
}
224+
201225
/**
202226
* Return category path by id
203227
*
@@ -319,7 +343,7 @@ protected function getNonAnchorCategoriesSelect(Store $store)
319343
}
320344

321345
/**
322-
* Add filtering by child products to select.
346+
* Add filtering by child products to select
323347
*
324348
* It's used for correct handling of composite products.
325349
* This method makes assumption that select already joins `catalog_product_entity` as `cpe`.
@@ -422,7 +446,7 @@ protected function reindexNonAnchorCategories(Store $store)
422446
$this->connection->query(
423447
$this->connection->insertFromSelect(
424448
$select,
425-
$this->getMainTmpTable(),
449+
$this->getIndexTable($store->getId()),
426450
['category_id', 'product_id', 'position', 'is_parent', 'store_id', 'visibility'],
427451
\Magento\Framework\DB\Adapter\AdapterInterface::INSERT_ON_DUPLICATE
428452
)
@@ -612,6 +636,12 @@ protected function makeTempCategoryTreeIndex()
612636
['type' => \Magento\Framework\DB\Adapter\AdapterInterface::INDEX_TYPE_PRIMARY]
613637
);
614638

639+
$temporaryTable->addIndex(
640+
'child_id',
641+
['child_id'],
642+
['type' => \Magento\Framework\DB\Adapter\AdapterInterface::INDEX_TYPE_INDEX]
643+
);
644+
615645
// Drop the temporary table in case it already exists on this (persistent?) connection.
616646
$this->connection->dropTemporaryTable($temporaryName);
617647
$this->connection->createTemporaryTable($temporaryTable);
@@ -678,7 +708,7 @@ protected function reindexAnchorCategories(Store $store)
678708
$this->connection->query(
679709
$this->connection->insertFromSelect(
680710
$select,
681-
$this->getMainTmpTable(),
711+
$this->getIndexTable($store->getId()),
682712
['category_id', 'product_id', 'position', 'is_parent', 'store_id', 'visibility'],
683713
\Magento\Framework\DB\Adapter\AdapterInterface::INSERT_ON_DUPLICATE
684714
)
@@ -804,7 +834,7 @@ protected function reindexRootCategory(Store $store)
804834
$this->connection->query(
805835
$this->connection->insertFromSelect(
806836
$select,
807-
$this->getMainTmpTable(),
837+
$this->getIndexTable($store->getId()),
808838
['category_id', 'product_id', 'position', 'is_parent', 'store_id', 'visibility'],
809839
\Magento\Framework\DB\Adapter\AdapterInterface::INSERT_ON_DUPLICATE
810840
)

‎app/code/Magento/Catalog/Model/Indexer/Category/Product/Action/Full.php

Lines changed: 50 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -88,18 +88,35 @@ public function __construct(
8888
}
8989

9090
/**
91-
*
92-
* Clear the table we'll be writing de-normalized data into
93-
* to prevent archived data getting in the way of actual data.
94-
*
9591
* @return void
9692
*/
97-
private function clearCurrentTable()
93+
private function createTables()
9894
{
99-
$this->connection->delete(
100-
$this->activeTableSwitcher
101-
->getAdditionalTableName($this->getMainTable())
102-
);
95+
foreach ($this->storeManager->getStores() as $store) {
96+
$this->tableMaintainer->createTablesForStore($store->getId());
97+
}
98+
}
99+
100+
/**
101+
* @return void
102+
*/
103+
private function clearReplicaTables()
104+
{
105+
foreach ($this->storeManager->getStores() as $store) {
106+
$this->connection->truncateTable($this->tableMaintainer->getMainReplicaTable($store->getId()));
107+
}
108+
}
109+
110+
/**
111+
* @return void
112+
*/
113+
private function switchTables()
114+
{
115+
$tablesToSwitch = [];
116+
foreach ($this->storeManager->getStores() as $store) {
117+
$tablesToSwitch[] = $this->tableMaintainer->getMainTable($store->getId());
118+
}
119+
$this->activeTableSwitcher->switchTable($this->connection, $tablesToSwitch);
103120
}
104121

105122
/**
@@ -109,22 +126,26 @@ private function clearCurrentTable()
109126
*/
110127
public function execute()
111128
{
112-
$this->clearCurrentTable();
129+
$this->createTables();
130+
$this->clearReplicaTables();
113131
$this->reindex();
114-
$this->activeTableSwitcher->switchTable($this->connection, [$this->getMainTable()]);
132+
$this->switchTables();
115133
return $this;
116134
}
117135

118136
/**
119-
* Publish data from tmp to index
137+
* Publish data from tmp to replica table
120138
*
139+
* @param \Magento\Store\Model\Store $store
121140
* @return void
122141
*/
123-
protected function publishData()
142+
private function publishData($store)
124143
{
125-
$select = $this->connection->select()->from($this->getMainTmpTable());
126-
$columns = array_keys($this->connection->describeTable($this->getMainTable()));
127-
$tableName = $this->activeTableSwitcher->getAdditionalTableName($this->getMainTable());
144+
$select = $this->connection->select()->from($this->tableMaintainer->getMainTmpTable($store->getId()));
145+
$columns = array_keys(
146+
$this->connection->describeTable($this->tableMaintainer->getMainReplicaTable($store->getId()))
147+
);
148+
$tableName = $this->tableMaintainer->getMainReplicaTable($store->getId());
128149

129150
$this->connection->query(
130151
$this->connection->insertFromSelect(
@@ -136,23 +157,13 @@ protected function publishData()
136157
);
137158
}
138159

139-
/**
140-
* Clear all index data
141-
*
142-
* @return void
143-
*/
144-
protected function clearTmpData()
145-
{
146-
$this->connection->delete($this->getMainTmpTable());
147-
}
148-
149160
/**
150161
* {@inheritdoc}
151162
*/
152163
protected function reindexRootCategory(\Magento\Store\Model\Store $store)
153164
{
154165
if ($this->isIndexRootCategoryNeeded()) {
155-
$this->reindexCategoriesBySelect($this->getAllProducts($store), 'cp.entity_id IN (?)');
166+
$this->reindexCategoriesBySelect($this->getAllProducts($store), 'cp.entity_id IN (?)', $store);
156167
}
157168
}
158169

@@ -164,7 +175,7 @@ protected function reindexRootCategory(\Magento\Store\Model\Store $store)
164175
*/
165176
protected function reindexAnchorCategories(\Magento\Store\Model\Store $store)
166177
{
167-
$this->reindexCategoriesBySelect($this->getAnchorCategoriesSelect($store), 'ccp.product_id IN (?)');
178+
$this->reindexCategoriesBySelect($this->getAnchorCategoriesSelect($store), 'ccp.product_id IN (?)', $store);
168179
}
169180

170181
/**
@@ -175,20 +186,25 @@ protected function reindexAnchorCategories(\Magento\Store\Model\Store $store)
175186
*/
176187
protected function reindexNonAnchorCategories(\Magento\Store\Model\Store $store)
177188
{
178-
$this->reindexCategoriesBySelect($this->getNonAnchorCategoriesSelect($store), 'ccp.product_id IN (?)');
189+
$this->reindexCategoriesBySelect($this->getNonAnchorCategoriesSelect($store), 'ccp.product_id IN (?)', $store);
179190
}
180191

181192
/**
182193
* Reindex categories using given SQL select and condition.
183194
*
184195
* @param \Magento\Framework\DB\Select $basicSelect
185196
* @param string $whereCondition
197+
* @param \Magento\Store\Model\Store $store
186198
* @return void
187199
*/
188-
private function reindexCategoriesBySelect(\Magento\Framework\DB\Select $basicSelect, $whereCondition)
200+
private function reindexCategoriesBySelect(\Magento\Framework\DB\Select $basicSelect, $whereCondition, $store)
189201
{
202+
$this->tableMaintainer->createMainTmpTable($store->getId());
203+
190204
$entityMetadata = $this->metadataPool->getMetadata(\Magento\Catalog\Api\Data\ProductInterface::class);
191-
$columns = array_keys($this->connection->describeTable($this->getMainTmpTable()));
205+
$columns = array_keys(
206+
$this->connection->describeTable($this->tableMaintainer->getMainTmpTable($store->getId()))
207+
);
192208
$this->batchSizeManagement->ensureBatchSize($this->connection, $this->batchRowsCount);
193209
$batches = $this->batchProvider->getBatches(
194210
$this->connection,
@@ -197,7 +213,7 @@ private function reindexCategoriesBySelect(\Magento\Framework\DB\Select $basicSe
197213
$this->batchRowsCount
198214
);
199215
foreach ($batches as $batch) {
200-
$this->clearTmpData();
216+
$this->connection->delete($this->tableMaintainer->getMainTmpTable($store->getId()));
201217
$resultSelect = clone $basicSelect;
202218
$select = $this->connection->select();
203219
$select->distinct(true);
@@ -207,12 +223,12 @@ private function reindexCategoriesBySelect(\Magento\Framework\DB\Select $basicSe
207223
$this->connection->query(
208224
$this->connection->insertFromSelect(
209225
$resultSelect,
210-
$this->getMainTmpTable(),
226+
$this->tableMaintainer->getMainTmpTable($store->getId()),
211227
$columns,
212228
\Magento\Framework\DB\Adapter\AdapterInterface::INSERT_ON_DUPLICATE
213229
)
214230
);
215-
$this->publishData();
231+
$this->publishData($store);
216232
}
217233
}
218234
}

0 commit comments

Comments
 (0)