Skip to content

Commit 9b5f0db

Browse files
duhonduhon
authored
Merge pull request #2287 from magento-performance/pr-2.2
Tasks: - MAGETWO-88775 Few optimizations on category & product pages - MAGETWO-88808 Implement parallelization for Category Product Indexer - MAGETWO-86046 [Performance] Temp table temp_catalog_category_tree_index has no indexes for the columns parent_id and child_
2 parents 02a512f + e01f7e9 commit 9b5f0db

File tree

15 files changed

+264
-106
lines changed

15 files changed

+264
-106
lines changed

‎app/code/Magento/Catalog/Model/Indexer/Category/Product/AbstractAction.php

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -608,6 +608,12 @@ protected function makeTempCategoryTreeIndex()
608608
['type' => \Magento\Framework\DB\Adapter\AdapterInterface::INDEX_TYPE_PRIMARY]
609609
);
610610

611+
$temporaryTable->addIndex(
612+
'child_id',
613+
['child_id'],
614+
['type' => \Magento\Framework\DB\Adapter\AdapterInterface::INDEX_TYPE_INDEX]
615+
);
616+
611617
// Drop the temporary table in case it already exists on this (persistent?) connection.
612618
$this->connection->dropTemporaryTable($temporaryName);
613619
$this->connection->createTemporaryTable($temporaryTable);

‎app/code/Magento/Catalog/Model/Indexer/Category/Product/Action/Full.php

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
use Magento\Catalog\Model\ResourceModel\Indexer\ActiveTableSwitcher;
99
use Magento\Framework\DB\Query\Generator as QueryGenerator;
1010
use Magento\Framework\App\ResourceConnection;
11+
use Magento\Indexer\Model\ProcessManager;
1112

1213
/**
1314
* Class Full reindex action
@@ -44,6 +45,16 @@ class Full extends \Magento\Catalog\Model\Indexer\Category\Product\AbstractActio
4445
*/
4546
private $activeTableSwitcher;
4647

48+
/**
49+
* @var ProcessManager
50+
*/
51+
private $processManager;
52+
53+
/**
54+
* @var string[]
55+
*/
56+
private $mainTmpTable;
57+
4758
/**
4859
* @param ResourceConnection $resource
4960
* @param \Magento\Store\Model\StoreManagerInterface $storeManager
@@ -54,6 +65,8 @@ class Full extends \Magento\Catalog\Model\Indexer\Category\Product\AbstractActio
5465
* @param \Magento\Framework\EntityManager\MetadataPool|null $metadataPool
5566
* @param int|null $batchRowsCount
5667
* @param ActiveTableSwitcher|null $activeTableSwitcher
68+
* @param ProcessManager $processManager
69+
* @SuppressWarnings(PHPMD.ExcessiveParameterList)
5770
*/
5871
public function __construct(
5972
\Magento\Framework\App\ResourceConnection $resource,
@@ -64,7 +77,8 @@ public function __construct(
6477
\Magento\Framework\Indexer\BatchProviderInterface $batchProvider = null,
6578
\Magento\Framework\EntityManager\MetadataPool $metadataPool = null,
6679
$batchRowsCount = null,
67-
ActiveTableSwitcher $activeTableSwitcher = null
80+
ActiveTableSwitcher $activeTableSwitcher = null,
81+
ProcessManager $processManager = null
6882
) {
6983
parent::__construct(
7084
$resource,
@@ -84,6 +98,7 @@ public function __construct(
8498
);
8599
$this->batchRowsCount = $batchRowsCount;
86100
$this->activeTableSwitcher = $activeTableSwitcher ?: $objectManager->get(ActiveTableSwitcher::class);
101+
$this->processManager = $processManager ?: $objectManager->get(ProcessManager::class);
87102
}
88103

89104
/**
@@ -113,6 +128,38 @@ public function execute()
113128
return $this;
114129
}
115130

131+
/**
132+
* Run reindexation
133+
*
134+
* @return void
135+
*/
136+
protected function reindex()
137+
{
138+
$userFunctions = [];
139+
140+
foreach ($this->storeManager->getStores() as $store) {
141+
if ($this->getPathFromCategoryId($store->getRootCategoryId())) {
142+
$userFunctions[$store->getId()] = function () use ($store) {
143+
return $this->reindexStore($store);
144+
};
145+
}
146+
}
147+
148+
$this->processManager->execute($userFunctions);
149+
}
150+
151+
/**
152+
* Execute indexation by store
153+
*
154+
* @param \Magento\Store\Model\Store $store
155+
*/
156+
private function reindexStore($store)
157+
{
158+
$this->reindexRootCategory($store);
159+
$this->reindexAnchorCategories($store);
160+
$this->reindexNonAnchorCategories($store);
161+
}
162+
116163
/**
117164
* Return select for remove unnecessary data
118165
*
@@ -254,4 +301,21 @@ private function reindexCategoriesBySelect(\Magento\Framework\DB\Select $basicSe
254301
$this->removeUnnecessaryData();
255302
}
256303
}
304+
305+
/**
306+
* Create and return temporary index table name for current pid
307+
*
308+
* @return string
309+
*/
310+
protected function getMainTmpTable()
311+
{
312+
$pid = getmypid();
313+
if (!isset($this->mainTmpTable[$pid])) {
314+
$originTableName = $this->connection->getTableName(parent::getMainTmpTable());
315+
$temporaryTableName = $originTableName . $pid;
316+
$this->connection->createTemporaryTableLike($temporaryTableName, $originTableName, true);
317+
$this->mainTmpTable[$pid] = $temporaryTableName;
318+
}
319+
return $this->mainTmpTable[$pid];
320+
}
257321
}

‎app/code/Magento/Catalog/Model/Product/Gallery/ReadHandler.php

Lines changed: 7 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,6 @@ public function __construct(
5555
*/
5656
public function execute($entity, $arguments = [])
5757
{
58-
$value = [];
59-
$value['images'] = [];
60-
6158
$mediaEntries = $this->resourceModel->loadProductGalleryByAttributeId(
6259
$entity,
6360
$this->getAttribute()->getAttributeId()
@@ -79,37 +76,13 @@ public function execute($entity, $arguments = [])
7976
*/
8077
public function addMediaDataToProduct(Product $product, array $mediaEntries)
8178
{
82-
$attrCode = $this->getAttribute()->getAttributeCode();
83-
$value = [];
84-
$value['images'] = [];
85-
$value['values'] = [];
86-
87-
foreach ($mediaEntries as $mediaEntry) {
88-
$mediaEntry = $this->substituteNullsWithDefaultValues($mediaEntry);
89-
$value['images'][$mediaEntry['value_id']] = $mediaEntry;
90-
}
91-
$product->setData($attrCode, $value);
92-
}
93-
94-
/**
95-
* @param array $rawData
96-
* @return array
97-
*/
98-
private function substituteNullsWithDefaultValues(array $rawData)
99-
{
100-
$processedData = [];
101-
foreach ($rawData as $key => $rawValue) {
102-
if (null !== $rawValue) {
103-
$processedValue = $rawValue;
104-
} elseif (isset($rawData[$key . '_default'])) {
105-
$processedValue = $rawData[$key . '_default'];
106-
} else {
107-
$processedValue = null;
108-
}
109-
$processedData[$key] = $processedValue;
110-
}
111-
112-
return $processedData;
79+
$product->setData(
80+
$this->getAttribute()->getAttributeCode(),
81+
[
82+
'images' => array_column($mediaEntries, null, 'value_id'),
83+
'values' => []
84+
]
85+
);
11386
}
11487

11588
/**

‎app/code/Magento/Catalog/Model/ResourceModel/Product/Collection.php

Lines changed: 25 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1623,7 +1623,7 @@ public function addOptionsToResult()
16231623
*/
16241624
public function addFilterByRequiredOptions()
16251625
{
1626-
$this->addAttributeToFilter('required_options', [['neq' => 1], ['null' => true]], 'left');
1626+
$this->addAttributeToFilter('required_options', [['neq' => 1]], 'left');
16271627
return $this;
16281628
}
16291629

@@ -2218,6 +2218,7 @@ public function addPriceDataFieldFilter($comparisonFormat, $fields)
22182218
* @SuppressWarnings(PHPMD.CyclomaticComplexity)
22192219
* @SuppressWarnings(PHPMD.NPathComplexity)
22202220
* @since 101.0.1
2221+
* @throws \Magento\Framework\Exception\LocalizedException
22212222
*/
22222223
public function addMediaGalleryData()
22232224
{
@@ -2229,34 +2230,36 @@ public function addMediaGalleryData()
22292230
return $this;
22302231
}
22312232

2232-
/** @var $attribute \Magento\Catalog\Model\ResourceModel\Eav\Attribute */
2233-
$attribute = $this->getAttribute('media_gallery');
2234-
$select = $this->getMediaGalleryResource()->createBatchBaseSelect(
2235-
$this->getStoreId(),
2236-
$attribute->getAttributeId()
2237-
);
2238-
2239-
$mediaGalleries = [];
2240-
$linkField = $this->getProductEntityMetadata()->getLinkField();
22412233
$items = $this->getItems();
2234+
$linkField = $this->getProductEntityMetadata()->getLinkField();
22422235

2243-
$select->where(
2244-
'entity.' . $linkField . ' IN (?)',
2245-
array_map(
2246-
function ($item) use ($linkField) {
2247-
return $item->getData($linkField);
2248-
},
2249-
$items
2250-
)
2251-
);
2236+
$select = $this->getMediaGalleryResource()
2237+
->createBatchBaseSelect(
2238+
$this->getStoreId(),
2239+
$this->getAttribute('media_gallery')->getAttributeId()
2240+
)->reset(
2241+
Select::ORDER // we don't care what order is in current scenario
2242+
)->where(
2243+
'entity.' . $linkField . ' IN (?)',
2244+
array_map(
2245+
function ($item) use ($linkField) {
2246+
return (int) $item->getOrigData($linkField);
2247+
},
2248+
$items
2249+
)
2250+
);
2251+
2252+
$mediaGalleries = [];
22522253
foreach ($this->getConnection()->fetchAll($select) as $row) {
22532254
$mediaGalleries[$row[$linkField]][] = $row;
22542255
}
22552256

22562257
foreach ($items as $item) {
2257-
$mediaEntries = isset($mediaGalleries[$item->getData($linkField)]) ?
2258-
$mediaGalleries[$item->getData($linkField)] : [];
2259-
$this->getGalleryReadHandler()->addMediaDataToProduct($item, $mediaEntries);
2258+
$this->getGalleryReadHandler()
2259+
->addMediaDataToProduct(
2260+
$item,
2261+
$mediaGalleries[$item->getOrigData($linkField)] ?? []
2262+
);
22602263
}
22612264

22622265
$this->setFlag('media_gallery_added', true);

‎app/code/Magento/Catalog/Model/ResourceModel/Product/Gallery.php

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,7 @@ public function createBatchBaseSelect($storeId, $attributeId)
190190
'value.' . $linkField . ' = entity.' . $linkField,
191191
]
192192
),
193-
['label', 'position', 'disabled']
193+
[]
194194
)->joinLeft(
195195
['default_value' => $this->getTable(self::GALLERY_VALUE_TABLE)],
196196
implode(
@@ -201,8 +201,15 @@ public function createBatchBaseSelect($storeId, $attributeId)
201201
'default_value.' . $linkField . ' = entity.' . $linkField,
202202
]
203203
),
204-
['label_default' => 'label', 'position_default' => 'position', 'disabled_default' => 'disabled']
205-
)->where(
204+
[]
205+
)->columns([
206+
'label' => $this->getConnection()->getIfNullSql('`value`.`label`', '`default_value`.`label`'),
207+
'position' => $this->getConnection()->getIfNullSql('`value`.`position`', '`default_value`.`position`'),
208+
'disabled' => $this->getConnection()->getIfNullSql('`value`.`disabled`', '`default_value`.`disabled`'),
209+
'label_default' => 'default_value.label',
210+
'position_default' => 'default_value.position',
211+
'disabled_default' => 'default_value.disabled'
212+
])->where(
206213
$mainTableAlias . '.attribute_id = ?',
207214
$attributeId
208215
)->where(

‎app/code/Magento/Catalog/Setup/InstallSchema.php

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2077,6 +2077,13 @@ public function install(SchemaSetupInterface $setup, ModuleContextInterface $con
20772077
['unsigned' => true, 'nullable' => false, 'default' => '0'],
20782078
'Is Disabled'
20792079
)
2080+
->addIndex(
2081+
$installer->getIdxName(
2082+
'catalog_product_entity_media_gallery_value',
2083+
['entity_id', 'value_id', 'store_id']
2084+
),
2085+
['entity_id', 'value_id', 'store_id']
2086+
)
20802087
->addIndex(
20812088
$installer->getIdxName('catalog_product_entity_media_gallery_value', ['store_id']),
20822089
['store_id']

‎app/code/Magento/Catalog/Setup/UpgradeSchema.php

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
namespace Magento\Catalog\Setup;
88

99
use Magento\Catalog\Model\Product\Attribute\Backend\Media\ImageEntryConverter;
10+
use Magento\Catalog\Model\Product\Exception;
1011
use Magento\Catalog\Model\ResourceModel\Product\Gallery;
1112
use Magento\Framework\Setup\ModuleContextInterface;
1213
use Magento\Framework\Setup\SchemaSetupInterface;
@@ -27,6 +28,7 @@ class UpgradeSchema implements UpgradeSchemaInterface
2728
public function upgrade(SchemaSetupInterface $setup, ModuleContextInterface $context)
2829
{
2930
$setup->startSetup();
31+
3032
if (version_compare($context->getVersion(), '2.0.1', '<')) {
3133
$this->addSupportVideoMediaAttributes($setup);
3234
$this->removeGroupPrice($setup);
@@ -132,6 +134,10 @@ public function upgrade(SchemaSetupInterface $setup, ModuleContextInterface $con
132134
$this->removeAttributeSetRelation($setup);
133135
}
134136

137+
if (version_compare($context->getVersion(), '2.2.5', '<')) {
138+
$this->addGeneralIndexOnGalleryValueTable($setup);
139+
}
140+
135141
$setup->endSetup();
136142
}
137143

@@ -523,6 +529,7 @@ private function addSupportVideoMediaAttributes(SchemaSetupInterface $setup)
523529
),
524530
'value_id'
525531
);
532+
526533
$this->addForeignKeys($setup);
527534
}
528535

@@ -721,4 +728,43 @@ private function removeAttributeSetRelation(SchemaSetupInterface $setup)
721728
$setup->getFkName('catalog_product_entity', 'attribute_set_id', 'eav_attribute_set', 'attribute_set_id')
722729
);
723730
}
731+
732+
/**
733+
* Adds index for table catalog_product_entity_media_gallery_value
734+
* It was added because it suits best for selecting media data for products
735+
*
736+
* @see \Magento\Catalog\Model\ResourceModel\Product\Gallery::createBatchBaseSelect
737+
* @param SchemaSetupInterface $setup
738+
* @return void
739+
* @throws \Exception
740+
*/
741+
private function addGeneralIndexOnGalleryValueTable(SchemaSetupInterface $setup)
742+
{
743+
$existingKeys = $setup->getConnection()->getIndexList(
744+
$setup->getTable(Gallery::GALLERY_VALUE_TABLE)
745+
);
746+
747+
$newIndexName = $setup->getConnection()->getIndexName(
748+
$setup->getTable(Gallery::GALLERY_VALUE_TABLE),
749+
['entity_id', 'value_id', 'store_id']
750+
);
751+
752+
if (!array_key_exists($newIndexName, $existingKeys)) {
753+
$entityIdKeyName = $setup->getConnection()->getIndexName(
754+
$setup->getTable(Gallery::GALLERY_VALUE_TABLE),
755+
['entity_id']
756+
);
757+
758+
if (array_key_exists($entityIdKeyName, $existingKeys)) {
759+
$keyColumns = $existingKeys[$entityIdKeyName]['COLUMNS_LIST'];
760+
$linkField = reset($keyColumns);
761+
762+
$setup->getConnection()->addIndex(
763+
$setup->getTable(Gallery::GALLERY_VALUE_TABLE),
764+
$newIndexName,
765+
[$linkField, 'value_id', 'store_id']
766+
);
767+
}
768+
}
769+
}
724770
}

0 commit comments

Comments
 (0)