Skip to content

Commit c515b40

Browse files
duhonduhon
authored
Merge pull request #3305 from magento-performance/pr-2.3-develop
### Task * [MAGETWO-95275](https://jira.corp.magento.com/browse/MAGETWO-95275) Varnish "Connection reset by peer" error when large catalog is reindexed on schedule
2 parents 7fc1ef6 + 287a5dc commit c515b40

File tree

7 files changed

+87
-26
lines changed

7 files changed

+87
-26
lines changed

‎app/code/Magento/CacheInvalidate/Model/PurgeCache.php

Lines changed: 67 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@
77

88
use Magento\Framework\Cache\InvalidateLogger;
99

10+
/**
11+
* Class PurgeCache
12+
*/
1013
class PurgeCache
1114
{
1215
const HEADER_X_MAGENTO_TAGS_PATTERN = 'X-Magento-Tags-Pattern';
@@ -26,6 +29,18 @@ class PurgeCache
2629
*/
2730
private $logger;
2831

32+
/**
33+
* Batch size of the purge request.
34+
*
35+
* Based on default Varnish 4 http_req_hdr_len size minus a 512 bytes margin for method,
36+
* header name, line feeds etc.
37+
*
38+
* @see https://varnish-cache.org/docs/4.1/reference/varnishd.html
39+
*
40+
* @var int
41+
*/
42+
private $requestSize = 7680;
43+
2944
/**
3045
* Constructor
3146
*
@@ -44,18 +59,65 @@ public function __construct(
4459
}
4560

4661
/**
47-
* Send curl purge request
48-
* to invalidate cache by tags pattern
62+
* Send curl purge request to invalidate cache by tags pattern
4963
*
5064
* @param string $tagsPattern
5165
* @return bool Return true if successful; otherwise return false
5266
*/
5367
public function sendPurgeRequest($tagsPattern)
5468
{
69+
$successful = true;
5570
$socketAdapter = $this->socketAdapterFactory->create();
5671
$servers = $this->cacheServer->getUris();
57-
$headers = [self::HEADER_X_MAGENTO_TAGS_PATTERN => $tagsPattern];
5872
$socketAdapter->setOptions(['timeout' => 10]);
73+
74+
$formattedTagsChunks = $this->splitTags($tagsPattern);
75+
foreach ($formattedTagsChunks as $formattedTagsChunk) {
76+
if (!$this->sendPurgeRequestToServers($socketAdapter, $servers, $formattedTagsChunk)) {
77+
$successful = false;
78+
}
79+
}
80+
81+
return $successful;
82+
}
83+
84+
/**
85+
* Split tags by batches
86+
*
87+
* @param string $tagsPattern
88+
* @return \Generator
89+
*/
90+
private function splitTags($tagsPattern)
91+
{
92+
$tagsBatchSize = 0;
93+
$formattedTagsChunk = [];
94+
$formattedTags = explode('|', $tagsPattern);
95+
foreach ($formattedTags as $formattedTag) {
96+
if ($tagsBatchSize + strlen($formattedTag) > $this->requestSize - count($formattedTagsChunk) - 1) {
97+
yield implode('|', array_unique($formattedTagsChunk));
98+
$formattedTagsChunk = [];
99+
$tagsBatchSize = 0;
100+
}
101+
102+
$tagsBatchSize += strlen($formattedTag);
103+
$formattedTagsChunk[] = $formattedTag;
104+
}
105+
if (!empty($formattedTagsChunk)) {
106+
yield implode('|', array_unique($formattedTagsChunk));
107+
}
108+
}
109+
110+
/**
111+
* Send curl purge request to servers to invalidate cache by tags pattern
112+
*
113+
* @param \Zend\Http\Client\Adapter\Socket $socketAdapter
114+
* @param \Zend\Uri\Uri[] $servers
115+
* @param string $formattedTagsChunk
116+
* @return bool Return true if successful; otherwise return false
117+
*/
118+
private function sendPurgeRequestToServers($socketAdapter, $servers, $formattedTagsChunk)
119+
{
120+
$headers = [self::HEADER_X_MAGENTO_TAGS_PATTERN => $formattedTagsChunk];
59121
foreach ($servers as $server) {
60122
$headers['Host'] = $server->getHost();
61123
try {
@@ -69,12 +131,11 @@ public function sendPurgeRequest($tagsPattern)
69131
$socketAdapter->read();
70132
$socketAdapter->close();
71133
} catch (\Exception $e) {
72-
$this->logger->critical($e->getMessage(), compact('server', 'tagsPattern'));
134+
$this->logger->critical($e->getMessage(), compact('server', 'formattedTagsChunk'));
73135
return false;
74136
}
75137
}
76-
77-
$this->logger->execute(compact('servers', 'tagsPattern'));
138+
$this->logger->execute(compact('servers', 'formattedTagsChunk'));
78139
return true;
79140
}
80141
}

‎setup/performance-toolkit/config/description.xml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,16 @@
77
-->
88
<description>
99
<paragraphs>
10-
<count-min>4</count-min>
11-
<count-max>10</count-max>
10+
<count-min>7</count-min>
11+
<count-max>7</count-max>
1212

1313
<sentences>
14-
<count-min>10</count-min>
15-
<count-max>15</count-max>
14+
<count-min>12</count-min>
15+
<count-max>12</count-max>
1616

1717
<words>
18-
<count-min>5</count-min>
19-
<count-max>7</count-max>
18+
<count-min>6</count-min>
19+
<count-max>6</count-max>
2020
</words>
2121
</sentences>
2222
</paragraphs>

‎setup/performance-toolkit/profiles/ce/extra_large.xml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,9 @@
4040
<cart_price_rules>20</cart_price_rules> <!-- Number of cart price rules -->
4141
<cart_price_rules_floor>2</cart_price_rules_floor>
4242

43-
<product_attribute_sets>200</product_attribute_sets> <!-- Number of product attribute sets -->
44-
<product_attribute_sets_attributes>50</product_attribute_sets_attributes> <!-- Number of attributes per set -->
45-
<product_attribute_sets_attributes_values>2</product_attribute_sets_attributes_values> <!-- Number of values per attribute -->
43+
<product_attribute_sets>100</product_attribute_sets> <!-- Number of product attribute sets -->
44+
<product_attribute_sets_attributes>30</product_attribute_sets_attributes> <!-- Number of attributes per set -->
45+
<product_attribute_sets_attributes_values>15</product_attribute_sets_attributes_values> <!-- Number of values per attribute -->
4646

4747
<order_quotes_enable>true</order_quotes_enable>
4848
<order_simple_product_count_from>2</order_simple_product_count_from>

‎setup/performance-toolkit/profiles/ce/large.xml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,9 @@
4040
<cart_price_rules>20</cart_price_rules> <!-- Number of cart price rules -->
4141
<cart_price_rules_floor>2</cart_price_rules_floor>
4242

43-
<product_attribute_sets>200</product_attribute_sets> <!-- Number of product attribute sets -->
44-
<product_attribute_sets_attributes>50</product_attribute_sets_attributes> <!-- Number of attributes per set -->
45-
<product_attribute_sets_attributes_values>2</product_attribute_sets_attributes_values> <!-- Number of values per attribute -->
43+
<product_attribute_sets>50</product_attribute_sets> <!-- Number of product attribute sets -->
44+
<product_attribute_sets_attributes>20</product_attribute_sets_attributes> <!-- Number of attributes per set -->
45+
<product_attribute_sets_attributes_values>15</product_attribute_sets_attributes_values> <!-- Number of values per attribute -->
4646

4747
<order_quotes_enable>true</order_quotes_enable>
4848
<order_simple_product_count_from>2</order_simple_product_count_from>

‎setup/performance-toolkit/profiles/ce/medium.xml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,9 @@
4040
<cart_price_rules>20</cart_price_rules> <!-- Number of cart price rules -->
4141
<cart_price_rules_floor>2</cart_price_rules_floor>
4242

43-
<product_attribute_sets>100</product_attribute_sets> <!-- Number of product attribute sets -->
44-
<product_attribute_sets_attributes>50</product_attribute_sets_attributes> <!-- Number of attributes per set -->
45-
<product_attribute_sets_attributes_values>2</product_attribute_sets_attributes_values> <!-- Number of values per attribute -->
43+
<product_attribute_sets>30</product_attribute_sets> <!-- Number of product attribute sets -->
44+
<product_attribute_sets_attributes>10</product_attribute_sets_attributes> <!-- Number of attributes per set -->
45+
<product_attribute_sets_attributes_values>8</product_attribute_sets_attributes_values> <!-- Number of values per attribute -->
4646

4747
<order_quotes_enable>true</order_quotes_enable>
4848
<order_simple_product_count_from>2</order_simple_product_count_from>

‎setup/performance-toolkit/profiles/ce/medium_msite.xml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,9 @@
4646
<cart_price_rules>20</cart_price_rules> <!-- Number of cart price rules -->
4747
<cart_price_rules_floor>2</cart_price_rules_floor>
4848

49-
<product_attribute_sets>100</product_attribute_sets> <!-- Number of product attribute sets -->
50-
<product_attribute_sets_attributes>50</product_attribute_sets_attributes> <!-- Number of attributes per set -->
51-
<product_attribute_sets_attributes_values>2</product_attribute_sets_attributes_values> <!-- Number of values per attribute -->
49+
<product_attribute_sets>30</product_attribute_sets> <!-- Number of product attribute sets -->
50+
<product_attribute_sets_attributes>10</product_attribute_sets_attributes> <!-- Number of attributes per set -->
51+
<product_attribute_sets_attributes_values>8</product_attribute_sets_attributes_values> <!-- Number of values per attribute -->
5252

5353
<order_quotes_enable>true</order_quotes_enable>
5454
<order_simple_product_count_from>2</order_simple_product_count_from>

‎setup/performance-toolkit/profiles/ce/small.xml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,8 @@
4141
<cart_price_rules_floor>2</cart_price_rules_floor>
4242

4343
<product_attribute_sets>10</product_attribute_sets> <!-- Number of product attribute sets -->
44-
<product_attribute_sets_attributes>10</product_attribute_sets_attributes> <!-- Number of attributes per set -->
45-
<product_attribute_sets_attributes_values>2</product_attribute_sets_attributes_values> <!-- Number of values per attribute -->
44+
<product_attribute_sets_attributes>5</product_attribute_sets_attributes> <!-- Number of attributes per set -->
45+
<product_attribute_sets_attributes_values>5</product_attribute_sets_attributes_values> <!-- Number of values per attribute -->
4646

4747
<order_quotes_enable>true</order_quotes_enable>
4848
<order_simple_product_count_from>2</order_simple_product_count_from>

0 commit comments

Comments
 (0)