Skip to content

Commit e86de06

Browse files
droberts195droberts195
authored
Allow total memory to be overridden (#78750)
Since #65905 Elasticsearch has determined the Java heap settings from node roles and total system memory. This change allows the total system memory used in that calculation to be overridden with a user-specified value. This is intended to be used when Elasticsearch is running on a machine where some other software that consumes a non-negligible amount of memory is running. For example, a user could tell Elasticsearch to assume it was running on a machine with 3GB of RAM when actually it was running on a machine with 4GB of RAM. The system property is `es.total_memory_bytes`, so, for example, could be specified using `-Des.total_memory_bytes=3221225472`. (It is specified in bytes rather than using a unit, because it needs to be parsed by startup code that does not have access to the utility classes that interpret byte size units.)
1 parent 510f54a commit e86de06

File tree

20 files changed

+364
-87
lines changed

20 files changed

+364
-87
lines changed

‎distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,6 @@ private List<String> jvmOptions(final Path config, Path plugins, final String es
123123
throws InterruptedException, IOException, JvmOptionsFileParserException {
124124

125125
final List<String> jvmOptions = readJvmOptionsFiles(config);
126-
final MachineDependentHeap machineDependentHeap = new MachineDependentHeap(new DefaultSystemMemoryInfo());
127126

128127
if (esJavaOpts != null) {
129128
jvmOptions.addAll(
@@ -132,6 +131,9 @@ private List<String> jvmOptions(final Path config, Path plugins, final String es
132131
}
133132

134133
final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
134+
final MachineDependentHeap machineDependentHeap = new MachineDependentHeap(
135+
new OverridableSystemMemoryInfo(substitutedJvmOptions, new DefaultSystemMemoryInfo())
136+
);
135137
substitutedJvmOptions.addAll(machineDependentHeap.determineHeapSettings(config, substitutedJvmOptions));
136138
final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions);
137139
final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions();

‎distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/OverridableSystemMemoryInfo.java

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
package org.elasticsearch.tools.launchers;
10+
11+
import java.util.List;
12+
import java.util.Objects;
13+
14+
/**
15+
* A {@link SystemMemoryInfo} which returns a user-overridden memory size if one
16+
* has been specified using the {@code es.total_memory_bytes} system property, or
17+
* else returns the value provided by a fallback provider.
18+
*/
19+
public final class OverridableSystemMemoryInfo implements SystemMemoryInfo {
20+
21+
private final List<String> userDefinedJvmOptions;
22+
private final SystemMemoryInfo fallbackSystemMemoryInfo;
23+
24+
public OverridableSystemMemoryInfo(final List<String> userDefinedJvmOptions, SystemMemoryInfo fallbackSystemMemoryInfo) {
25+
this.userDefinedJvmOptions = Objects.requireNonNull(userDefinedJvmOptions);
26+
this.fallbackSystemMemoryInfo = Objects.requireNonNull(fallbackSystemMemoryInfo);
27+
}
28+
29+
@Override
30+
public long availableSystemMemory() throws SystemMemoryInfoException {
31+
32+
return userDefinedJvmOptions.stream()
33+
.filter(option -> option.startsWith("-Des.total_memory_bytes="))
34+
.map(totalMemoryBytesOption -> {
35+
try {
36+
long bytes = Long.parseLong(totalMemoryBytesOption.split("=", 2)[1]);
37+
if (bytes < 0) {
38+
throw new IllegalArgumentException("Negative memory size specified in [" + totalMemoryBytesOption + "]");
39+
}
40+
return bytes;
41+
} catch (NumberFormatException e) {
42+
throw new IllegalArgumentException("Unable to parse number of bytes from [" + totalMemoryBytesOption + "]", e);
43+
}
44+
})
45+
.reduce((previous, current) -> current) // this is effectively findLast(), so that ES_JAVA_OPTS overrides jvm.options
46+
.orElse(fallbackSystemMemoryInfo.availableSystemMemory());
47+
}
48+
}

‎distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/OverridableSystemMemoryInfoTests.java

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
package org.elasticsearch.tools.launchers;
10+
11+
import org.elasticsearch.tools.launchers.SystemMemoryInfo.SystemMemoryInfoException;
12+
13+
import java.util.List;
14+
15+
import static org.hamcrest.Matchers.is;
16+
import static org.junit.Assert.assertThat;
17+
import static org.junit.Assert.fail;
18+
19+
public class OverridableSystemMemoryInfoTests extends LaunchersTestCase {
20+
21+
private static final long FALLBACK = -1L;
22+
23+
public void testNoOptions() throws SystemMemoryInfoException {
24+
final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(List.of(), fallbackSystemMemoryInfo());
25+
assertThat(memoryInfo.availableSystemMemory(), is(FALLBACK));
26+
}
27+
28+
public void testNoOverrides() throws SystemMemoryInfoException {
29+
final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(List.of("-Da=b", "-Dx=y"), fallbackSystemMemoryInfo());
30+
assertThat(memoryInfo.availableSystemMemory(), is(FALLBACK));
31+
}
32+
33+
public void testValidSingleOverride() throws SystemMemoryInfoException {
34+
final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(
35+
List.of("-Des.total_memory_bytes=123456789"),
36+
fallbackSystemMemoryInfo()
37+
);
38+
assertThat(memoryInfo.availableSystemMemory(), is(123456789L));
39+
}
40+
41+
public void testValidOverrideInList() throws SystemMemoryInfoException {
42+
final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(
43+
List.of("-Da=b", "-Des.total_memory_bytes=987654321", "-Dx=y"),
44+
fallbackSystemMemoryInfo()
45+
);
46+
assertThat(memoryInfo.availableSystemMemory(), is(987654321L));
47+
}
48+
49+
public void testMultipleValidOverridesInList() throws SystemMemoryInfoException {
50+
final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(
51+
List.of("-Des.total_memory_bytes=123456789", "-Da=b", "-Des.total_memory_bytes=987654321", "-Dx=y"),
52+
fallbackSystemMemoryInfo()
53+
);
54+
assertThat(memoryInfo.availableSystemMemory(), is(987654321L));
55+
}
56+
57+
public void testNegativeOverride() throws SystemMemoryInfoException {
58+
final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(
59+
List.of("-Da=b", "-Des.total_memory_bytes=-123", "-Dx=y"),
60+
fallbackSystemMemoryInfo()
61+
);
62+
try {
63+
memoryInfo.availableSystemMemory();
64+
fail("expected to fail");
65+
} catch (IllegalArgumentException e) {
66+
assertThat(e.getMessage(), is("Negative memory size specified in [-Des.total_memory_bytes=-123]"));
67+
}
68+
}
69+
70+
public void testUnparsableOverride() throws SystemMemoryInfoException {
71+
final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(
72+
List.of("-Da=b", "-Des.total_memory_bytes=invalid", "-Dx=y"),
73+
fallbackSystemMemoryInfo()
74+
);
75+
try {
76+
memoryInfo.availableSystemMemory();
77+
fail("expected to fail");
78+
} catch (IllegalArgumentException e) {
79+
assertThat(e.getMessage(), is("Unable to parse number of bytes from [-Des.total_memory_bytes=invalid]"));
80+
}
81+
}
82+
83+
private static SystemMemoryInfo fallbackSystemMemoryInfo() {
84+
return () -> FALLBACK;
85+
}
86+
}

‎docs/changelog/78750.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 78750
2+
summary: Allow total memory to be overridden
3+
area: Packaging
4+
type: enhancement
5+
issues:
6+
- 65905

‎docs/reference/cluster/nodes-stats.asciidoc

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1036,6 +1036,18 @@ Total amount of physical memory.
10361036
(integer)
10371037
Total amount of physical memory in bytes.
10381038

1039+
`adjusted_total`::
1040+
(<<byte-units,byte value>>)
1041+
If the amount of physical memory has been overridden using the `es.total_memory_bytes`
1042+
system property then this reports the overridden value. Otherwise it reports the same
1043+
value as `total`.
1044+
1045+
`adjusted_total_in_bytes`::
1046+
(integer)
1047+
If the amount of physical memory has been overridden using the `es.total_memory_bytes`
1048+
system property then this reports the overridden value in bytes. Otherwise it reports
1049+
the same value as `total_in_bytes`.
1050+
10391051
`free`::
10401052
(<<byte-units,byte value>>)
10411053
Amount of free physical memory.

‎docs/reference/cluster/stats.asciidoc

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -916,6 +916,18 @@ Total amount of physical memory across all selected nodes.
916916
(integer)
917917
Total amount, in bytes, of physical memory across all selected nodes.
918918
919+
`adjusted_total`::
920+
(<<byte-units,byte value>>)
921+
Total amount of memory across all selected nodes, but using the value specified
922+
using the `es.total_memory_bytes` system property instead of measured total
923+
memory for those nodes where that system property was set.
924+
925+
`adjusted_total_in_bytes`::
926+
(integer)
927+
Total amount, in bytes, of memory across all selected nodes, but using the
928+
value specified using the `es.total_memory_bytes` system property instead
929+
of measured total memory for those nodes where that system property was set.
930+
919931
`free`::
920932
(<<byte-units, byte units>>)
921933
Amount of free physical memory across all selected nodes.
@@ -1399,6 +1411,8 @@ The API returns the following response:
13991411
"mem" : {
14001412
"total" : "16gb",
14011413
"total_in_bytes" : 17179869184,
1414+
"adjusted_total" : "16gb",
1415+
"adjusted_total_in_bytes" : 17179869184,
14021416
"free" : "78.1mb",
14031417
"free_in_bytes" : 81960960,
14041418
"used" : "15.9gb",

‎qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -511,6 +511,27 @@ public void test73CustomJvmOptionsDirectoryFilesWithoutOptionsExtensionIgnored()
511511
}
512512
}
513513

514+
public void test74CustomJvmOptionsTotalMemoryOverride() throws Exception {
515+
final Path heapOptions = installation.config(Paths.get("jvm.options.d", "total_memory.options"));
516+
try {
517+
setHeap(null); // delete default options
518+
// Work as though total system memory is 850MB
519+
append(heapOptions, "-Des.total_memory_bytes=891289600\n");
520+
521+
startElasticsearch();
522+
523+
final String nodesStatsResponse = makeRequest("https://localhost:9200/_nodes/stats");
524+
assertThat(nodesStatsResponse, containsString("\"adjusted_total_in_bytes\":891289600"));
525+
final String nodesResponse = makeRequest("https://localhost:9200/_nodes");
526+
// 40% of 850MB
527+
assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":356515840"));
528+
529+
stopElasticsearch();
530+
} finally {
531+
rm(heapOptions);
532+
}
533+
}
534+
514535
public void test80RelativePathConf() throws Exception {
515536
withCustomConfig(tempConf -> {
516537
append(tempConf.resolve("elasticsearch.yml"), "node.name: relative");

‎qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -883,22 +883,45 @@ public void test140CgroupOsStatsAreAvailable() throws Exception {
883883
* logic sets the correct heap size, based on the container limits.
884884
*/
885885
public void test150MachineDependentHeap() throws Exception {
886+
final List<String> xArgs = machineDependentHeapTest("942m", List.of());
887+
888+
// This is roughly 0.4 * 942
889+
assertThat(xArgs, hasItems("-Xms376m", "-Xmx376m"));
890+
}
891+
892+
/**
893+
* Check that when available system memory is constrained by a total memory override as well as Docker,
894+
* the machine-dependant heap sizing logic sets the correct heap size, preferring the override to the
895+
* container limits.
896+
*/
897+
public void test151MachineDependentHeapWithSizeOverride() throws Exception {
898+
final List<String> xArgs = machineDependentHeapTest(
899+
"942m",
900+
// 799014912 = 762m
901+
List.of("-Des.total_memory_bytes=799014912")
902+
);
903+
904+
// This is roughly 0.4 * 762, in particular it's NOT 0.4 * 942
905+
assertThat(xArgs, hasItems("-Xms304m", "-Xmx304m"));
906+
}
907+
908+
private List<String> machineDependentHeapTest(final String containerMemory, final List<String> extraJvmOptions) throws Exception {
886909
// Start by ensuring `jvm.options` doesn't define any heap options
887910
final Path jvmOptionsPath = tempDir.resolve("jvm.options");
888911
final Path containerJvmOptionsPath = installation.config("jvm.options");
889912
copyFromContainer(containerJvmOptionsPath, jvmOptionsPath);
890913

891-
final List<String> jvmOptions = Files.readAllLines(jvmOptionsPath)
892-
.stream()
893-
.filter(line -> (line.startsWith("-Xms") || line.startsWith("-Xmx")) == false)
894-
.collect(Collectors.toList());
914+
final List<String> jvmOptions = Stream.concat(
915+
Files.readAllLines(jvmOptionsPath).stream().filter(line -> (line.startsWith("-Xms") || line.startsWith("-Xmx")) == false),
916+
extraJvmOptions.stream()
917+
).collect(Collectors.toList());
895918

896919
Files.writeString(jvmOptionsPath, String.join("\n", jvmOptions));
897920

898921
// Now run the container, being explicit about the available memory
899922
runContainer(
900923
distribution(),
901-
builder().memory("942m").volume(jvmOptionsPath, containerJvmOptionsPath).envVar("ELASTIC_PASSWORD", PASSWORD)
924+
builder().memory(containerMemory).volume(jvmOptionsPath, containerJvmOptionsPath).envVar("ELASTIC_PASSWORD", PASSWORD)
902925
);
903926

904927
waitForElasticsearch(installation, "elastic", PASSWORD);
@@ -913,12 +936,9 @@ public void test150MachineDependentHeap() throws Exception {
913936
final JsonNode jsonNode = new ObjectMapper().readTree(jvmArgumentsLine.get());
914937

915938
final String argsStr = jsonNode.get("message").textValue();
916-
final List<String> xArgs = Arrays.stream(argsStr.substring(1, argsStr.length() - 1).split(",\\s*"))
939+
return Arrays.stream(argsStr.substring(1, argsStr.length() - 1).split(",\\s*"))
917940
.filter(arg -> arg.startsWith("-X"))
918941
.collect(Collectors.toList());
919-
920-
// This is roughly 0.4 * 942
921-
assertThat(xArgs, hasItems("-Xms376m", "-Xmx376m"));
922942
}
923943

924944
/**

‎qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,34 @@ public void test70RestartServer() throws Exception {
237237
}
238238
}
239239

240+
public void test71JvmOptionsTotalMemoryOverride() throws Exception {
241+
try {
242+
install();
243+
assertPathsExist(installation.envFile);
244+
setHeap(null);
245+
246+
// Recreate file realm users that have been deleted in earlier tests
247+
setFileSuperuser("test_superuser", "test_superuser_password");
248+
249+
withCustomConfig(tempConf -> {
250+
// Work as though total system memory is 850MB
251+
append(installation.envFile, "ES_JAVA_OPTS=\"-Des.total_memory_bytes=891289600\"");
252+
253+
startElasticsearch();
254+
255+
final String nodesStatsResponse = makeRequest("https://localhost:9200/_nodes/stats");
256+
assertThat(nodesStatsResponse, containsString("\"adjusted_total_in_bytes\":891289600"));
257+
258+
// 40% of 850MB
259+
assertThat(sh.run("ps auwwx").stdout, containsString("-Xms340m -Xmx340m"));
260+
261+
stopElasticsearch();
262+
});
263+
} finally {
264+
cleanup();
265+
}
266+
}
267+
240268
public void test72TestRuntimeDirectory() throws Exception {
241269
try {
242270
install();

‎server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -266,20 +266,26 @@ private OsStats(List<NodeInfo> nodeInfos, List<NodeStats> nodeStatsList) {
266266
this.allocatedProcessors = allocatedProcessors;
267267

268268
long totalMemory = 0;
269+
long adjustedTotalMemory = 0;
269270
long freeMemory = 0;
270271
for (NodeStats nodeStats : nodeStatsList) {
271272
if (nodeStats.getOs() != null) {
272-
long total = nodeStats.getOs().getMem().getTotal().getBytes();
273+
org.elasticsearch.monitor.os.OsStats.Mem mem = nodeStats.getOs().getMem();
274+
long total = mem.getTotal().getBytes();
273275
if (total > 0) {
274276
totalMemory += total;
275277
}
276-
long free = nodeStats.getOs().getMem().getFree().getBytes();
278+
long adjustedTotal = mem.getAdjustedTotal().getBytes();
279+
if (adjustedTotal > 0) {
280+
adjustedTotalMemory += adjustedTotal;
281+
}
282+
long free = mem.getFree().getBytes();
277283
if (free > 0) {
278284
freeMemory += free;
279285
}
280286
}
281287
}
282-
this.mem = new org.elasticsearch.monitor.os.OsStats.Mem(totalMemory, freeMemory);
288+
this.mem = new org.elasticsearch.monitor.os.OsStats.Mem(totalMemory, adjustedTotalMemory, freeMemory);
283289
}
284290

285291
public int getAvailableProcessors() {

0 commit comments

Comments
 (0)