Skip to content

Commit 9f84037

Browse files
chernserchernser
authored
Merge pull request #2445 from ClickHouse/restore_performance
restore /performance
2 parents d1c50b8 + 4206c2c commit 9f84037

25 files changed

+3024
-0
lines changed

‎performance/README.md‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
2+
## JMH Benchmarks
3+
4+
5+
### Dependencies
6+
7+
8+
9+
### How to Run
10+
11+
12+
#### Generating Dataset
13+
14+
```shell
15+
mvn compile exec:exec -Dexec.executable=java -Dexec.args="-classpath %classpath com.clickhouse.benchmark.data.DataSetGenerator \
16+
-input sample_dataset.sql -name default -rows 10"
17+
```
18+
19+
#### Running Benchmarks
20+
21+
With default settings :
22+
```shell
23+
mvn compile exec:exec
24+
```
25+
26+
With custom measurement iterations:
27+
```shell
28+
mvn compile exec:exec -Dexec.executable=java -Dexec.args="-classpath %classpath com.clickhouse.benchmark.BenchmarkRunner -m 3"
29+
```
30+
31+
Other options:
32+
- "-d" - dataset name or file path (like `file://default.csv`)
33+
- "-l" - dataset limits to test coma separated (ex.: `-l 10000,10000`)
34+
- "-m" - number of measurement iterations
35+
- "-t" - time in seconds per iteration
36+
- "-b" - benchmark mask coma separated. Ex.: `-b writer,reader,i`. Default : `-b i,q`
37+
- "all" - Run alpl benchmarks
38+
- "i" - InsertClient - insert operation benchmarks
39+
- "q" - QueryClient - query operation benchmarks
40+
- "ci" - ConcurrentInsertClient - concurrent version of insert benchmarks
41+
- "cq" - ConcurrentQueryClient - concurrent version of query benchmarks
42+
- "lz" - Compression - compression related benchmarks
43+
- "writer" - Serializer - serialization only logic benchmarks
44+
- "reader" - DeSerilalizer - deserialization only logic benchmarks
45+
- "mixed" - MixedWorkload
46+
- "jq" - JDBCQuery - query operations using JDBC
47+
- "ji" - JDBCInsert - insert operation using JDBC

‎performance/pom.xml‎

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
<project xmlns="http://maven.apache.org/POM/4.0.0"
2+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
3+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
4+
<modelVersion>4.0.0</modelVersion>
5+
6+
<artifactId>jmh-benchmark</artifactId>
7+
<groupId>clickhouse.com</groupId>
8+
<version>0.1-SNAPSHOT</version>
9+
<packaging>jar</packaging>
10+
11+
<name>JMH Benchmark for Java Client</name>
12+
<description>JMH Benchmark for Java Client</description>
13+
<url>https://github.com/ClickHouse/clickhouse-java/tree/main/performance</url>
14+
15+
<properties>
16+
<apache.httpclient.version>5.3.1</apache.httpclient.version>
17+
<slf4j.version>2.0.17</slf4j.version>
18+
<ch.jdbc.revision>0.8.6-SNAPSHOT</ch.jdbc.revision>
19+
<jmh.version>1.37</jmh.version>
20+
<testcontainers.version>1.20.6</testcontainers.version>
21+
22+
<antrun-plugin.version>3.1.0</antrun-plugin.version>
23+
<assembly-plugin.version>3.6.0</assembly-plugin.version>
24+
<clean-plugin.version>3.3.1</clean-plugin.version>
25+
<compiler-plugin.version>3.13.0</compiler-plugin.version>
26+
<deploy-plugin.version>3.1.1</deploy-plugin.version>
27+
<enforcer-plugin.version>3.3.0</enforcer-plugin.version>
28+
<exec-plugin.version>3.5.0</exec-plugin.version>
29+
<failsafe-plugin.version>3.1.2</failsafe-plugin.version>
30+
<flatten-plugin.version>1.2.7</flatten-plugin.version>
31+
<git-plugin.version>6.0.0</git-plugin.version>
32+
<gpg-plugin.version>3.1.0</gpg-plugin.version>
33+
<helper-plugin.version>3.4.0</helper-plugin.version>
34+
<jacoco-plugin.version>0.8.12</jacoco-plugin.version>
35+
<jar-plugin.version>3.3.0</jar-plugin.version>
36+
<javadoc-plugin.version>3.5.0</javadoc-plugin.version>
37+
<native-plugin.version>0.9.23</native-plugin.version>
38+
<os-plugin.version>1.7.1</os-plugin.version>
39+
<protobuf-plugin.version>0.6.1</protobuf-plugin.version>
40+
<shade-plugin.version>3.5.0</shade-plugin.version>
41+
<source-plugin.version>3.2.1</source-plugin.version>
42+
<staging-plugin.version>1.6.13</staging-plugin.version>
43+
<surefire-plugin.version>3.2.5</surefire-plugin.version>
44+
<toolchains-plugin.version>3.1.0</toolchains-plugin.version>
45+
<versions-plugin.version>2.16.0</versions-plugin.version>
46+
<resource-plugin.version>3.3.1</resource-plugin.version>
47+
48+
<jmh.measure-iter>10</jmh.measure-iter>
49+
<jmh.measure-time>10</jmh.measure-time>
50+
51+
</properties>
52+
53+
<dependencies>
54+
<dependency>
55+
<groupId>org.slf4j</groupId>
56+
<artifactId>slf4j-api</artifactId>
57+
<version>${slf4j.version}</version>
58+
</dependency>
59+
<dependency>
60+
<groupId>org.slf4j</groupId>
61+
<artifactId>slf4j-simple</artifactId>
62+
<version>${slf4j.version}</version>
63+
</dependency>
64+
<dependency>
65+
<groupId>org.openjdk.jmh</groupId>
66+
<artifactId>jmh-core</artifactId>
67+
<version>${jmh.version}</version>
68+
</dependency>
69+
<dependency>
70+
<groupId>org.openjdk.jmh</groupId>
71+
<artifactId>jmh-generator-annprocess</artifactId>
72+
<version>${jmh.version}</version>
73+
</dependency>
74+
75+
<!-- Performance Test Dependencies -->
76+
<!-- <dependency>-->
77+
<!-- <groupId>org.testcontainers</groupId>-->
78+
<!-- <artifactId>testcontainers</artifactId>-->
79+
<!-- <version>${testcontainers.version}</version>-->
80+
<!-- <scope>compile</scope>-->
81+
<!-- </dependency>-->
82+
<!-- https://mvnrepository.com/artifact/org.testcontainers/clickhouse -->
83+
<dependency>
84+
<groupId>org.testcontainers</groupId>
85+
<artifactId>clickhouse</artifactId>
86+
<version>${testcontainers.version}</version>
87+
</dependency>
88+
89+
<!-- Contains both JDBC drivers and clients -->
90+
<dependency>
91+
<groupId>com.clickhouse</groupId>
92+
<artifactId>clickhouse-jdbc</artifactId>
93+
<version>${ch.jdbc.revision}</version>
94+
<classifier>all</classifier>
95+
</dependency>
96+
97+
</dependencies>
98+
99+
<build>
100+
<plugins>
101+
<plugin>
102+
<groupId>org.apache.maven.plugins</groupId>
103+
<artifactId>maven-compiler-plugin</artifactId>
104+
<version>${compiler-plugin.version}</version>
105+
<configuration>
106+
<annotationProcessorPaths>
107+
<path>
108+
<groupId>org.projectlombok</groupId>
109+
<artifactId>lombok</artifactId>
110+
<version>1.18.32</version>
111+
</path>
112+
<path>
113+
<groupId>org.openjdk.jmh</groupId>
114+
<artifactId>jmh-generator-annprocess</artifactId>
115+
<version>${jmh.version}</version>
116+
</path>
117+
</annotationProcessorPaths>
118+
<release>11</release>
119+
</configuration>
120+
</plugin>
121+
<plugin>
122+
<groupId>org.codehaus.mojo</groupId>
123+
<artifactId>build-helper-maven-plugin</artifactId>
124+
<version>3.6.0</version>
125+
<executions>
126+
<execution>
127+
<id>add-test-source</id>
128+
<phase>generate-test-sources</phase>
129+
<goals>
130+
<goal>add-test-source</goal>
131+
</goals>
132+
<configuration>
133+
<sources>
134+
<source>src/test</source>
135+
</sources>
136+
</configuration>
137+
</execution>
138+
</executions>
139+
</plugin>
140+
<plugin>
141+
<groupId>org.apache.maven.plugins</groupId>
142+
<artifactId>maven-assembly-plugin</artifactId>
143+
<version>3.6.0</version>
144+
<configuration>
145+
<descriptorRefs>
146+
<descriptorRef>jar-with-dependencies</descriptorRef>
147+
</descriptorRefs>
148+
<archive>
149+
<manifest>
150+
<mainClass>com.clickhouse.com.clickhouse.benchmark.BenchmarkRunner</mainClass>
151+
</manifest>
152+
</archive>
153+
</configuration>
154+
<executions>
155+
<execution>
156+
<id>make-assembly</id>
157+
<phase>package</phase>
158+
<goals>
159+
<goal>single</goal>
160+
</goals>
161+
</execution>
162+
</executions>
163+
</plugin>
164+
<plugin>
165+
<groupId>org.codehaus.mojo</groupId>
166+
<artifactId>exec-maven-plugin</artifactId>
167+
<version>${exec-plugin.version}</version>
168+
<executions>
169+
<execution>
170+
<id>run-benchmarks</id>
171+
<goals>
172+
<goal>exec</goal>
173+
</goals>
174+
<configuration>
175+
<executable>java</executable>
176+
<arguments>
177+
<argument>-classpath</argument>
178+
<classpath/>
179+
<argument>com.clickhouse.benchmark.BenchmarkRunner</argument>
180+
<argument>-m ${jmh.measure-iter}</argument>
181+
<argument>-t ${jmh.measure-time}</argument>
182+
</arguments>
183+
</configuration>
184+
</execution>
185+
</executions>
186+
</plugin>
187+
</plugins>
188+
</build>
189+
</project>

‎performance/src/main/java/com/clickhouse/benchmark/BenchmarkRunner.java‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
package com.clickhouse.benchmark;
2+
3+
import com.clickhouse.benchmark.clients.*;
4+
import org.openjdk.jmh.annotations.Mode;
5+
import org.openjdk.jmh.profile.GCProfiler;
6+
import org.openjdk.jmh.profile.MemPoolProfiler;
7+
import org.openjdk.jmh.results.format.ResultFormatType;
8+
import org.openjdk.jmh.runner.Runner;
9+
import org.openjdk.jmh.runner.options.ChainedOptionsBuilder;
10+
import org.openjdk.jmh.runner.options.OptionsBuilder;
11+
import org.openjdk.jmh.runner.options.TimeValue;
12+
import org.slf4j.Logger;
13+
import org.slf4j.LoggerFactory;
14+
15+
import java.util.*;
16+
import java.util.concurrent.TimeUnit;
17+
18+
import static com.clickhouse.benchmark.TestEnvironment.isCloud;
19+
20+
21+
public class BenchmarkRunner {
22+
private static final Logger LOGGER = LoggerFactory.getLogger(BenchmarkRunner.class);
23+
24+
public static void main(String[] args) throws Exception {
25+
LOGGER.info("Starting Benchmarks");
26+
Map<String, String> options = parseArgs(args);
27+
System.out.println("Start Benchmarks with options: " + options);
28+
final String env = isCloud() ? "cloud" : "local";
29+
final long time = System.currentTimeMillis();
30+
31+
final int measurementIterations = Integer.parseInt(options.getOrDefault("-m", "10"));
32+
final int measurementTime = Integer.parseInt(options.getOrDefault("-t", "" + (isCloud() ? 30 : 10)));
33+
final String resultFile = String.format("jmh-results-%s-%s.json", env, time);
34+
final String outputFile = String.format("jmh-results-%s-%s.out", env, time);
35+
final String datasetName = options.getOrDefault("-d", "file://default.csv");
36+
final String[] limits = options.getOrDefault("-l", "300000,100000,10000").split(",");
37+
38+
System.out.println("Measurement iterations: " + measurementIterations);
39+
System.out.println("Measurement time: " + measurementTime + "s");
40+
System.out.println("Env: " + env);
41+
System.out.println("Dataset: " + datasetName);
42+
System.out.println("Limits: " + Arrays.asList(limits));
43+
System.out.println("Epoch Time: " + time);
44+
45+
ChainedOptionsBuilder optBuilder = new OptionsBuilder()
46+
.forks(1) // must be a fork. No fork only for debugging
47+
.mode(Mode.SampleTime)
48+
.param("datasetSourceName", datasetName)
49+
.param("limit", limits)
50+
.timeUnit(TimeUnit.MILLISECONDS)
51+
.addProfiler(GCProfiler.class)
52+
.addProfiler(MemPoolProfiler.class)
53+
.warmupIterations(1)
54+
.warmupTime(TimeValue.seconds(5))
55+
.measurementIterations(measurementIterations)
56+
.jvmArgs("-Xms8g", "-Xmx8g")
57+
.measurementTime(TimeValue.seconds(measurementTime))
58+
.resultFormat(ResultFormatType.JSON)
59+
.output(outputFile)
60+
.result(resultFile)
61+
.shouldFailOnError(true);
62+
63+
String testMask = options.getOrDefault("-b", "q,i");
64+
String[] testMaskParts = testMask.split(",");
65+
66+
SortedSet<String> benchmarks = new TreeSet<>();
67+
if (testMaskParts[0].equalsIgnoreCase("all")) {
68+
BENCHMARK_FLAGS.values().forEach((b) -> {
69+
optBuilder.include(b);
70+
benchmarks.add(b);
71+
});
72+
} else {
73+
for (String p : testMaskParts) {
74+
String benchmark = BENCHMARK_FLAGS.get(p);
75+
if (benchmark != null) {
76+
optBuilder.include(benchmark);
77+
benchmarks.add(benchmark);
78+
}
79+
}
80+
}
81+
82+
System.out.println("Running benchmarks: " + benchmarks);
83+
new Runner(optBuilder.build()).run();
84+
}
85+
86+
private static final Map<String, String> BENCHMARK_FLAGS = buildBenchmarkFlags();
87+
88+
private static Map<String, String> buildBenchmarkFlags() {
89+
HashMap<String, String> map = new HashMap<>();
90+
map.put("q", QueryClient.class.getName());
91+
map.put("i", InsertClient.class.getName());
92+
map.put("cq", ConcurrentQueryClient.class.getName());
93+
map.put("ci", ConcurrentInsertClient.class.getName());
94+
map.put("lz", Compression.class.getName());
95+
map.put("reader", Deserializers.class.getName());
96+
map.put("writer", Serializers.class.getName());
97+
map.put("mixed", MixedWorkload.class.getName());
98+
map.put("jq", JDBCQuery.class.getName());
99+
map.put("ji", JDBCInsert.class.getName());
100+
return map;
101+
}
102+
103+
private static Map<String, String> parseArgs(String[] args) {
104+
Map<String, String> options = new HashMap<>();
105+
for (int i = 0; i < args.length; i+=2) {
106+
options.put(args[i], args[i+1]);
107+
}
108+
return options;
109+
}
110+
}

0 commit comments

Comments
 (0)