Skip to content

Commit 13dddee

Browse files
MaxGekkmaropu
authored andcommitted
[MINOR][SQL][TESTS] Disable UI in SQL benchmarks by default
### What changes were proposed in this pull request? Set `spark.ui.enabled` to `false` in `SqlBasedBenchmark.getSparkSession`. This disables UI in all SQL benchmarks by default. ### Why are the changes needed? UI overhead lowers numbers in the `Relative` column and impacts on `Stdev` in benchmark results. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Checked by running `DateTimeRebaseBenchmark`. Closes #28432 from MaxGekk/ui-off-in-benchmarks. Authored-by: Max Gekk <[email protected]> Signed-off-by: Takeshi Yamamuro <[email protected]>
1 parent 75da050 commit 13dddee

File tree

3 files changed

+2
-4
lines changed

3 files changed

+2
-4
lines changed

sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DataSourceReadBenchmark.scala

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import scala.util.Random
2323

2424
import org.apache.spark.SparkConf
2525
import org.apache.spark.benchmark.Benchmark
26-
import org.apache.spark.internal.config.UI._
2726
import org.apache.spark.sql.{DataFrame, DataFrameWriter, Row, SparkSession}
2827
import org.apache.spark.sql.catalyst.InternalRow
2928
import org.apache.spark.sql.execution.datasources.parquet.{SpecificParquetRecordReaderBase, VectorizedParquetRecordReader}
@@ -52,7 +51,6 @@ object DataSourceReadBenchmark extends SqlBasedBenchmark {
5251
.set("spark.master", "local[1]")
5352
.setIfMissing("spark.driver.memory", "3g")
5453
.setIfMissing("spark.executor.memory", "3g")
55-
.setIfMissing(UI_ENABLED, false)
5654

5755
val sparkSession = SparkSession.builder.config(conf).getOrCreate()
5856

sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/FilterPushdownBenchmark.scala

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import scala.util.Random
2323

2424
import org.apache.spark.SparkConf
2525
import org.apache.spark.benchmark.Benchmark
26-
import org.apache.spark.internal.config.UI._
2726
import org.apache.spark.sql.{DataFrame, SparkSession}
2827
import org.apache.spark.sql.functions.monotonically_increasing_id
2928
import org.apache.spark.sql.internal.SQLConf
@@ -49,7 +48,6 @@ object FilterPushdownBenchmark extends SqlBasedBenchmark {
4948
.set("spark.master", "local[1]")
5049
.setIfMissing("spark.driver.memory", "3g")
5150
.setIfMissing("spark.executor.memory", "3g")
52-
.setIfMissing(UI_ENABLED, false)
5351
.setIfMissing("orc.compression", "snappy")
5452
.setIfMissing("spark.sql.parquet.compression.codec", "snappy")
5553

sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/SqlBasedBenchmark.scala

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.spark.sql.execution.benchmark
1919

2020
import org.apache.spark.benchmark.{Benchmark, BenchmarkBase}
21+
import org.apache.spark.internal.config.UI.UI_ENABLED
2122
import org.apache.spark.sql.{Dataset, SparkSession}
2223
import org.apache.spark.sql.SaveMode.Overwrite
2324
import org.apache.spark.sql.catalyst.plans.SQLHelper
@@ -37,6 +38,7 @@ trait SqlBasedBenchmark extends BenchmarkBase with SQLHelper {
3738
.appName(this.getClass.getCanonicalName)
3839
.config(SQLConf.SHUFFLE_PARTITIONS.key, 1)
3940
.config(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key, 1)
41+
.config(UI_ENABLED.key, false)
4042
.getOrCreate()
4143
}
4244

0 commit comments

Comments
 (0)