Introduce EngineTestKit support for test discovery

documentation/src/docs/asciidoc/release-notes/release-notes-5.13.0-M1.adoc

@@ -31,6 +31,9 @@ repository on GitHub.
   redirecting `stdout` and `stderr` output streams to files.
 * Add `TestDescriptor.Visitor.composite(List)` factory method for creating a composite
   visitor that delegates to the given visitors in order.
+* Introduce test _discovery_ support in `EngineTestKit` to ease testing for discovery
+  issues produced by a `TestEngine`. Please refer to the
+  <<../user-guide/index.adoc#testkit-engine, User Guide>> for details.
 
 
 [[release-notes-5.13.0-M1-junit-jupiter]]

documentation/src/docs/asciidoc/user-guide/advanced-topics/testkit.adoc

@@ -1,3 +1,5 @@
+:testDir: ../../../../../src/test/java
+
 [[testkit]]
 === JUnit Platform Test Kit
 
@@ -9,16 +11,17 @@ JUnit Platform and then verifying the expected results. As of JUnit Platform
 [[testkit-engine]]
 ==== Engine Test Kit
 
-The `{testkit-engine-package}` package provides support for executing a `{TestPlan}` for a
-given `{TestEngine}` running on the JUnit Platform and then accessing the results via a
-fluent API to verify the expected results. The key entry point into this API is the
-`{EngineTestKit}` which provides static factory methods named `engine()` and `execute()`.
-It is recommended that you select one of the `engine()` variants to benefit from the
-fluent API for building a `LauncherDiscoveryRequest`.
+The `{testkit-engine-package}` package provides support for discovering and executing a
+`{TestPlan}` for a given `{TestEngine}` running on the JUnit Platform and then accessing
+the results via convenient result objects. For execution, a fluent API may be used to
+verify the expected execution events were received. The key entry point into this API is
+the `{EngineTestKit}` which provides static factory methods named `engine()`,
+`discover()`, and `execute()`. It is recommended that you select one of the `engine()`
+variants to benefit from the fluent API for building a `LauncherDiscoveryRequest`.
 
 NOTE: If you prefer to use the `LauncherDiscoveryRequestBuilder` from the `Launcher` API
-to build your `LauncherDiscoveryRequest`, you must use one of the `execute()` variants in
-`EngineTestKit`.
+to build your `LauncherDiscoveryRequest`, you must use one of the `discover()` or
+`execute()` variants in `EngineTestKit`.
 
 The following test class written using JUnit Jupiter will be used in subsequent examples.
 
@@ -34,8 +37,24 @@ own `TestEngine` implementation, you need to use its unique engine ID. Alternati
 may test your own `TestEngine` by supplying an instance of it to the
 `EngineTestKit.engine(TestEngine)` static factory method.
 
+[[testkit-engine-discovery]]
+==== Verifying Test Discovery
+
+The following test demonstrates how to verify that a `TestPlan` was discovered as expected
+by the JUnit Jupiter `TestEngine`.
+
+[source,java,indent=0]
+----
+include::{testDir}/example/testkit/EngineTestKitDiscoveryDemo.java[tags=user_guide]
+----
+<1> Select the JUnit Jupiter `TestEngine`.
+<2> Select the <<testkit-engine-ExampleTestCase, `ExampleTestCase`>> test class.
+<3> Discover the `TestPlan`.
+<4> Assert engine root descriptor has expected display name.
+<5> Assert no discovery issues were encountered.
+
 [[testkit-engine-statistics]]
-==== Asserting Statistics
+==== Asserting Execution Statistics
 
 One of the most common features of the Test Kit is the ability to assert statistics
 against events fired during the execution of a `TestPlan`. The following tests demonstrate

documentation/src/test/java/example/testkit/EngineTestKitDiscoveryDemo.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2015-2025 the original author or authors.
+ *
+ * All rights reserved. This program and the accompanying materials are
+ * made available under the terms of the Eclipse Public License v2.0 which
+ * accompanies this distribution and is available at
+ *
+ * https://www.eclipse.org/legal/epl-v20.html
+ */
+
+package example.testkit;
+
+// tag::user_guide[]
+import static java.util.Collections.emptyList;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
+
+import example.ExampleTestCase;
+
+import org.junit.jupiter.api.Test;
+import org.junit.platform.testkit.engine.EngineDiscoveryResults;
+import org.junit.platform.testkit.engine.EngineTestKit;
+
+class EngineTestKitDiscoveryDemo {
+
+	@Test
+	void verifyJupiterDiscovery() {
+		EngineDiscoveryResults results = EngineTestKit.engine("junit-jupiter") // <1>
+				.selectors(selectClass(ExampleTestCase.class)) // <2>
+				.discover(); // <3>
+
+		assertEquals("JUnit Jupiter", results.getEngineDescriptor().getDisplayName()); // <4>
+		assertEquals(emptyList(), results.getDiscoveryIssues()); // <5>
+	}
+
+}
+// end::user_guide[]

junit-platform-launcher/src/main/java/org/junit/platform/launcher/core/DiscoveryIssueNotifier.java

@@ -16,6 +16,7 @@
 
 import java.io.PrintWriter;
 import java.io.StringWriter;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Consumer;
@@ -35,9 +36,10 @@
  */
 class DiscoveryIssueNotifier {
 
-	static final DiscoveryIssueNotifier NO_ISSUES = new DiscoveryIssueNotifier(emptyList(), emptyList());
+	static final DiscoveryIssueNotifier NO_ISSUES = new DiscoveryIssueNotifier(emptyList(), emptyList(), emptyList());
 	private static final Logger logger = LoggerFactory.getLogger(DiscoveryIssueNotifier.class);
 
+	private final List<DiscoveryIssue> allIssues;
 	private final List<DiscoveryIssue> criticalIssues;
 	private final List<DiscoveryIssue> nonCriticalIssues;
 
@@ -47,14 +49,20 @@ static DiscoveryIssueNotifier from(Severity criticalSeverity, List<DiscoveryIssu
 				.collect(partitioningBy(issue -> issue.severity().compareTo(criticalSeverity) >= 0));
 		List<DiscoveryIssue> criticalIssues = issuesByCriticality.get(true);
 		List<DiscoveryIssue> nonCriticalIssues = issuesByCriticality.get(false);
-		return new DiscoveryIssueNotifier(criticalIssues, nonCriticalIssues);
+		return new DiscoveryIssueNotifier(new ArrayList<>(issues), criticalIssues, nonCriticalIssues);
 	}
 
-	private DiscoveryIssueNotifier(List<DiscoveryIssue> criticalIssues, List<DiscoveryIssue> nonCriticalIssues) {
+	private DiscoveryIssueNotifier(List<DiscoveryIssue> allIssues, List<DiscoveryIssue> criticalIssues,
+			List<DiscoveryIssue> nonCriticalIssues) {
+		this.allIssues = allIssues;
 		this.criticalIssues = criticalIssues;
 		this.nonCriticalIssues = nonCriticalIssues;
 	}
 
+	List<DiscoveryIssue> getAllIssues() {
+		return allIssues;
+	}
+
 	boolean hasCriticalIssues() {
 		return !criticalIssues.isEmpty();
 	}

junit-platform-launcher/src/main/java/org/junit/platform/launcher/core/LauncherDiscoveryResult.java

@@ -16,13 +16,15 @@
 
 import java.util.Collection;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 import org.apiguardian.api.API;
 import org.junit.platform.engine.ConfigurationParameters;
+import org.junit.platform.engine.DiscoveryIssue;
 import org.junit.platform.engine.TestDescriptor;
 import org.junit.platform.engine.TestEngine;
 import org.junit.platform.engine.reporting.OutputDirectoryProvider;
@@ -51,6 +53,11 @@ public TestDescriptor getEngineTestDescriptor(TestEngine testEngine) {
 		return getEngineResult(testEngine).getRootDescriptor();
 	}
 
+	@API(status = INTERNAL, since = "1.13")
+	public List<DiscoveryIssue> getDiscoveryIssues(TestEngine testEngine) {
+		return getEngineResult(testEngine).getDiscoveryIssueNotifier().getAllIssues();
+	}
+
 	EngineResultInfo getEngineResult(TestEngine testEngine) {
 		return this.testEngineResults.get(testEngine);
 	}

junit-platform-testkit/src/main/java/org/junit/platform/testkit/engine/EngineDiscoveryResults.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2015-2025 the original author or authors.
+ *
+ * All rights reserved. This program and the accompanying materials are
+ * made available under the terms of the Eclipse Public License v2.0 which
+ * accompanies this distribution and is available at
+ *
+ * https://www.eclipse.org/legal/epl-v20.html
+ */
+
+package org.junit.platform.testkit.engine;
+
+import static java.util.Collections.unmodifiableList;
+import static org.apiguardian.api.API.Status.EXPERIMENTAL;
+
+import java.util.List;
+
+import org.apiguardian.api.API;
+import org.junit.platform.commons.util.Preconditions;
+import org.junit.platform.engine.DiscoveryIssue;
+import org.junit.platform.engine.TestDescriptor;
+
+/**
+ * {@code EngineDiscoveryResults} represents the results of test discovery
+ * by a {@link org.junit.platform.engine.TestEngine TestEngine} on the JUnit
+ * Platform and provides access to the {@link TestDescriptor} of the engine
+ * and any {@link DiscoveryIssue DiscoveryIssues} that were encountered.
+ *
+ * @since 1.13
+ */
+@API(status = EXPERIMENTAL, since = "1.13")
+public class EngineDiscoveryResults {
+
+	private final TestDescriptor engineDescriptor;
+	private final List<DiscoveryIssue> discoveryIssues;
+
+	EngineDiscoveryResults(TestDescriptor engineDescriptor, List<DiscoveryIssue> discoveryIssues) {
+		this.engineDescriptor = Preconditions.notNull(engineDescriptor, "Engine descriptor must not be null");
+		this.discoveryIssues = unmodifiableList(
+			Preconditions.notNull(discoveryIssues, "Discovery issues list must not be null"));
+		Preconditions.containsNoNullElements(discoveryIssues, "Discovery issues list must not contain null elements");
+	}
+
+	/**
+	 * {@return the root {@link TestDescriptor} of the engine}
+	 */
+	public TestDescriptor getEngineDescriptor() {
+		return engineDescriptor;
+	}
+
+	/**
+	 * {@return the issues that were encountered during discovery}
+	 */
+	public List<DiscoveryIssue> getDiscoveryIssues() {
+		return discoveryIssues;
+	}
+
+}

junit-platform-testkit/src/main/java/org/junit/platform/testkit/engine/EngineTestKit.java

@@ -16,9 +16,11 @@
 import static org.apiguardian.api.API.Status.EXPERIMENTAL;
 import static org.apiguardian.api.API.Status.MAINTAINED;
 import static org.apiguardian.api.API.Status.STABLE;
+import static org.junit.platform.launcher.core.EngineDiscoveryOrchestrator.Phase.DISCOVERY;
 import static org.junit.platform.launcher.core.EngineDiscoveryOrchestrator.Phase.EXECUTION;
 
 import java.nio.file.Path;
+import java.util.List;
 import java.util.Map;
 import java.util.ServiceLoader;
 import java.util.stream.Stream;
@@ -29,6 +31,7 @@
 import org.junit.platform.commons.util.CollectionUtils;
 import org.junit.platform.commons.util.Preconditions;
 import org.junit.platform.engine.DiscoveryFilter;
+import org.junit.platform.engine.DiscoveryIssue;
 import org.junit.platform.engine.DiscoverySelector;
 import org.junit.platform.engine.EngineDiscoveryRequest;
 import org.junit.platform.engine.EngineExecutionListener;
@@ -46,15 +49,24 @@
 import org.junit.platform.launcher.core.ServiceLoaderTestEngineRegistry;
 
 /**
- * {@code EngineTestKit} provides support for executing a test plan for a given
- * {@link TestEngine} and then accessing the results via
- * {@linkplain EngineExecutionResults a fluent API} to verify the expected results.
+ * {@code EngineTestKit} provides support for discovering and executing tests
+ * for a given {@link TestEngine} and provides convenient access to the results.
+ *
+ * <p>For <em>discovery</em>, {@link EngineDiscoveryResults} provides access to
+ * the {@link TestDescriptor} of the engine and any {@link DiscoveryIssue
+ * DiscoveryIssues} that were encountered.
+ *
+ * <p>For <em>execution</em>, {@link EngineExecutionResults} provides a fluent
+ * API to verify the expected results.
  *
  * @since 1.4
  * @see #engine(String)
  * @see #engine(TestEngine)
+ * @see #discover(String, LauncherDiscoveryRequest)
+ * @see #discover(TestEngine, LauncherDiscoveryRequest)
  * @see #execute(String, LauncherDiscoveryRequest)
  * @see #execute(TestEngine, LauncherDiscoveryRequest)
+ * @see EngineDiscoveryResults
  * @see EngineExecutionResults
  */
 @API(status = MAINTAINED, since = "1.7")
@@ -121,6 +133,65 @@ public static Builder engine(TestEngine testEngine) {
 		return new Builder(testEngine);
 	}
 
+	/**
+	 * Discover tests for the given {@link LauncherDiscoveryRequest} using the
+	 * {@link TestEngine} with the supplied ID.
+	 *
+	 * <p>The {@code TestEngine} will be loaded via Java's {@link ServiceLoader}
+	 * mechanism, analogous to the manner in which test engines are loaded in
+	 * the JUnit Platform Launcher API.
+	 *
+	 * <p>{@link org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder}
+	 * provides a convenient way to build an appropriate discovery request to
+	 * supply to this method. As an alternative, consider using
+	 * {@link #engine(TestEngine)} for a more fluent API.
+	 *
+	 * @param engineId the ID of the {@code TestEngine} to use; must not be
+	 * {@code null} or <em>blank</em>
+	 * @param discoveryRequest the {@code LauncherDiscoveryRequest} to use
+	 * @return the results of the discovery
+	 * @throws PreconditionViolationException for invalid arguments or if the
+	 * {@code TestEngine} with the supplied ID cannot be loaded
+	 * @since 1.13
+	 * @see #discover(TestEngine, LauncherDiscoveryRequest)
+	 * @see #engine(String)
+	 * @see #engine(TestEngine)
+	 */
+	@API(status = EXPERIMENTAL, since = "1.13")
+	public static EngineDiscoveryResults discover(String engineId, LauncherDiscoveryRequest discoveryRequest) {
+		Preconditions.notBlank(engineId, "TestEngine ID must not be null or blank");
+		return discover(loadTestEngine(engineId.trim()), discoveryRequest);
+	}
+
+	/**
+	 * Discover tests for the given {@link LauncherDiscoveryRequest} using the
+	 * supplied {@link TestEngine}.
+	 *
+	 * <p>{@link org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder}
+	 * provides a convenient way to build an appropriate discovery request to
+	 * supply to this method. As an alternative, consider using
+	 * {@link #engine(TestEngine)} for a more fluent API.
+	 *
+	 * @param testEngine the {@code TestEngine} to use; must not be {@code null}
+	 * @param discoveryRequest the {@code EngineDiscoveryResults} to use; must
+	 * not be {@code null}
+	 * @return the recorded {@code EngineExecutionResults}
+	 * @throws PreconditionViolationException for invalid arguments
+	 * @since 1.13
+	 * @see #discover(String, LauncherDiscoveryRequest)
+	 * @see #engine(String)
+	 * @see #engine(TestEngine)
+	 */
+	@API(status = EXPERIMENTAL, since = "1.13")
+	public static EngineDiscoveryResults discover(TestEngine testEngine, LauncherDiscoveryRequest discoveryRequest) {
+		Preconditions.notNull(testEngine, "TestEngine must not be null");
+		Preconditions.notNull(discoveryRequest, "EngineDiscoveryRequest must not be null");
+		LauncherDiscoveryResult discoveryResult = discover(testEngine, discoveryRequest, DISCOVERY);
+		TestDescriptor engineDescriptor = discoveryResult.getEngineTestDescriptor(testEngine);
+		List<DiscoveryIssue> discoveryIssues = discoveryResult.getDiscoveryIssues(testEngine);
+		return new EngineDiscoveryResults(engineDescriptor, discoveryIssues);
+	}
+
 	/**
 	 * Execute tests for the given {@link EngineDiscoveryRequest} using the
 	 * {@link TestEngine} with the supplied ID.
@@ -260,13 +331,16 @@ private static void executeDirectly(TestEngine testEngine, EngineDiscoveryReques
 
 	private static void executeUsingLauncherOrchestration(TestEngine testEngine,
 			LauncherDiscoveryRequest discoveryRequest, EngineExecutionListener listener) {
-		LauncherDiscoveryResult discoveryResult = new EngineDiscoveryOrchestrator(singleton(testEngine),
-			emptySet()).discover(discoveryRequest, EXECUTION);
-		TestDescriptor engineTestDescriptor = discoveryResult.getEngineTestDescriptor(testEngine);
-		Preconditions.notNull(engineTestDescriptor, "TestEngine did not yield a TestDescriptor");
+		LauncherDiscoveryResult discoveryResult = discover(testEngine, discoveryRequest, EXECUTION);
 		new EngineExecutionOrchestrator().execute(discoveryResult, listener);
 	}
 
+	private static LauncherDiscoveryResult discover(TestEngine testEngine, LauncherDiscoveryRequest discoveryRequest,
+			EngineDiscoveryOrchestrator.Phase phase) {
+		return new EngineDiscoveryOrchestrator(singleton(testEngine), emptySet()) //
+				.discover(discoveryRequest, phase);
+	}
+
 	@SuppressWarnings("unchecked")
 	private static TestEngine loadTestEngine(String engineId) {
 		Iterable<TestEngine> testEngines = new ServiceLoaderTestEngineRegistry().loadTestEngines();
@@ -446,6 +520,25 @@ public Builder outputDirectoryProvider(OutputDirectoryProvider outputDirectoryPr
 			return this;
 		}
 
+		/**
+		 * Discover tests for the configured {@link TestEngine},
+		 * {@linkplain DiscoverySelector discovery selectors},
+		 * {@linkplain DiscoveryFilter discovery filters}, and
+		 * <em>configuration parameters</em>.
+		 *
+		 * @return the recorded {@code EngineDiscoveryResults}
+		 * @since 1.13
+		 * @see #selectors(DiscoverySelector...)
+		 * @see #filters(Filter...)
+		 * @see #configurationParameter(String, String)
+		 * @see #configurationParameters(Map)
+		 */
+		@API(status = EXPERIMENTAL, since = "1.13")
+		public EngineDiscoveryResults discover() {
+			LauncherDiscoveryRequest request = this.requestBuilder.build();
+			return EngineTestKit.discover(this.testEngine, request);
+		}
+
 		/**
 		 * Execute tests for the configured {@link TestEngine},
 		 * {@linkplain DiscoverySelector discovery selectors},