[SUREFIRE-1643] surefire junit5 parallel tests (#815)

* [SUREFIRE-1643] Make surefire junit5 supporting parallel execution and not mixing result when re running tests


---------

Signed-off-by: Olivier Lamy <[email protected]>

Author: olamy

Jenkinsfile

@@ -21,19 +21,18 @@
 
 properties(
     [
-        buildDiscarder(logRotator(artifactDaysToKeepStr: env.BRANCH_NAME == 'master' ? '14' : '7',
-                                  artifactNumToKeepStr: '50',
+        buildDiscarder(logRotator(artifactNumToKeepStr: env.BRANCH_NAME == 'master' ? '15' : '5',
                                   daysToKeepStr: env.BRANCH_NAME == 'master' ? '30' : '14',
                                   numToKeepStr: env.BRANCH_NAME == 'master' ? '20' : '10')
-        ),
-        disableConcurrentBuilds()
+        )//,
+        //disableConcurrentBuilds()
     ]
 )
 // final def oses = ['linux':'ubuntu && maven', 'windows':'windows-he']
-final def oses = ['linux':'ubuntu && maven']
+final def oses = ['linux':'ubuntu']
 final def mavens = env.BRANCH_NAME == 'master' ? ['3.x.x', '3.6.3'] : ['3.x.x']
 // all non-EOL versions and the first EA
-final def jdks = [20, 17, 11, 8]
+final def jdks = [21, 17, 11, 8]
 
 final def options = ['-e', '-V', '-B', '-nsu', '-P', 'run-its']
 final def goals = ['clean', 'install']
@@ -134,7 +133,7 @@ def buildProcess(String stageKey, String jdkName, String mvnName, goals, options
         println "Maven Local Repository = ${mvnLocalRepoDir}."
         assert mvnLocalRepoDir != null : 'Local Maven Repository is undefined.'
 
-        def properties = ["-Djacoco.skip=${!makeReports}", "\"-Dmaven.repo.local=${mvnLocalRepoDir}\""]
+        def properties = ["-Papache.ci -Djacoco.skip=${!makeReports}", "\"-Dmaven.repo.local=${mvnLocalRepoDir}\""]
         def cmd = ['mvn'] + goals + options + properties
 
         stage("build ${stageKey}") {

maven-failsafe-plugin/pom.xml

@@ -233,30 +233,6 @@
   </reporting>
 
   <profiles>
-    <profile>
-      <id>ci</id>
-      <activation>
-        <property>
-          <name>enableCiProfile</name>
-          <value>true</value>
-        </property>
-      </activation>
-      <build>
-        <plugins>
-          <plugin>
-            <artifactId>maven-docck-plugin</artifactId>
-            <version>1.2</version>
-            <executions>
-              <execution>
-                <goals>
-                  <goal>check</goal>
-                </goals>
-              </execution>
-            </executions>
-          </plugin>
-        </plugins>
-      </build>
-    </profile>
     <profile>
       <id>run-its</id>
       <build>

maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/AbstractSurefireMojo.java

@@ -85,6 +85,7 @@
 import org.apache.maven.surefire.api.booter.Shutdown;
 import org.apache.maven.surefire.api.cli.CommandLineOption;
 import org.apache.maven.surefire.api.report.ReporterConfiguration;
+import org.apache.maven.surefire.api.report.ReporterFactoryOptions;
 import org.apache.maven.surefire.api.suite.RunResult;
 import org.apache.maven.surefire.api.testset.DirectoryScannerParameters;
 import org.apache.maven.surefire.api.testset.RunOrderParameters;
@@ -2056,7 +2057,8 @@ private Artifact getShadefireArtifact() {
         return getPluginArtifactMap().get("org.apache.maven.surefire:surefire-shadefire");
     }
 
-    private StartupReportConfiguration getStartupReportConfiguration(String configChecksum, boolean isForking) {
+    private StartupReportConfiguration getStartupReportConfiguration(
+            String configChecksum, boolean isForking, ProviderInfo providerInfo) {
         SurefireStatelessReporter xmlReporter =
                 statelessTestsetReporter == null ? new SurefireStatelessReporter() : statelessTestsetReporter;
 
@@ -2071,6 +2073,11 @@ private StartupReportConfiguration getStartupReportConfiguration(String configCh
                 ? new SurefireStatelessTestsetInfoReporter()
                 : statelessTestsetInfoReporter;
 
+        // SUREFIRE-1643 we can force this here as junit5 will need this for multi thread test
+        ReporterFactoryOptions reporterFactoryOptions = new ReporterFactoryOptions();
+        if ("org.apache.maven.surefire.junitplatform.JUnitPlatformProvider".equals(providerInfo.getProviderName())) {
+            reporterFactoryOptions.setStatPerSourceName(true);
+        }
         return new StartupReportConfiguration(
                 isUseFile(),
                 isPrintSummary(),
@@ -2089,7 +2096,8 @@ private StartupReportConfiguration getStartupReportConfiguration(String configCh
                 isEnablePropertiesElement(),
                 xmlReporter,
                 outReporter,
-                testsetReporter);
+                testsetReporter,
+                reporterFactoryOptions);
     }
 
     private boolean isSpecificTestSpecified() {
@@ -2317,7 +2325,9 @@ private ForkStarter createForkStarter(
                 platform,
                 resolvedJavaModularityResult);
         String configChecksum = getConfigChecksum();
-        StartupReportConfiguration startupReportConfiguration = getStartupReportConfiguration(configChecksum, true);
+        StartupReportConfiguration startupReportConfiguration =
+                getStartupReportConfiguration(configChecksum, true, provider);
+
         ProviderConfiguration providerConfiguration = createProviderConfiguration(runOrderParameters);
         return new ForkStarter(
                 providerConfiguration,
@@ -2345,7 +2355,8 @@ private InPluginVMSurefireStarter createInprocessStarter(
                 platform,
                 new ResolvePathResultWrapper(null, true));
         String configChecksum = getConfigChecksum();
-        StartupReportConfiguration startupReportConfiguration = getStartupReportConfiguration(configChecksum, false);
+        StartupReportConfiguration startupReportConfiguration =
+                getStartupReportConfiguration(configChecksum, false, provider);
         ProviderConfiguration providerConfiguration = createProviderConfiguration(runOrderParameters);
         return new InPluginVMSurefireStarter(
                 startupConfiguration, providerConfiguration, startupReportConfiguration, getConsoleLogger(), platform);

maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/CommonReflector.java

@@ -29,6 +29,7 @@
 import org.apache.maven.plugin.surefire.log.api.ConsoleLogger;
 import org.apache.maven.plugin.surefire.log.api.ConsoleLoggerDecorator;
 import org.apache.maven.plugin.surefire.report.DefaultReporterFactory;
+import org.apache.maven.surefire.api.report.ReporterFactoryOptions;
 import org.apache.maven.surefire.api.util.SurefireReflectionException;
 
 import static org.apache.maven.surefire.api.util.ReflectionUtils.getConstructor;
@@ -44,6 +45,7 @@ public class CommonReflector {
     private final Class<?> statelessTestsetReporter;
     private final Class<?> consoleOutputReporter;
     private final Class<?> statelessTestsetInfoReporter;
+    private final Class<?> reporterFactoryOptions;
     private final ClassLoader surefireClassLoader;
 
     public CommonReflector(@Nonnull ClassLoader surefireClassLoader) {
@@ -56,6 +58,7 @@ public CommonReflector(@Nonnull ClassLoader surefireClassLoader) {
             consoleOutputReporter = surefireClassLoader.loadClass(SurefireConsoleOutputReporter.class.getName());
             statelessTestsetInfoReporter =
                     surefireClassLoader.loadClass(SurefireStatelessTestsetInfoReporter.class.getName());
+            reporterFactoryOptions = surefireClassLoader.loadClass(ReporterFactoryOptions.class.getName());
         } catch (ClassNotFoundException e) {
             throw new SurefireReflectionException(e);
         }
@@ -90,7 +93,8 @@ private Object createStartupReportConfiguration(@Nonnull StartupReportConfigurat
                 boolean.class,
                 statelessTestsetReporter,
                 consoleOutputReporter,
-                statelessTestsetInfoReporter);
+                statelessTestsetInfoReporter,
+                reporterFactoryOptions);
         Object[] params = {
             reporterConfiguration.isUseFile(),
             reporterConfiguration.isPrintSummary(),
@@ -109,7 +113,8 @@ private Object createStartupReportConfiguration(@Nonnull StartupReportConfigurat
             reporterConfiguration.isEnablePropertiesElement(),
             reporterConfiguration.getXmlReporter().clone(surefireClassLoader),
             reporterConfiguration.getConsoleOutputReporter().clone(surefireClassLoader),
-            reporterConfiguration.getTestsetReporter().clone(surefireClassLoader)
+            reporterConfiguration.getTestsetReporter().clone(surefireClassLoader),
+            reporterConfiguration.getReporterFactoryOptions().clone(surefireClassLoader)
         };
         return newInstance(constructor, params);
     }

maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/StartupReportConfiguration.java

@@ -35,6 +35,7 @@
 import org.apache.maven.plugin.surefire.report.TestSetStats;
 import org.apache.maven.plugin.surefire.report.WrappedReportEntry;
 import org.apache.maven.plugin.surefire.runorder.StatisticsReporter;
+import org.apache.maven.surefire.api.report.ReporterFactoryOptions;
 import org.apache.maven.surefire.extensions.ConsoleOutputReportEventListener;
 import org.apache.maven.surefire.extensions.StatelessReportEventListener;
 import org.apache.maven.surefire.extensions.StatelessTestsetInfoConsoleReportEventListener;
@@ -97,6 +98,8 @@ public final class StartupReportConfiguration {
 
     private StatisticsReporter statisticsReporter;
 
+    private final ReporterFactoryOptions reporterFactoryOptions;
+
     /**
      * @since 3.3.1
      */
@@ -119,7 +122,8 @@ public StartupReportConfiguration(
             boolean enablePropertiesElement,
             SurefireStatelessReporter xmlReporter,
             SurefireConsoleOutputReporter consoleOutputReporter,
-            SurefireStatelessTestsetInfoReporter testsetReporter) {
+            SurefireStatelessTestsetInfoReporter testsetReporter,
+            ReporterFactoryOptions reporterFactoryOptions) {
         this.useFile = useFile;
         this.printSummary = printSummary;
         this.reportFormat = reportFormat;
@@ -141,6 +145,7 @@ public StartupReportConfiguration(
         this.xmlReporter = xmlReporter;
         this.consoleOutputReporter = consoleOutputReporter;
         this.testsetReporter = testsetReporter;
+        this.reporterFactoryOptions = reporterFactoryOptions;
     }
 
     @SuppressWarnings("checkstyle:parameternumber")
@@ -180,7 +185,8 @@ public StartupReportConfiguration(
                 true,
                 xmlReporter,
                 consoleOutputReporter,
-                testsetReporter);
+                testsetReporter,
+                new ReporterFactoryOptions(false));
     }
 
     public boolean isUseFile() {
@@ -220,7 +226,7 @@ public StatelessReportEventListener<WrappedReportEntry, TestSetStats> instantiat
         // In the in-plugin execution of parallel JUnit4.7 with rerun the map must be shared because reports and
         // listeners are in ThreadLocal, see Surefire1122ParallelAndFlakyTestsIT.
         Map<String, Deque<WrappedReportEntry>> testClassMethodRunHistory =
-                isForking ? new ConcurrentHashMap<String, Deque<WrappedReportEntry>>() : this.testClassMethodRunHistory;
+                isForking ? new ConcurrentHashMap<>() : this.testClassMethodRunHistory;
 
         DefaultStatelessReportMojoConfiguration xmlReporterConfig = new DefaultStatelessReportMojoConfiguration(
                 resolveReportsDirectory(forkNumber),
@@ -319,4 +325,8 @@ public SurefireStatelessTestsetInfoReporter getTestsetReporter() {
     private boolean shouldReportToConsole() {
         return isUseFile() ? isPrintSummary() : isRedirectTestOutputToFile() || isBriefOrPlainFormat();
     }
+
+    public ReporterFactoryOptions getReporterFactoryOptions() {
+        return reporterFactoryOptions;
+    }
 }

maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java

@@ -103,7 +103,8 @@ public TestReportListener<TestOutputReportEntry> createTestReportListener() {
                 reportConfiguration.isTrimStackTrace(),
                 PLAIN.equals(reportConfiguration.getReportFormat()),
                 reportConfiguration.isBriefOrPlainFormat(),
-                consoleLogger);
+                consoleLogger,
+                reportConfiguration.getReporterFactoryOptions().isStatPerSourceName());
         addListener(testSetRunListener);
         return testSetRunListener;
     }

maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java

@@ -20,12 +20,12 @@
 
 import java.io.BufferedOutputStream;
 import java.io.File;
-import java.io.FileOutputStream;
 import java.io.FilterOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
+import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Deque;
 import java.util.LinkedHashMap;
@@ -392,7 +392,7 @@ private OutputStream getOutputStream(WrappedReportEntry testSetReportEntry) thro
         reportFile.delete();
         //noinspection ResultOfMethodCallIgnored
         reportDir.mkdirs();
-        return new BufferedOutputStream(new FileOutputStream(reportFile), 64 * 1024);
+        return new BufferedOutputStream(Files.newOutputStream(reportFile.toPath()), 64 * 1024);
     }
 
     private static OutputStreamWriter getWriter(OutputStream fos) {