Skip to content

differenciate between errors and failures for junitlauncher #221

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

differenciate between errors and failures for junitlauncher #221

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions WHATSNEW
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ Fixed bugs:
path even if it was not a file separator.
Bugzilla Report 69680

* <junitlauncher>'s legacy formatters now separate errors from
failures like their <junit> counterparts did.
Bugzilla Report 69687

Other changes:
--------------

Expand Down
5 changes: 5 additions & 0 deletions ...in/org/apache/tools/ant/taskdefs/optional/junitlauncher/AbstractJUnitResultFormatter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import org.apache.tools.ant.Project;
import org.apache.tools.ant.util.FileUtils;
import org.junit.platform.engine.TestExecutionResult;
import org.junit.platform.engine.TestSource;
import org.junit.platform.engine.support.descriptor.ClassSource;
import org.junit.platform.launcher.TestIdentifier;
Expand Down Expand Up @@ -187,6 +188,10 @@ protected void handleException(final Throwable t) {
}


protected static boolean isFailure(final TestExecutionResult executionResult) {
return executionResult.getThrowable().orElse(null) instanceof AssertionError;
}

/*
A "store" for sysout/syserr content that gets sent to the AbstractJUnitResultFormatter.
This store first uses a relatively decent sized in-memory buffer for storing the sysout/syserr
Expand Down
14 changes: 12 additions & 2 deletions ...main/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyPlainResultFormatter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ public void testPlanExecutionFinished(final TestPlan testPlan) {
final Stats stats = entry.getValue();
final StringBuilder sb = new StringBuilder("Tests run: ").append(stats.numTestsRun.get());
sb.append(", Failures: ").append(stats.numTestsFailed.get());
sb.append(", Errors: ").append(stats.numTestsWithError.get());
sb.append(", Skipped: ").append(stats.numTestsSkipped.get());
sb.append(", Aborted: ").append(stats.numTestsAborted.get());
sb.append(", Time elapsed: ");
Expand Down Expand Up @@ -188,7 +189,11 @@ public void executionFinished(final TestIdentifier testIdentifier, final TestExe
break;
}
case FAILED: {
sb.append(" FAILED");
if (isFailure(testExecutionResult)) {
sb.append(" FAILED");
} else {
sb.append(" Caused an ERROR");
}
appendThrowable(sb, testExecutionResult);
break;
}
Expand All @@ -214,7 +219,11 @@ public void executionFinished(final TestIdentifier testIdentifier, final TestExe
break;
}
case FAILED: {
parentClassStats.numTestsFailed.incrementAndGet();
if (isFailure(testExecutionResult)) {
parentClassStats.numTestsFailed.incrementAndGet();
} else {
parentClassStats.numTestsWithError.incrementAndGet();
}
break;
}
}
Expand Down Expand Up @@ -264,6 +273,7 @@ private final class Stats {
private final TestIdentifier testIdentifier;
private final AtomicLong numTestsRun = new AtomicLong(0);
private final AtomicLong numTestsFailed = new AtomicLong(0);
private final AtomicLong numTestsWithError = new AtomicLong(0);
private final AtomicLong numTestsSkipped = new AtomicLong(0);
private final AtomicLong numTestsAborted = new AtomicLong(0);
private final long startedAt;
Expand Down
35 changes: 30 additions & 5 deletions src/main/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyXmlResultFormatter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,15 @@ class LegacyXmlResultFormatter extends AbstractJUnitResultFormatter implements T
private final Map<TestIdentifier, Stats> testIds = new ConcurrentHashMap<>();
private final Map<TestIdentifier, Optional<String>> skipped = new ConcurrentHashMap<>();
private final Map<TestIdentifier, Optional<Throwable>> failed = new ConcurrentHashMap<>();
private final Map<TestIdentifier, Optional<Throwable>> errored = new ConcurrentHashMap<>();
private final Map<TestIdentifier, Optional<Throwable>> aborted = new ConcurrentHashMap<>();

private TestPlan testPlan;
private long testPlanStartedAt = -1;
private long testPlanEndedAt = -1;
private final AtomicLong numTestsRun = new AtomicLong(0);
private final AtomicLong numTestsFailed = new AtomicLong(0);
private final AtomicLong numTestsErrored = new AtomicLong(0);
private final AtomicLong numTestsSkipped = new AtomicLong(0);
private final AtomicLong numTestsAborted = new AtomicLong(0);
private boolean useLegacyReportingName = true;
Expand Down Expand Up @@ -126,8 +128,13 @@ public void executionFinished(final TestIdentifier testIdentifier, final TestExe
break;
}
case FAILED: {
this.numTestsFailed.incrementAndGet();
this.failed.put(testIdentifier, testExecutionResult.getThrowable());
if (isFailure(testExecutionResult)) {
this.numTestsFailed.incrementAndGet();
this.failed.put(testIdentifier, testExecutionResult.getThrowable());
} else {
this.numTestsErrored.incrementAndGet();
this.errored.put(testIdentifier, testExecutionResult.getThrowable());
}
break;
}
}
Expand Down Expand Up @@ -168,6 +175,7 @@ private final class XMLReportWriter {
private static final String ELEM_TESTCASE = "testcase";
private static final String ELEM_SKIPPED = "skipped";
private static final String ELEM_FAILURE = "failure";
private static final String ELEM_ERROR = "error";
private static final String ELEM_ABORTED = "aborted";
private static final String ELEM_SYSTEM_OUT = "system-out";
private static final String ELEM_SYSTEM_ERR = "system-err";
Expand All @@ -180,6 +188,7 @@ private final class XMLReportWriter {
private static final String ATTR_TIMESTAMP = "timestamp";
private static final String ATTR_NUM_ABORTED = "aborted";
private static final String ATTR_NUM_FAILURES = "failures";
private static final String ATTR_NUM_ERRORS = "errors";
private static final String ATTR_NUM_TESTS = "tests";
private static final String ATTR_NUM_SKIPPED = "skipped";
private static final String ATTR_MESSAGE = "message";
Expand Down Expand Up @@ -208,6 +217,7 @@ void writeTestSuite(final XMLStreamWriter writer) throws XMLStreamException, IOE
writeAttribute(writer, ATTR_TIMESTAMP, timestamp);
writeAttribute(writer, ATTR_NUM_TESTS, String.valueOf(numTestsRun.longValue()));
writeAttribute(writer, ATTR_NUM_FAILURES, String.valueOf(numTestsFailed.longValue()));
writeAttribute(writer, ATTR_NUM_ERRORS, String.valueOf(numTestsErrored.longValue()));
writeAttribute(writer, ATTR_NUM_SKIPPED, String.valueOf(numTestsSkipped.longValue()));
writeAttribute(writer, ATTR_NUM_ABORTED, String.valueOf(numTestsAborted.longValue()));

Expand Down Expand Up @@ -239,7 +249,7 @@ void writeProperties(final XMLStreamWriter writer) throws XMLStreamException {
void writeTestCase(final XMLStreamWriter writer) throws XMLStreamException {
for (final Map.Entry<TestIdentifier, Stats> entry : testIds.entrySet()) {
final TestIdentifier testId = entry.getKey();
if (!testId.isTest() && !failed.containsKey(testId)) {
if (!testId.isTest() && !failed.containsKey(testId) && !errored.containsKey(testId)) {
// only interested in test methods unless there was a failure,
// in which case we want the exception reported
// (https://bz.apache.org/bugzilla/show_bug.cgi?id=63850)
Expand Down Expand Up @@ -283,6 +293,8 @@ void writeTestCase(final XMLStreamWriter writer) throws XMLStreamException {
writeSkipped(writer, testId);
// failed element if the test failed
writeFailed(writer, testId);
// error element if the test caused an error
writeErrored(writer, testId);
// aborted element if the test was aborted
writeAborted(writer, testId);

Expand All @@ -306,8 +318,21 @@ private void writeFailed(final XMLStreamWriter writer, final TestIdentifier test
if (!failed.containsKey(testIdentifier)) {
return;
}
writer.writeStartElement(ELEM_FAILURE);
final Optional<Throwable> cause = failed.get(testIdentifier);
writeFailedOrErrored(writer, ELEM_FAILURE, failed.get(testIdentifier));
}

private void writeErrored(final XMLStreamWriter writer, final TestIdentifier testIdentifier) throws XMLStreamException {
if (!errored.containsKey(testIdentifier)) {
return;
}
writeFailedOrErrored(writer, ELEM_ERROR, errored.get(testIdentifier));
}

private void writeFailedOrErrored(final XMLStreamWriter writer,
final String elementName,
final Optional<Throwable> cause)
throws XMLStreamException {
writer.writeStartElement(elementName);
if (cause.isPresent()) {
final Throwable t = cause.get();
final String message = t.getMessage();
Expand Down
79 changes: 79 additions & 0 deletions ...it/org/apache/tools/ant/taskdefs/optional/junitlauncher/LegacyXmlResultFormatterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,18 +20,36 @@
import org.apache.tools.ant.Project;
import org.junit.Test;
import org.junit.platform.engine.ConfigurationParameters;
import org.junit.platform.engine.TestDescriptor;
import org.junit.platform.engine.TestExecutionResult;
import org.junit.platform.engine.UniqueId;
import org.junit.platform.engine.support.descriptor.AbstractTestDescriptor;
import org.junit.platform.launcher.TestIdentifier;
import org.junit.platform.launcher.TestPlan;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import java.util.function.Function;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.sax.SAXSource;
import javax.xml.transform.stream.StreamSource;

import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.junit.Assert.assertThat;

public class LegacyXmlResultFormatterTest {
Expand All @@ -57,6 +75,56 @@ public void encodesSysOutProperly() throws Exception {
assertThat(result, containsString(ENCODED));
}

@Test
public void properlyReportsFailures() throws Exception {
properlyReportsFailuresAndErrors(new AssertionError("failed", null), true);
}

@Test
public void properlyReportsErrors() throws Exception {
properlyReportsFailuresAndErrors(new NullPointerException("failed"), false);
}

private void properlyReportsFailuresAndErrors(Throwable errorOrFailure,
boolean shouldBeFailure)
throws Exception {

final TestPlan plan = startTest(false);
final TestDescriptor testDescriptor = new DummyTestDescriptor("failure");
final TestIdentifier test = TestIdentifier.from(testDescriptor);
f.executionStarted(test);
final TestExecutionResult testResult = TestExecutionResult.failed(errorOrFailure);
f.executionFinished(test, testResult);
final String result = finishTest(plan);

final int expectedFailureCount = shouldBeFailure ? 1 : 0;
final int expectedErrorCount = shouldBeFailure ? 0 : 1;

final StreamSource source = new StreamSource() {
@Override
public Reader getReader() {
return new StringReader(result);
}
};
final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
final InputSource is = SAXSource.sourceToInputSource(source);
final DocumentBuilder b = dbf.newDocumentBuilder();
final Document doc = b.parse(is);
final Element suite = doc.getDocumentElement();
assertThat(suite.getTagName(), equalTo("testsuite"));
assertThat(suite.getAttribute("failures"), equalTo(expectedFailureCount + ""));
assertThat(suite.getAttribute("errors"), equalTo(expectedErrorCount + ""));
final NodeList testCases = suite.getElementsByTagName("testcase");
assertThat(testCases.getLength(), equalTo(1));
final Node testCase = testCases.item(0);
assertThat(testCase, instanceOf(Element.class));
final Element testCaseElement = (Element) testCase;
NodeList failureElements = testCaseElement.getElementsByTagName("failure");
assertThat(failureElements.getLength(), equalTo(expectedFailureCount));
NodeList errorElements = testCaseElement.getElementsByTagName("error");
assertThat(errorElements.getLength(), equalTo(expectedErrorCount));
}

private TestPlan startTest(final boolean withProperties) {
f.setContext(new TestExecutionContext() {
@Override
Expand Down Expand Up @@ -111,4 +179,15 @@ private String finishTest(final TestPlan testPlan) throws IOException {
return new String(bos.toByteArray(), StandardCharsets.UTF_8);
}
}

private static class DummyTestDescriptor extends AbstractTestDescriptor {
private DummyTestDescriptor(String displayName) {
super(UniqueId.forEngine("testengine"), displayName);
}

@Override
public TestDescriptor.Type getType() {
return TestDescriptor.Type.TEST;
}
}
}