diff --git a/pkl-cli/src/main/kotlin/org/pkl/cli/CliTestRunner.kt b/pkl-cli/src/main/kotlin/org/pkl/cli/CliTestRunner.kt index b4415dd8..4ce530e0 100644 --- a/pkl-cli/src/main/kotlin/org/pkl/cli/CliTestRunner.kt +++ b/pkl-cli/src/main/kotlin/org/pkl/cli/CliTestRunner.kt @@ -60,13 +60,16 @@ constructor( evaluator.use { var failed = false val moduleNames = mutableSetOf() - for (moduleUri in sources) { + for ((idx, moduleUri) in sources.withIndex()) { try { val results = evaluator.evaluateTest(uri(moduleUri), testOptions.overwrite) if (!failed) { failed = results.failed() } SimpleReport().report(results, consoleWriter) + if (sources.size > 1 && idx != sources.size - 1) { + consoleWriter.append('\n') + } consoleWriter.flush() val junitDir = testOptions.junitDir if (junitDir != null) { diff --git a/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt b/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt index 3f728604..6b9ee7f9 100644 --- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt +++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt @@ -63,8 +63,10 @@ class CliTestRunnerTest { .isEqualTo( """ module test - succeed ✅ - + facts + ✅ succeed + ✅ 100.0% tests pass [1 passed], 100.0% asserts pass [2 passed] + """ .trimIndent() ) @@ -80,7 +82,7 @@ class CliTestRunnerTest { facts { ["fail"] { 4 == 9 - "foo" == "bar" + "foo" != "bar" } } """ @@ -97,10 +99,11 @@ class CliTestRunnerTest { .isEqualTo( """ module test - fail ❌ - 4 == 9 ❌ - "foo" == "bar" ❌ - + facts + ❌ fail + 4 == 9 + ❌ 0.0% tests pass [1/1 failed], 50.0% asserts pass [1/2 failed] + """ .trimIndent() ) @@ -132,14 +135,15 @@ class CliTestRunnerTest { .isEqualToNormalizingNewlines( """ module test - fail ❌ - Error: - –– Pkl Error –– - uh oh - - 5 | throw("uh oh") - ^^^^^^^^^^^^^^ - at test#facts["fail"][#1] + facts + ❌ fail + –– Pkl Error –– + uh oh + + 5 | throw("uh oh") + ^^^^^^^^^^^^^^ + at test#facts["fail"][#1] + ❌ 0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed] """ .trimIndent() @@ -172,14 +176,15 @@ class CliTestRunnerTest { .isEqualTo( """ module test - fail ❌ - Error: - –– Pkl Error –– - uh oh - - 5 | throw("uh oh") - ^^^^^^^^^^^^^^ - at test#examples["fail"][#1] + examples + ❌ fail + –– Pkl Error –– + uh oh + + 5 | throw("uh oh") + ^^^^^^^^^^^^^^ + at test#examples["fail"][#1] + ❌ 0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed] """ .trimIndent() @@ -226,14 +231,15 @@ class CliTestRunnerTest { .isEqualToNormalizingNewlines( """ module test - fail ❌ - Error: - –– Pkl Error –– - uh oh - - 5 | throw("uh oh") - ^^^^^^^^^^^^^^ - at test#examples["fail"][#1] + examples + ❌ fail + –– Pkl Error –– + uh oh + + 5 | throw("uh oh") + ^^^^^^^^^^^^^^ + at test#examples["fail"][#1] + ❌ 0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed] """ .trimIndent() @@ -252,7 +258,8 @@ class CliTestRunnerTest { 9 == trace(9) "foo" == "foo" } - ["fail"] { + ["bar"] { + "foo" == "foo" 5 == 9 } } @@ -270,9 +277,9 @@ class CliTestRunnerTest { """ - - - 5 == 9 ❌ + + + 5 == 9 @@ -311,9 +318,9 @@ class CliTestRunnerTest { .isEqualTo( """ - - - + + + –– Pkl Error –– uh oh diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/TestResults.java b/pkl-core/src/main/java/org/pkl/core/runtime/TestResults.java index 830116aa..a623cbf8 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/TestResults.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/TestResults.java @@ -20,61 +20,40 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import org.pkl.core.PklException; +import org.pkl.core.runtime.TestResults.TestSectionResults.TestSection; /** Aggregate test results for a module. Used to verify test failures and generate reports. */ public final class TestResults { - - private final String module; - private final String displayUri; - private final List results = new ArrayList<>(); + public final String moduleName; + public final String displayUri; + public final TestSectionResults module = new TestSectionResults(TestSection.MODULE); + public final TestSectionResults facts = new TestSectionResults(TestSection.FACTS); + public final TestSectionResults examples = new TestSectionResults(TestSection.EXAMPLES); private String err = ""; - public TestResults(String module, String displayUri) { - this.module = module; + public TestResults(String moduleName, String displayUri) { + this.moduleName = moduleName; this.displayUri = displayUri; } - public String getModuleName() { - return module; - } - - public String getDisplayUri() { - return displayUri; - } - - public List getResults() { - return Collections.unmodifiableList(results); - } - - public TestResult newResult(String name) { - var result = new TestResult(name); - results.add(result); - return result; - } - - public void newResult(String name, Failure failure) { - var result = new TestResult(name); - result.addFailure(failure); - results.add(result); - } - public int totalTests() { - return results.size(); + return module.totalTests() + facts.totalTests() + examples.totalTests(); } public int totalFailures() { - int total = 0; - for (var res : results) { - total += res.getFailures().size(); - } - return total; + return module.totalFailures() + facts.totalFailures() + examples.totalFailures(); + } + + public int totalAsserts() { + return module.totalAsserts() + facts.totalAsserts() + examples.totalAsserts(); + } + + public int totalAssertsFailed() { + return module.totalAssertsFailed() + facts.totalAssertsFailed() + examples.totalAssertsFailed(); } public boolean failed() { - for (var res : results) { - if (res.isFailure()) return true; - } - return false; + return module.failed() || facts.failed() || examples.failed(); } public String getErr() { @@ -85,147 +64,295 @@ public final class TestResults { this.err = err; } - public static class TestResult { + public static class TestSectionResults { + public final TestSection name; + private final List results = new ArrayList<>(); + private Error error; - private final String name; - private final List failures = new ArrayList<>(); - private final List errors = new ArrayList<>(); - private boolean isExampleWritten = false; - - public TestResult(String name) { + public TestSectionResults(TestSection name) { this.name = name; } - public boolean isSuccess() { - return failures.isEmpty() && errors.isEmpty(); + public void setError(Error error) { + this.error = error; } - boolean isFailure() { - return !isSuccess(); + public Error getError() { + return error; } - public String getName() { - return name; + public boolean hasError() { + return error != null; } - public boolean isExampleWritten() { - return isExampleWritten; + public List getResults() { + return Collections.unmodifiableList(results); } - public void setExampleWritten(boolean exampleWritten) { - isExampleWritten = exampleWritten; + public TestResult newResult(String name) { + var result = new TestResult(name, this.name == TestSection.EXAMPLES); + results.add(result); + return result; } - public List getFailures() { - return Collections.unmodifiableList(failures); + public TestResult newResult(String name, Failure failure) { + var result = new TestResult(name, this.name == TestSection.EXAMPLES); + result.addFailure(failure); + results.add(result); + return result; } - public void addFailure(Failure description) { - failures.add(description); + public TestResult newResult(String name, Error error) { + var result = new TestResult(name, this.name == TestSection.EXAMPLES); + result.addError(error); + results.add(result); + return result; } - public List getErrors() { - return Collections.unmodifiableList(errors); + public int totalTests() { + var total = results.size(); + return (hasError() ? ++total : total); } - public void addError(Error err) { - errors.add(err); - } - } - - public static class Failure { - - private final String kind; - private final String rendered; - - private Failure(String kind, String rendered) { - this.kind = kind; - this.rendered = rendered; - } - - public String getKind() { - return kind; - } - - public String getRendered() { - return rendered; - } - - public static Failure buildFactFailure(SourceSection sourceSection, String description) { - return new Failure( - "Fact Failure", sourceSection.getCharacters() + " ❌ (" + description + ")"); - } - - public static Failure buildExampleLengthMismatchFailure( - String location, String property, int expectedLength, int actualLength) { - String builder = - "(" - + location - + ")\n" - + "Output mismatch: Expected \"" - + property - + "\" to contain " - + expectedLength - + " examples, but found " - + actualLength; - return new Failure("Output Mismatch (Length)", builder); - } - - public static Failure buildExamplePropertyMismatchFailure( - String location, String property, boolean isMissingInExpected) { - var builder = new StringBuilder(); - builder - .append("(") - .append(location) - .append(")\n") - .append("Output mismatch: \"") - .append(property); - if (isMissingInExpected) { - builder.append("\" exists in actual but not in expected output"); - } else { - builder.append("\" exists in expected but not in actual output"); + public int totalAsserts() { + int total = 0; + for (var res : results) { + total += res.totalAsserts(); } - return new Failure("Output Mismatch", builder.toString()); + return (hasError() ? ++total : total); } - public static Failure buildExampleFailure( - String location, - String expectedLocation, - String expectedValue, - String actualLocation, - String actualValue) { - String builder = - "(" - + location - + ")\n" - + "Expected: (" - + expectedLocation - + ")\n" - + expectedValue - + "\nActual: (" - + actualLocation - + ")\n" - + actualValue; - return new Failure("Example Failure", builder); - } - } - - public static class Error { - - private final String message; - private final PklException exception; - - public Error(String message, PklException exception) { - this.message = message; - this.exception = exception; + public int totalAssertsFailed() { + int total = 0; + for (var res : results) { + total += res.totalAssertsFailed(); + } + return (hasError() ? ++total : total); } - public String getMessage() { - return message; + public int totalFailures() { + int total = 0; + for (var res : results) { + if (res.isFailure()) total++; + } + return (hasError() ? ++total : total); } - public Exception getException() { - return exception; + public boolean failed() { + if (hasError()) return true; + + for (var res : results) { + if (res.isFailure()) return true; + } + return false; + } + + public static class TestResult { + public final String name; + private int totalAsserts = 0; + private int totalAssertsFailed = 0; + private final List failures = new ArrayList<>(); + private final List errors = new ArrayList<>(); + public final boolean isExample; + private boolean isExampleWritten = false; + + public TestResult(String name, boolean isExample) { + this.name = name; + this.isExample = isExample; + } + + public boolean isSuccess() { + return failures.isEmpty() && errors.isEmpty(); + } + + public boolean isFailure() { + return !isSuccess(); + } + + public boolean isExampleWritten() { + return isExampleWritten; + } + + public void setExampleWritten(boolean exampleWritten) { + isExampleWritten = exampleWritten; + } + + public List getFailures() { + return Collections.unmodifiableList(failures); + } + + public void addFailure(Failure description) { + failures.add(description); + totalAssertsFailed++; + } + + public List getErrors() { + return Collections.unmodifiableList(errors); + } + + public void addError(Error err) { + errors.add(err); + totalAssertsFailed++; + } + + public int totalAsserts() { + return totalAsserts; + } + + public void countAssert() { + totalAsserts++; + } + + public int totalAssertsFailed() { + return totalAssertsFailed; + } + } + + public static class Failure { + + private final String kind; + private final String failure; + private final String location; + + private Failure(String kind, String failure, String location) { + this.kind = kind; + this.failure = failure; + this.location = location; + } + + public String getKind() { + return kind; + } + + public String getFailure() { + return failure; + } + + public String getLocation() { + return location; + } + + public static String renderLocation(String location) { + return "(" + location + ")"; + } + + public String getRendered() { + String rendered; + + if (kind == "Fact Failure") { + rendered = failure + " " + renderLocation(getLocation()); + } else { + rendered = renderLocation(getLocation()) + "\n" + failure; + } + + return rendered; + } + + public static Failure buildFactFailure(SourceSection sourceSection, String location) { + return new Failure("Fact Failure", sourceSection.getCharacters().toString(), location); + } + + public static Failure buildExampleLengthMismatchFailure( + String location, String property, int expectedLength, int actualLength) { + var builder = new StringBuilder(); + builder + .append("Output mismatch: Expected \"") + .append(property) + .append("\" to contain ") + .append(expectedLength) + .append(" examples, but found ") + .append(actualLength); + + return new Failure("Output Mismatch (Length)", builder.toString(), location); + } + + public static Failure buildExamplePropertyMismatchFailure( + String location, String property, boolean isMissingInExpected) { + + String exists_in; + String missing_in; + + if (isMissingInExpected) { + exists_in = "actual"; + missing_in = "expected"; + } else { + exists_in = "expected"; + missing_in = "actual"; + } + + var builder = new StringBuilder(); + builder + .append("Output mismatch: \"") + .append(property) + .append("\" exists in ") + .append(exists_in) + .append(" but not in ") + .append(missing_in) + .append(" output"); + + return new Failure("Output Mismatch", builder.toString(), location); + } + + public static Failure buildExampleFailure( + String location, + String expectedLocation, + String expectedValue, + String actualLocation, + String actualValue) { + var builder = new StringBuilder(); + builder + .append("Expected: ") + .append(renderLocation(expectedLocation)) + .append("\n") + .append(expectedValue) + .append("\n") + .append("Actual: ") + .append(renderLocation(actualLocation)) + .append("\n") + .append(actualValue); + + return new Failure("Example Failure", builder.toString(), location); + } + } + + public static class Error { + + private final String message; + private final PklException exception; + + public Error(String message, PklException exception) { + this.message = message; + this.exception = exception; + } + + public String getMessage() { + return message; + } + + public Exception getException() { + return exception; + } + + public String getRendered() { + return exception.getMessage(); + } + } + + public enum TestSection { + MODULE("module"), + FACTS("facts"), + EXAMPLES("examples"); + + private final String name; + + TestSection(final String name) { + this.name = name; + } + + @Override + public String toString() { + return name; + } } } } diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java b/pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java index 2bc76c0c..74b21c1c 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java @@ -23,8 +23,9 @@ import org.pkl.core.BufferedLogger; import org.pkl.core.StackFrameTransformer; import org.pkl.core.ast.member.ObjectMember; import org.pkl.core.module.ModuleKeys; -import org.pkl.core.runtime.TestResults.Error; -import org.pkl.core.runtime.TestResults.Failure; +import org.pkl.core.runtime.TestResults.TestSectionResults; +import org.pkl.core.runtime.TestResults.TestSectionResults.Error; +import org.pkl.core.runtime.TestResults.TestSectionResults.Failure; import org.pkl.core.stdlib.PklConverter; import org.pkl.core.stdlib.base.PcfRenderer; import org.pkl.core.util.EconomicMaps; @@ -51,12 +52,25 @@ public final class TestRunner { try { checkAmendsPklTest(testModule); - runFacts(testModule, results); - runExamples(testModule, info, results); } catch (VmException v) { - var meta = results.newResult(info.getModuleName()); - meta.addError(new Error(v.getMessage(), v.toPklException(stackFrameTransformer))); + var error = new Error(v.getMessage(), v.toPklException(stackFrameTransformer)); + results.module.setError(error); } + + try { + runFacts(testModule, results.facts); + } catch (VmException v) { + var error = new Error(v.getMessage(), v.toPklException(stackFrameTransformer)); + results.facts.setError(error); + } + + try { + runExamples(testModule, info, results.examples); + } catch (VmException v) { + var error = new Error(v.getMessage(), v.toPklException(stackFrameTransformer)); + results.examples.setError(error); + } + results.setErr(logger.getLogs()); return results; } @@ -72,7 +86,7 @@ public final class TestRunner { } } - private void runFacts(VmTyped testModule, TestResults results) { + private void runFacts(VmTyped testModule, TestSectionResults results) { var facts = VmUtils.readMember(testModule, Identifier.FACTS); if (facts instanceof VmNull) return; @@ -86,6 +100,9 @@ public final class TestRunner { if (member.isLocalOrExternalOrHidden()) { return true; } + + result.countAssert(); + try { var factValue = VmUtils.readMember(listing, idx); if (factValue == Boolean.FALSE) { @@ -101,7 +118,7 @@ public final class TestRunner { }); } - private void runExamples(VmTyped testModule, ModuleInfo info, TestResults results) { + private void runExamples(VmTyped testModule, ModuleInfo info, TestSectionResults results) { var examples = VmUtils.readMember(testModule, Identifier.EXAMPLES); if (examples instanceof VmNull) return; @@ -144,7 +161,10 @@ public final class TestRunner { } private void doRunAndValidateExamples( - VmMapping examples, Path expectedOutputFile, Path actualOutputFile, TestResults results) { + VmMapping examples, + Path expectedOutputFile, + Path actualOutputFile, + TestSectionResults results) { var expectedExampleOutputs = loadExampleOutputs(expectedOutputFile); var actualExampleOutputs = new MutableReference(null); var allGroupsSucceeded = new MutableBoolean(true); @@ -155,23 +175,27 @@ public final class TestRunner { var group = (VmListing) groupValue; var expectedGroup = (VmDynamic) VmUtils.readMemberOrNull(expectedExampleOutputs, groupKey); - var result = results.newResult(testName); if (expectedGroup == null) { - results.newResult( - testName, - Failure.buildExamplePropertyMismatchFailure( - getDisplayUri(groupMember), String.valueOf(groupKey), true)); + results + .newResult( + testName, + Failure.buildExamplePropertyMismatchFailure( + getDisplayUri(groupMember), testName, true)) + .countAssert(); return true; } if (group.getLength() != expectedGroup.getLength()) { - result.addFailure( - Failure.buildExampleLengthMismatchFailure( - getDisplayUri(groupMember), - String.valueOf(groupKey), - expectedGroup.getLength(), - group.getLength())); + results + .newResult( + testName, + Failure.buildExampleLengthMismatchFailure( + getDisplayUri(groupMember), + testName, + expectedGroup.getLength(), + group.getLength())) + .countAssert(); return true; } @@ -181,13 +205,20 @@ public final class TestRunner { if (exampleMember.isLocalOrExternalOrHidden()) { return true; } + + var exampleName = + group.getLength() == 1 ? testName : testName + " #" + exampleIndex; + Object exampleValue; try { exampleValue = VmUtils.readMember(group, exampleIndex); } catch (VmException err) { errored.set(true); - result.addError( - new Error(err.getMessage(), err.toPklException(stackFrameTransformer))); + results + .newResult( + exampleName, + new Error(err.getMessage(), err.toPklException(stackFrameTransformer))) + .countAssert(); groupSucceeded.set(false); return true; } @@ -222,13 +253,18 @@ public final class TestRunner { .build(); } - result.addFailure( - Failure.buildExampleFailure( - getDisplayUri(exampleMember), - getDisplayUri(expectedMember), - expectedValuePcf, - getDisplayUri(actualMember), - exampleValuePcf)); + results + .newResult( + exampleName, + Failure.buildExampleFailure( + getDisplayUri(exampleMember), + getDisplayUri(expectedMember), + expectedValuePcf, + getDisplayUri(actualMember), + exampleValuePcf)) + .countAssert(); + } else { + results.newResult(exampleName).countAssert(); } return true; @@ -247,12 +283,14 @@ public final class TestRunner { return true; } if (examples.getCachedValue(groupKey) == null) { + var testName = String.valueOf(groupKey); allGroupsSucceeded.set(false); results - .newResult(String.valueOf(groupKey)) - .addFailure( + .newResult( + testName, Failure.buildExamplePropertyMismatchFailure( - getDisplayUri(groupMember), String.valueOf(groupKey), false)); + getDisplayUri(groupMember), testName, false)) + .countAssert(); } return true; }); @@ -262,10 +300,12 @@ public final class TestRunner { } } - private void doRunAndWriteExamples(VmMapping examples, Path outputFile, TestResults results) { + private void doRunAndWriteExamples( + VmMapping examples, Path outputFile, TestSectionResults results) { var allSucceeded = examples.forceAndIterateMemberValues( (groupKey, groupMember, groupValue) -> { + var testName = String.valueOf(groupKey); var listing = (VmListing) groupValue; var success = listing.iterateMembers( @@ -273,22 +313,29 @@ public final class TestRunner { if (member.isLocalOrExternalOrHidden()) { return true; } + + var exampleName = + listing.getLength() == 1 ? testName : testName + " #" + idx; + try { VmUtils.readMember(listing, idx); return true; } catch (VmException err) { results - .newResult(String.valueOf(groupKey)) - .addError( + .newResult( + exampleName, new Error( - err.getMessage(), err.toPklException(stackFrameTransformer))); + err.getMessage(), err.toPklException(stackFrameTransformer))) + .countAssert(); return false; } }); if (!success) { return false; } - results.newResult(String.valueOf(groupKey)).setExampleWritten(true); + var result = results.newResult(testName); + result.countAssert(); + result.setExampleWritten(true); return true; }); if (allSucceeded) { diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java index 98106798..65a4addc 100644 --- a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java @@ -24,7 +24,9 @@ import org.pkl.core.ast.member.ObjectMember; import org.pkl.core.runtime.BaseModule; import org.pkl.core.runtime.Identifier; import org.pkl.core.runtime.TestResults; -import org.pkl.core.runtime.TestResults.TestResult; +import org.pkl.core.runtime.TestResults.TestSectionResults; +import org.pkl.core.runtime.TestResults.TestSectionResults.Error; +import org.pkl.core.runtime.TestResults.TestSectionResults.TestResult; import org.pkl.core.runtime.VmDynamic; import org.pkl.core.runtime.VmMapping; import org.pkl.core.runtime.VmTyped; @@ -41,25 +43,45 @@ public final class JUnitReport implements TestReport { writer.append(renderXML(" ", "1.0", buildSuite(results))); } - private VmDynamic buildSuite(TestResults res) { - var testCases = testCases(res); - if (!res.getErr().isBlank()) { + private VmDynamic buildSuite(TestResults results) { + var testCases = testCases(results.moduleName, results.facts); + testCases.addAll(testCases(results.moduleName, results.examples)); + + if (!results.getErr().isBlank()) { var err = buildXmlElement( "system-err", VmMapping.empty(), - members -> members.put("body", syntheticElement(makeCdata(res.getErr())))); + members -> members.put("body", syntheticElement(makeCdata(results.getErr())))); testCases.add(err); } - return buildXmlElement( - "testsuite", buildRootAttributes(res), testCases.toArray(new VmDynamic[0])); + + var attrs = + buildAttributes( + "name", results.moduleName, + "tests", (long) results.totalTests(), + "failures", (long) results.totalFailures()); + + return buildXmlElement("testsuite", attrs, testCases.toArray(new VmDynamic[0])); } - private ArrayList testCases(TestResults results) { - var className = results.getModuleName(); - var elements = new ArrayList(results.totalTests()); - for (var res : results.getResults()) { - var attrs = buildAttributes("classname", className, "name", res.getName()); + private ArrayList testCases(String moduleName, TestSectionResults testSectionResults) { + var elements = new ArrayList(testSectionResults.totalTests()); + + if (testSectionResults.hasError()) { + var error = error(testSectionResults.getError()); + + var attrs = + buildAttributes("classname", moduleName + "." + testSectionResults.name, "name", "error"); + var element = buildXmlElement("testcase", attrs, error.toArray(new VmDynamic[0])); + + elements.add(element); + } + + for (var res : testSectionResults.getResults()) { + var attrs = + buildAttributes( + "classname", moduleName + "." + testSectionResults.name, "name", res.name); var failures = failures(res); failures.addAll(errors(res)); var element = buildXmlElement("testcase", attrs, failures.toArray(new VmDynamic[0])); @@ -99,6 +121,17 @@ public final class JUnitReport implements TestReport { return list; } + private ArrayList error(Error error) { + var list = new ArrayList(); + var attrs = buildAttributes("message", error.getMessage()); + list.add( + buildXmlElement( + "error", + attrs, + members -> members.put(1, syntheticElement("\n" + error.getRendered())))); + return list; + } + private VmDynamic buildXmlElement(String name, VmMapping attributes, VmDynamic... elements) { return buildXmlElement( name, @@ -130,16 +163,6 @@ public final class JUnitReport implements TestReport { members.size() - 4); } - private VmMapping buildRootAttributes(TestResults results) { - return buildAttributes( - "name", - results.getModuleName(), - "tests", - (long) results.totalTests(), - "failures", - (long) results.totalFailures()); - } - private VmMapping buildAttributes(Object... attributes) { EconomicMap attrs = EconomicMaps.create(attributes.length); for (int i = 0; i < attributes.length; i += 2) { diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java index 7ea290fd..1a1390cd 100644 --- a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java @@ -19,8 +19,8 @@ import java.io.IOException; import java.io.Writer; import java.util.stream.Collectors; import org.pkl.core.runtime.TestResults; -import org.pkl.core.runtime.TestResults.Failure; -import org.pkl.core.runtime.TestResults.TestResult; +import org.pkl.core.runtime.TestResults.TestSectionResults; +import org.pkl.core.runtime.TestResults.TestSectionResults.TestResult; import org.pkl.core.util.StringUtils; public final class SimpleReport implements TestReport { @@ -28,38 +28,66 @@ public final class SimpleReport implements TestReport { @Override public void report(TestResults results, Writer writer) throws IOException { var builder = new StringBuilder(); - builder.append("module "); - builder.append(results.getModuleName()); - builder.append(" (").append(results.getDisplayUri()).append(")\n"); - StringUtils.joinToStringBuilder( - builder, results.getResults(), "\n", res -> reportResult(res, builder)); + + builder.append("module ").append(results.moduleName).append("\n"); + + reportResults(results.facts, builder); + reportResults(results.examples, builder); + + builder.append(results.failed() ? "❌ " : "✅ "); + + var totalStatsLine = + makeStatsLine("tests", results.totalTests(), results.totalFailures(), results.failed()); + builder.append(totalStatsLine); + + var totalAssertsStatsLine = + makeStatsLine( + "asserts", results.totalAsserts(), results.totalAssertsFailed(), results.failed()); + builder.append(", ").append(totalAssertsStatsLine); + builder.append("\n"); + writer.append(builder); } - private void reportResult(TestResult result, StringBuilder builder) { - builder.append(" ").append(result.getName()); - if (result.isExampleWritten()) { - builder.append(" ✍️"); - } else if (result.isSuccess()) { - builder.append(" ✅"); - } else { - builder.append(" ❌\n"); + private void reportResults(TestSectionResults section, StringBuilder builder) { + if (!section.getResults().isEmpty()) { + builder.append(" ").append(section.name).append("\n"); + StringUtils.joinToStringBuilder( - builder, result.getFailures(), "\n", failure -> reportFailure(failure, builder)); - StringUtils.joinToStringBuilder( - builder, - result.getErrors(), - "\n", - error -> { - builder.append(" Error:\n"); - appendPadded(builder, error.getException().getMessage(), " "); - }); + builder, section.getResults(), "\n", res -> reportResult(res, builder)); + builder.append("\n"); + } else if (section.hasError()) { + builder.append(" ").append(section.name).append("\n"); + var error = section.getError().getRendered(); + appendPadded(builder, error, " "); + builder.append("\n"); } } - public static void reportFailure(Failure failure, StringBuilder builder) { - appendPadded(builder, failure.getRendered(), " "); + private void reportResult(TestResult result, StringBuilder builder) { + builder.append(" "); + + if (result.isExampleWritten()) { + builder.append(result.name).append(" ✍️"); + } else { + builder.append(result.isFailure() ? "❌ " : "✅ ").append(result.name); + + if (result.isFailure()) { + var failurePadding = " "; + builder.append("\n"); + StringUtils.joinToStringBuilder( + builder, + result.getFailures(), + "\n", + failure -> appendPadded(builder, failure.getRendered(), failurePadding)); + StringUtils.joinToStringBuilder( + builder, + result.getErrors(), + "\n", + error -> appendPadded(builder, error.getException().getMessage(), failurePadding)); + } + } } private static void appendPadded(StringBuilder builder, String lines, String padding) { @@ -67,6 +95,23 @@ public final class SimpleReport implements TestReport { builder, lines.lines().collect(Collectors.toList()), "\n", - str -> builder.append(padding).append(str)); + str -> { + if (str.length() > 0) builder.append(padding).append(str); + }); + } + + private String makeStatsLine(String kind, int total, int failed, boolean isFailed) { + var passed = total - failed; + var pct_passed = total > 0 ? 100.0 * passed / total : 0.0; + + String line = String.format("%.1f%% %s pass", pct_passed, kind); + + if (isFailed) { + line += String.format(" [%d/%d failed]", failed, total); + } else { + line += String.format(" [%d passed]", passed); + } + + return line; } } diff --git a/pkl-core/src/test/kotlin/org/pkl/core/EvaluateTestsTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/EvaluateTestsTest.kt index 2f55df2e..4c190b99 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/EvaluateTestsTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/EvaluateTestsTest.kt @@ -54,7 +54,7 @@ class EvaluateTestsTest { assertThat(results.displayUri).isEqualTo("repl:text") assertThat(results.totalTests()).isEqualTo(1) assertThat(results.failed()).isFalse - assertThat(results.results[0].name).isEqualTo("should pass") + assertThat(results.facts.results[0].name).isEqualTo("should pass") assertThat(results.err.isBlank()).isTrue } @@ -79,18 +79,19 @@ class EvaluateTestsTest { ) assertThat(results.totalTests()).isEqualTo(1) - assertThat(results.totalFailures()).isEqualTo(2) + assertThat(results.totalFailures()).isEqualTo(1) assertThat(results.failed()).isTrue - val res = results.results[0] + val res = results.facts.results[0] assertThat(res.name).isEqualTo("should fail") - assertThat(res.errors).isEmpty() + assertThat(results.facts.hasError()).isFalse + assertThat(res.failures.size).isEqualTo(2) val fail1 = res.failures[0] - assertThat(fail1.rendered).isEqualTo("1 == 2 ❌ (repl:text)") + assertThat(fail1.rendered).isEqualTo("1 == 2 (repl:text)") val fail2 = res.failures[1] - assertThat(fail2.rendered).isEqualTo(""""foo" == "bar" ❌ (repl:text)""") + assertThat(fail2.rendered).isEqualTo(""""foo" == "bar" (repl:text)""") } @Test @@ -117,7 +118,7 @@ class EvaluateTestsTest { assertThat(results.totalFailures()).isEqualTo(1) assertThat(results.failed()).isTrue - val res = results.results[0] + val res = results.facts.results[0] assertThat(res.name).isEqualTo("should fail") assertThat(res.failures).hasSize(1) assertThat(res.errors).hasSize(1) @@ -179,7 +180,121 @@ class EvaluateTestsTest { assertThat(results.displayUri).startsWith("file:///").endsWith(".pkl") assertThat(results.totalTests()).isEqualTo(1) assertThat(results.failed()).isFalse - assertThat(results.results[0].name).isEqualTo("user") + assertThat(results.examples.results[0].name).isEqualTo("user") + } + + @Test + fun `test fact failures with successful example`(@TempDir tempDir: Path) { + val file = tempDir.createTempFile(prefix = "example", suffix = ".pkl") + Files.writeString( + file, + """ + amends "pkl:test" + + facts { + ["should fail"] { + 1 == 2 + "foo" == "bar" + } + } + + examples { + ["user"] { + new { + name = "Bob" + age = 33 + } + } + } + """ + .trimIndent() + ) + + Files.writeString( + createExpected(file), + """ + examples { + ["user"] { + new { + name = "Bob" + age = 33 + } + } + } + """ + .trimIndent() + ) + + val results = evaluator.evaluateTest(path(file), false) + assertThat(results.moduleName).startsWith("example") + assertThat(results.displayUri).startsWith("file:///").endsWith(".pkl") + assertThat(results.totalTests()).isEqualTo(2) + assertThat(results.totalFailures()).isEqualTo(1) + assertThat(results.failed()).isTrue + + assertThat(results.facts.results[0].name).isEqualTo("should fail") + assertThat(results.facts.results[0].failures.size).isEqualTo(2) + assertThat(results.examples.results[0].name).isEqualTo("user") + } + + @Test + fun `test fact error with successful example`(@TempDir tempDir: Path) { + val file = tempDir.createTempFile(prefix = "example", suffix = ".pkl") + Files.writeString( + file, + """ + amends "pkl:test" + + facts { + ["should fail"] { + throw("exception") + } + } + + examples { + ["user"] { + new { + name = "Bob" + age = 33 + } + } + } + """ + .trimIndent() + ) + + Files.writeString( + createExpected(file), + """ + examples { + ["user"] { + new { + name = "Bob" + age = 33 + } + } + } + """ + .trimIndent() + ) + + val results = evaluator.evaluateTest(path(file), false) + assertThat(results.moduleName).startsWith("example") + assertThat(results.displayUri).startsWith("file:///").endsWith(".pkl") + + assertThat(results.totalTests()).isEqualTo(2) + assertThat(results.totalFailures()).isEqualTo(1) + assertThat(results.failed()).isTrue + + val res = results.facts.results[0] + assertThat(res.name).isEqualTo("should fail") + assertThat(res.failures).hasSize(0) + assertThat(res.errors).hasSize(1) + + val error = res.errors[0] + assertThat(error.message).isEqualTo("exception") + + assertThat(results.examples.results[0].name).isEqualTo("user") } @Test @@ -224,9 +339,9 @@ class EvaluateTestsTest { assertThat(results.failed()).isTrue assertThat(results.totalFailures()).isEqualTo(1) - val res = results.results[0] + val res = results.examples.results[0] assertThat(res.name).isEqualTo("user") - assertThat(res.errors.isEmpty()).isTrue + assertFalse(results.examples.hasError()) val fail1 = res.failures[0] assertThat(fail1.rendered.stripFileAndLines(tempDir)) diff --git a/pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt b/pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt index c39f93a0..468736e9 100644 --- a/pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt +++ b/pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt @@ -30,7 +30,7 @@ class TestsTest : AbstractTest() { writePklFile() val res = runTask("evalTest") - assertThat(res.output).contains("should pass ✅") + assertThat(res.output).contains("✅ should pass") } @Test @@ -49,9 +49,9 @@ class TestsTest : AbstractTest() { ) val res = runTask("evalTest", expectFailure = true) - assertThat(res.output).contains("should fail ❌") - assertThat(res.output).contains("1 == 3 ❌") - assertThat(res.output).contains(""""foo" == "bar" ❌""") + assertThat(res.output).contains("❌ should fail") + assertThat(res.output).contains("1 == 3") + assertThat(res.output).contains(""""foo" == "bar"""") } @Test @@ -68,22 +68,28 @@ class TestsTest : AbstractTest() { .trimIndent() ) - val output = runTask("evalTest", expectFailure = true).output.stripFilesAndLines() + val output = + runTask("evalTest", expectFailure = true) + .output + .stripFilesAndLines() + .lineSequence() + .joinToString("\n") assertThat(output) .containsIgnoringNewLines( """ > Task :evalTest FAILED - module test (file:///file, line x) - should pass ✅ - error ❌ - Error: - –– Pkl Error –– - exception - - 9 | throw("exception") - ^^^^^^^^^^^^^^^^^^ - at test#facts["error"][#1] (file:///file, line x) + module test + facts + ✅ should pass + ❌ error + –– Pkl Error –– + exception + + 9 | throw("exception") + ^^^^^^^^^^^^^^^^^^ + at test#facts["error"][#1] (file:///file, line x) + ❌ 50.0% tests pass [1/2 failed], 66.7% asserts pass [1/3 failed] """ .trimIndent() ) @@ -96,41 +102,41 @@ class TestsTest : AbstractTest() { writeBuildFile() - val output = runTask("evalTest", expectFailure = true).output.stripFilesAndLines() + val output = + runTask("evalTest", expectFailure = true) + .output + .stripFilesAndLines() + .lineSequence() + .joinToString("\n") assertThat(output.trimStart()) - .contains( + .startsWith( """ - module test (file:///file, line x) - sum numbers ✅ - divide numbers ✅ - fail ❌ - 4 == 9 ❌ (file:///file, line x) - "foo" == "bar" ❌ (file:///file, line x) - user 0 ✅ - user 1 ❌ - (file:///file, line x) - Expected: (file:///file, line x) - new { - name = "Pigeon" - age = 40 - } - Actual: (file:///file, line x) - new { - name = "Pigeon" - age = 41 - } - (file:///file, line x) - Expected: (file:///file, line x) - new { - name = "Parrot" - age = 35 - } - Actual: (file:///file, line x) - new { - name = "Welma" - age = 35 - } + > Task :evalTest FAILED + pkl: TRACE: 8 = 8 (file:///file, line x) + module test + facts + ✅ sum numbers + ✅ divide numbers + ❌ fail + 4 == 9 (file:///file, line x) + "foo" == "bar" (file:///file, line x) + examples + ✅ user 0 + ✅ user 1 #0 + ❌ user 1 #1 + (file:///file, line x) + Expected: (file:///file, line x) + new { + name = "Parrot" + age = 35 + } + Actual: (file:///file, line x) + new { + name = "Welma" + age = 35 + } + ❌ 66.7% tests pass [2/6 failed], 66.7% asserts pass [3/9 failed] """ .trimIndent() ) @@ -138,28 +144,7 @@ class TestsTest : AbstractTest() { @Test fun `overwrite expected examples`() { - writePklFile( - additionalExamples = - """ - ["user 0"] { - new { - name = "Cool" - age = 11 - } - } - ["user 1"] { - new { - name = "Pigeon" - age = 41 - } - new { - name = "Welma" - age = 35 - } - } - """ - .trimIndent() - ) + writePklFile(additionalExamples = examples) writeFile("test.pkl-expected.pcf", bigTestExpected) writeBuildFile("overwrite = true") @@ -170,6 +155,64 @@ class TestsTest : AbstractTest() { assertThat(output).contains("user 1 ✍️") } + @Test + fun `full example with error`() { + writeBuildFile() + + writePklFile( + additionalFacts = + """ + ["error"] { + throw("exception") + } + """ + .trimIndent(), + additionalExamples = examples + ) + writeFile("test.pkl-expected.pcf", bigTestExpected) + + val output = + runTask("evalTest", expectFailure = true) + .output + .stripFilesAndLines() + .lineSequence() + .joinToString("\n") + + assertThat(output.trimStart()) + .startsWith( + """ + > Task :evalTest FAILED + module test + facts + ✅ should pass + ❌ error + –– Pkl Error –– + exception + + 9 | throw("exception") + ^^^^^^^^^^^^^^^^^^ + at test#facts["error"][#1] (file:///file, line x) + examples + ✅ user 0 + ✅ user 1 #0 + ❌ user 1 #1 + (file:///file, line x) + Expected: (file:///file, line x) + new { + name = "Parrot" + age = 35 + } + Actual: (file:///file, line x) + new { + name = "Welma" + age = 35 + } + ❌ 60.0% tests pass [2/5 failed], 66.7% asserts pass [2/6 failed] + """ + .trimIndent() + ) + } + @Test fun `JUnit reports`() { val pklFile = writePklFile(contents = bigTest) @@ -186,26 +229,16 @@ class TestsTest : AbstractTest() { .isEqualTo( """ - - - - - 4 == 9 ❌ (file:///file, line x) - "foo" == "bar" ❌ (file:///file, line x) + + + + + 4 == 9 (file:///file, line x) + "foo" == "bar" (file:///file, line x) - - - (file:///file, line x) - Expected: (file:///file, line x) - new { - name = "Pigeon" - age = 40 - } - Actual: (file:///file, line x) - new { - name = "Pigeon" - age = 41 - } + + + (file:///file, line x) Expected: (file:///file, line x) new { @@ -227,6 +260,86 @@ class TestsTest : AbstractTest() { ) } + @Test + fun `JUnit reports with error`() { + val pklFile = + writePklFile( + additionalFacts = + """ + ["error"] { + throw("exception") + } + """ + .trimIndent(), + additionalExamples = examples + ) + writeFile("test.pkl-expected.pcf", bigTestExpected) + + writeBuildFile("junitReportsDir = file('${pklFile.parent.toNormalizedPathString()}/build')") + + runTask("evalTest", expectFailure = true) + + val outputFile = testProjectDir.resolve("build/test.xml") + val report = outputFile.readText().stripFilesAndLines() + + assertThat(report) + .isEqualTo( + """ + + + + + –– Pkl Error –– + exception + + 9 | throw("exception") + ^^^^^^^^^^^^^^^^^^ + at test#facts["error"][#1] (file:///file, line x) + + + + + + (file:///file, line x) + Expected: (file:///file, line x) + new { + name = "Parrot" + age = 35 + } + Actual: (file:///file, line x) + new { + name = "Welma" + age = 35 + } + + + + """ + .trimIndent() + ) + } + + private val examples = + """ + ["user 0"] { + new { + name = "Cool" + age = 11 + } + } + ["user 1"] { + new { + name = "Pigeon" + age = 40 + } + new { + name = "Welma" + age = 35 + } + } + """ + .trimIndent() + private val bigTest = """ amends "pkl:test" @@ -249,22 +362,7 @@ class TestsTest : AbstractTest() { } examples { - ["user 0"] { - new { - name = "Cool" - age = 11 - } - } - ["user 1"] { - new { - name = "Pigeon" - age = 41 - } - new { - name = "Welma" - age = 35 - } - } + $examples } """ .trimIndent()