starred/pkl
1
0
Fork 0
mirror of https://github.com/apple/pkl.git synced 2026-04-22 16:28:34 +02:00
Code Issues 198 Packages Projects Releases 10 Wiki Activity

Polish test result running and reporting (#738)

Browse Source 
Changes:
* Move class `TestResults` to package `org.pkl.core`, because it is a public class (it's the result of `Evaluator#evaluateTest`)
* Change examples to treat individual examples as assertions in the same test. Previously, they were considered different tests with an incrementing number. This better aligns with how facts are treated.
* Change `TestResults` to be a record, and introduce builders.
* Remove "module" test result section (it is not really a section).
* Add javadoc to `TestResults`
* Formatting fix: prefix ✍️ emoji just like we do the ❌ and ✅ emojis 
* Consider writing examples as failures, not successes. `pkl test` will exit with code 10 if the only failing tests are due to writing examples.
This commit is contained in:
Daniel Chao
2024-10-28 21:05:13 -07:00
committed by GitHub
parent 666f8c3939
commit acd2222534
15 changed files with 851 additions and 708 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
pkl-core/src/main/java/org/pkl/core/Evaluator.java
View File

@@ -16,7 +16,6 @@
package org.pkl.core;
import java.util.Map;
import org.pkl.core.runtime.TestResults;
import org.pkl.core.runtime.VmEvalException;
/**
@@ -194,9 +193,9 @@ public interface Evaluator extends AutoCloseable {
* <p>This requires that the target module be a test module; it must either amend or extend module
* {@code "pkl:test"}. Otherwise, a type mismatch error is thrown.
*
* <p>This method will write possibly {@code pcf-expected.pkl} and {@code pcf-actual.pcf} files as
* <p>This method will write possibly {@code pkl-expected.pcf} and {@code pkl-actual.pcf} files as
* a sibling of the test module. The {@code overwrite} parameter causes the evaluator to overwrite
* {@code pcf-expected.pkl} files if they currently exist.
* {@code pkl-expected.pkl} files if they currently exist.
*
* @throws PklException if an error occurs during evaluation
* @throws IllegalStateException if this evaluator has already been closed

1
pkl-core/src/main/java/org/pkl/core/EvaluatorImpl.java
View File

@@ -40,7 +40,6 @@ import org.pkl.core.runtime.BaseModule;
import org.pkl.core.runtime.Identifier;
import org.pkl.core.runtime.ModuleResolver;
import org.pkl.core.runtime.ResourceManager;
import org.pkl.core.runtime.TestResults;
import org.pkl.core.runtime.TestRunner;
import org.pkl.core.runtime.VmContext;
import org.pkl.core.runtime.VmException;

301
pkl-core/src/main/java/org/pkl/core/TestResults.java Normal file
View File

@@ -0,0 +1,301 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core;
import java.util.ArrayList;
import java.util.List;
import org.pkl.core.util.Nullable;
/**
* The results of testing a Pkl test module.
*
* <p>A test module is a module that amends {@code pkl:test}, and is evaluated with {@link
* Evaluator#evaluateTest(ModuleSource, boolean)}.
*
* <p>Test results have two sections; facts, and examples. Each section has multiple individual
* results, which themselves contain individual assertions.
*
* @since 0.27.0
* @param moduleName The name of the module that was tested.
* @param displayUri A location URI formatted to be displayed.
* @param facts The result of testing facts.
* @param examples The result of testing examples.
* @param logs The log output resulting from running the test.
* @param error An error that arose from evaluating the test module itself.
* <p>If non-null, {@code facts} and {@code examples} are guaranteed to have 0 results.
*/
@SuppressWarnings("UnusedReturnValue")
public record TestResults(
String moduleName,
String displayUri,
TestSectionResults facts,
TestSectionResults examples,
String logs,
@Nullable Error error) {
/** The total number of tests between facts and examples. */
public int totalTests() {
return facts.totalTests() + examples.totalTests();
}
/**
* The total number of failed {@linkplain TestResult test results}.
*
* <p>A {@link TestResult} has failed if any of its assertions have a {@link Failure} or an {@link
* Error}.
*/
public int totalFailures() {
return facts.totalFailures() + examples.totalFailures();
}
/** The total number of assertions between facts and examples. */
public int totalAsserts() {
return facts.totalAsserts() + examples.totalAsserts();
}
/**
* The total number of individual assertions that have failed.
*
* <p>A single test can have multiple failed assertions.
*/
public int totalAssertsFailed() {
return facts.totalAssertsFailedOrErrored() + examples.totalAssertsFailedOrErrored();
}
/**
* Whether the test in aggregate has failed or not.
*
* <p>An individual {@link TestResult} has failed if any of its assertions have a {@link Failure}
* or an {@link Error}.
*/
public boolean failed() {
return error != null || facts.failed() || examples.failed();
}
/**
* Whether the test result has failed due to examples being written.
*
* <p>Returns {@code true} if and only if there are failures, and all failures are due to examples
* being written.
*/
public boolean isExampleWrittenFailure() {
if (!failed() || !examples.failed()) return false;
for (var testResult : examples.results) {
if (!testResult.isExampleWritten) {
return false;
}
}
return true;
}
public static class Builder {
private final String moduleName;
private final String displayUri;
private TestSectionResults factsSection =
new TestSectionResults(TestSectionName.FACTS, List.of());
private TestSectionResults examplesSection =
new TestSectionResults(TestSectionName.EXAMPLES, List.of());
private String stdErr = "";
private @Nullable Error error = null;
public Builder(String moduleName, String displayUri) {
this.moduleName = moduleName;
this.displayUri = displayUri;
}
public Builder setError(Error error) {
this.error = error;
return this;
}
public Builder setStdErr(String stdErr) {
this.stdErr = stdErr;
return this;
}
public Builder setFactsSection(TestSectionResults factsSection) {
assert factsSection.name() == TestSectionName.FACTS;
this.factsSection = factsSection;
return this;
}
public Builder setExamplesSection(TestSectionResults examplesSection) {
assert examplesSection.name() == TestSectionName.EXAMPLES;
this.examplesSection = examplesSection;
return this;
}
public TestResults build() {
return new TestResults(moduleName, displayUri, factsSection, examplesSection, stdErr, error);
}
}
public enum TestSectionName {
FACTS("facts"),
EXAMPLES("examples");
private final String name;
TestSectionName(final String name) {
this.name = name;
}
@Override
public String toString() {
return name;
}
}
/**
* The results of either facts or examples.
*
* @param name The name of this section
* @param results The results of the individual tests in this section.
*/
public record TestSectionResults(TestSectionName name, List<TestResult> results) {
public int totalTests() {
return results.size();
}
public int totalAsserts() {
var total = 0;
for (var res : results) {
total += res.totalAsserts();
}
return total;
}
public int totalAssertsFailedOrErrored() {
var total = 0;
for (var res : results) {
total += res.totalAssertsFailedOrErrored();
}
return total;
}
public int totalFailures() {
var total = 0;
for (var res : results) {
if (res.isFailure()) {
total++;
}
}
return total;
}
public boolean failed() {
for (var res : results) {
if (res.isFailure()) {
return true;
}
}
return false;
}
public boolean hasError() {
for (var res : results) {
if (res.hasErrors()) {
return true;
}
}
return false;
}
}
/**
* The result of a single test.
*
* <p>A test can have multiple assertions, where the assertion itself can have failed, or have
* thrown an exception.
*
* @param name The name of the test.
* @param totalAsserts The total number of assertions in the test.
* @param failures The number of assertion failures in the test.
* @param errors The number of errors that were thrown when the test was run.
* @param isExampleWritten Whether the test is considered as having its example written or not.
*/
public record TestResult(
String name,
int totalAsserts,
List<Failure> failures,
List<Error> errors,
boolean isExampleWritten) {
public int totalAssertsFailedOrErrored() {
return failures().size() + errors.size();
}
public boolean isFailure() {
return totalAssertsFailedOrErrored() > 0;
}
public boolean hasErrors() {
return !errors().isEmpty();
}
public boolean hasFailures() {
return !failures.isEmpty();
}
public static class Builder {
private final String name;
private final List<Failure> failures = new ArrayList<>();
private final List<Error> errors = new ArrayList<>();
private boolean isExampleWritten;
private int count = 0;
public Builder(String name) {
this.name = name;
}
public Builder addSuccess() {
count++;
return this;
}
public int getCount() {
return count;
}
public Builder addFailure(Failure failure) {
this.failures.add(failure);
count++;
return this;
}
public Builder addError(Error error) {
this.errors.add(error);
count++;
return this;
}
public Builder setExampleWritten(boolean exampleWritten) {
this.isExampleWritten = exampleWritten;
return this;
}
public TestResult build() {
return new TestResult(name, count, failures, errors, isExampleWritten);
}
}
}
public record Error(String message, PklException exception) {}
public record Failure(String kind, String message) {}
}

358
pkl-core/src/main/java/org/pkl/core/runtime/TestResults.java
View File

@@ -1,358 +0,0 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.runtime;
import com.oracle.truffle.api.source.SourceSection;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.pkl.core.PklException;
import org.pkl.core.runtime.TestResults.TestSectionResults.TestSection;
/** Aggregate test results for a module. Used to verify test failures and generate reports. */
public final class TestResults {
public final String moduleName;
public final String displayUri;
public final TestSectionResults module = new TestSectionResults(TestSection.MODULE);
public final TestSectionResults facts = new TestSectionResults(TestSection.FACTS);
public final TestSectionResults examples = new TestSectionResults(TestSection.EXAMPLES);
private String err = "";
public TestResults(String moduleName, String displayUri) {
this.moduleName = moduleName;
this.displayUri = displayUri;
}
public int totalTests() {
return module.totalTests() + facts.totalTests() + examples.totalTests();
}
public int totalFailures() {
return module.totalFailures() + facts.totalFailures() + examples.totalFailures();
}
public int totalAsserts() {
return module.totalAsserts() + facts.totalAsserts() + examples.totalAsserts();
}
public int totalAssertsFailed() {
return module.totalAssertsFailed() + facts.totalAssertsFailed() + examples.totalAssertsFailed();
}
public boolean failed() {
return module.failed() || facts.failed() || examples.failed();
}
public String getErr() {
return err;
}
public void setErr(String err) {
this.err = err;
}
public static class TestSectionResults {
public final TestSection name;
private final List<TestResult> results = new ArrayList<>();
private Error error;
public TestSectionResults(TestSection name) {
this.name = name;
}
public void setError(Error error) {
this.error = error;
}
public Error getError() {
return error;
}
public boolean hasError() {
return error != null;
}
public List<TestResult> getResults() {
return Collections.unmodifiableList(results);
}
public TestResult newResult(String name) {
var result = new TestResult(name, this.name == TestSection.EXAMPLES);
results.add(result);
return result;
}
public TestResult newResult(String name, Failure failure) {
var result = new TestResult(name, this.name == TestSection.EXAMPLES);
result.addFailure(failure);
results.add(result);
return result;
}
public TestResult newResult(String name, Error error) {
var result = new TestResult(name, this.name == TestSection.EXAMPLES);
result.addError(error);
results.add(result);
return result;
}
public int totalTests() {
var total = results.size();
return (hasError() ? ++total : total);
}
public int totalAsserts() {
int total = 0;
for (var res : results) {
total += res.totalAsserts();
}
return (hasError() ? ++total : total);
}
public int totalAssertsFailed() {
int total = 0;
for (var res : results) {
total += res.totalAssertsFailed();
}
return (hasError() ? ++total : total);
}
public int totalFailures() {
int total = 0;
for (var res : results) {
if (res.isFailure()) total++;
}
return (hasError() ? ++total : total);
}
public boolean failed() {
if (hasError()) return true;
for (var res : results) {
if (res.isFailure()) return true;
}
return false;
}
public static class TestResult {
public final String name;
private int totalAsserts = 0;
private int totalAssertsFailed = 0;
private final List<Failure> failures = new ArrayList<>();
private final List<Error> errors = new ArrayList<>();
public final boolean isExample;
private boolean isExampleWritten = false;
public TestResult(String name, boolean isExample) {
this.name = name;
this.isExample = isExample;
}
public boolean isSuccess() {
return failures.isEmpty() && errors.isEmpty();
}
public boolean isFailure() {
return !isSuccess();
}
public boolean isExampleWritten() {
return isExampleWritten;
}
public void setExampleWritten(boolean exampleWritten) {
isExampleWritten = exampleWritten;
}
public List<Failure> getFailures() {
return Collections.unmodifiableList(failures);
}
public void addFailure(Failure description) {
failures.add(description);
totalAssertsFailed++;
}
public List<Error> getErrors() {
return Collections.unmodifiableList(errors);
}
public void addError(Error err) {
errors.add(err);
totalAssertsFailed++;
}
public int totalAsserts() {
return totalAsserts;
}
public void countAssert() {
totalAsserts++;
}
public int totalAssertsFailed() {
return totalAssertsFailed;
}
}
public static class Failure {
private final String kind;
private final String failure;
private final String location;
private Failure(String kind, String failure, String location) {
this.kind = kind;
this.failure = failure;
this.location = location;
}
public String getKind() {
return kind;
}
public String getFailure() {
return failure;
}
public String getLocation() {
return location;
}
public static String renderLocation(String location) {
return "(" + location + ")";
}
public String getRendered() {
String rendered;
if (kind == "Fact Failure") {
rendered = failure + " " + renderLocation(getLocation());
} else {
rendered = renderLocation(getLocation()) + "\n" + failure;
}
return rendered;
}
public static Failure buildFactFailure(SourceSection sourceSection, String location) {
return new Failure("Fact Failure", sourceSection.getCharacters().toString(), location);
}
public static Failure buildExampleLengthMismatchFailure(
String location, String property, int expectedLength, int actualLength) {
var builder = new StringBuilder();
builder
.append("Output mismatch: Expected \"")
.append(property)
.append("\" to contain ")
.append(expectedLength)
.append(" examples, but found ")
.append(actualLength);
return new Failure("Output Mismatch (Length)", builder.toString(), location);
}
public static Failure buildExamplePropertyMismatchFailure(
String location, String property, boolean isMissingInExpected) {
String exists_in;
String missing_in;
if (isMissingInExpected) {
exists_in = "actual";
missing_in = "expected";
} else {
exists_in = "expected";
missing_in = "actual";
}
var builder = new StringBuilder();
builder
.append("Output mismatch: \"")
.append(property)
.append("\" exists in ")
.append(exists_in)
.append(" but not in ")
.append(missing_in)
.append(" output");
return new Failure("Output Mismatch", builder.toString(), location);
}
public static Failure buildExampleFailure(
String location,
String expectedLocation,
String expectedValue,
String actualLocation,
String actualValue) {
var builder = new StringBuilder();
builder
.append("Expected: ")
.append(renderLocation(expectedLocation))
.append("\n")
.append(expectedValue)
.append("\n")
.append("Actual: ")
.append(renderLocation(actualLocation))
.append("\n")
.append(actualValue);
return new Failure("Example Failure", builder.toString(), location);
}
}
public static class Error {
private final String message;
private final PklException exception;
public Error(String message, PklException exception) {
this.message = message;
this.exception = exception;
}
public String getMessage() {
return message;
}
public Exception getException() {
return exception;
}
public String getRendered() {
return exception.getMessage();
}
}
public enum TestSection {
MODULE("module"),
FACTS("facts"),
EXAMPLES("examples");
private final String name;
TestSection(final String name) {
this.name = name;
}
@Override
public String toString() {
return name;
}
}
}
}

319
pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java
View File

@@ -15,17 +15,23 @@
*/
package org.pkl.core.runtime;
import com.oracle.truffle.api.source.SourceSection;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.pkl.core.BufferedLogger;
import org.pkl.core.StackFrameTransformer;
import org.pkl.core.TestResults;
import org.pkl.core.TestResults.Failure;
import org.pkl.core.TestResults.TestResult;
import org.pkl.core.TestResults.TestSectionName;
import org.pkl.core.TestResults.TestSectionResults;
import org.pkl.core.ast.member.ObjectMember;
import org.pkl.core.module.ModuleKeys;
import org.pkl.core.runtime.TestResults.TestSectionResults;
import org.pkl.core.runtime.TestResults.TestSectionResults.Error;
import org.pkl.core.runtime.TestResults.TestSectionResults.Failure;
import org.pkl.core.stdlib.PklConverter;
import org.pkl.core.stdlib.base.PcfRenderer;
import org.pkl.core.util.EconomicMaps;
@@ -48,31 +54,20 @@ public final class TestRunner {
public TestResults run(VmTyped testModule) {
var info = VmUtils.getModuleInfo(testModule);
var results = new TestResults(info.getModuleName(), getDisplayUri(info));
var resultsBuilder = new TestResults.Builder(info.getModuleName(), getDisplayUri(info));
try {
checkAmendsPklTest(testModule);
} catch (VmException v) {
var error = new Error(v.getMessage(), v.toPklException(stackFrameTransformer));
results.module.setError(error);
var error = new TestResults.Error(v.getMessage(), v.toPklException(stackFrameTransformer));
return resultsBuilder.setError(error).build();
}
try {
runFacts(testModule, results.facts);
} catch (VmException v) {
var error = new Error(v.getMessage(), v.toPklException(stackFrameTransformer));
results.facts.setError(error);
}
resultsBuilder.setFactsSection(runFacts(testModule));
resultsBuilder.setExamplesSection(runExamples(testModule, info));
try {
runExamples(testModule, info, results.examples);
} catch (VmException v) {
var error = new Error(v.getMessage(), v.toPklException(stackFrameTransformer));
results.examples.setError(error);
}
results.setErr(logger.getLogs());
return results;
resultsBuilder.setStdErr(logger.getLogs());
return resultsBuilder.build();
}
private void checkAmendsPklTest(VmTyped value) {
@@ -86,41 +81,48 @@ public final class TestRunner {
}
}
private void runFacts(VmTyped testModule, TestSectionResults results) {
private TestSectionResults runFacts(VmTyped testModule) {
var facts = VmUtils.readMember(testModule, Identifier.FACTS);
if (facts instanceof VmNull) return;
if (facts instanceof VmNull) return new TestSectionResults(TestSectionName.FACTS, List.of());
var testResults = new ArrayList<TestResult>();
var factsMapping = (VmMapping) facts;
factsMapping.forceAndIterateMemberValues(
(groupKey, groupMember, groupValue) -> {
var listing = (VmListing) groupValue;
var result = results.newResult(String.valueOf(groupKey));
return listing.iterateMembers(
var name = String.valueOf(groupKey);
var resultBuilder = new TestResult.Builder(name);
listing.iterateMembers(
(idx, member) -> {
if (member.isLocalOrExternalOrHidden()) {
return true;
}
result.countAssert();
try {
var factValue = VmUtils.readMember(listing, idx);
if (factValue == Boolean.FALSE) {
result.addFailure(
Failure.buildFactFailure(member.getSourceSection(), getDisplayUri(member)));
var failure = factFailure(member.getSourceSection(), getDisplayUri(member));
resultBuilder.addFailure(failure);
} else {
resultBuilder.addSuccess();
}
} catch (VmException err) {
result.addError(
new Error(err.getMessage(), err.toPklException(stackFrameTransformer)));
var error =
new TestResults.Error(
err.getMessage(), err.toPklException(stackFrameTransformer));
resultBuilder.addError(error);
}
return true;
});
testResults.add(resultBuilder.build());
return true;
});
return new TestSectionResults(TestSectionName.FACTS, Collections.unmodifiableList(testResults));
}
private void runExamples(VmTyped testModule, ModuleInfo info, TestSectionResults results) {
private TestSectionResults runExamples(VmTyped testModule, ModuleInfo info) {
var examples = VmUtils.readMember(testModule, Identifier.EXAMPLES);
if (examples instanceof VmNull) return;
if (examples instanceof VmNull)
return new TestSectionResults(TestSectionName.EXAMPLES, List.of());
var moduleUri = info.getModuleKey().getUri();
if (!moduleUri.getScheme().equalsIgnoreCase("file")) {
@@ -154,72 +156,59 @@ public final class TestRunner {
}
if (Files.exists(expectedOutputFile)) {
doRunAndValidateExamples(examplesMapping, expectedOutputFile, actualOutputFile, results);
return doRunAndValidateExamples(examplesMapping, expectedOutputFile, actualOutputFile);
} else {
doRunAndWriteExamples(examplesMapping, expectedOutputFile, results);
return doRunAndWriteExamples(examplesMapping, expectedOutputFile);
}
}
private void doRunAndValidateExamples(
VmMapping examples,
Path expectedOutputFile,
Path actualOutputFile,
TestSectionResults results) {
private TestSectionResults doRunAndValidateExamples(
VmMapping examples, Path expectedOutputFile, Path actualOutputFile) {
var expectedExampleOutputs = loadExampleOutputs(expectedOutputFile);
var actualExampleOutputs = new MutableReference<VmDynamic>(null);
var allGroupsSucceeded = new MutableBoolean(true);
var errored = new MutableBoolean(false);
var testResults = new ArrayList<TestResult>();
examples.forceAndIterateMemberValues(
(groupKey, groupMember, groupValue) -> {
var testName = String.valueOf(groupKey);
var group = (VmListing) groupValue;
var expectedGroup =
(VmDynamic) VmUtils.readMemberOrNull(expectedExampleOutputs, groupKey);
var testResultBuilder = new TestResult.Builder(testName);
if (expectedGroup == null) {
results
.newResult(
testName,
Failure.buildExamplePropertyMismatchFailure(
getDisplayUri(groupMember), testName, true))
.countAssert();
testResultBuilder.addFailure(
examplePropertyMismatchFailure(getDisplayUri(groupMember), testName, true));
testResults.add(testResultBuilder.build());
return true;
}
if (group.getLength() != expectedGroup.getLength()) {
results
.newResult(
testResultBuilder.addFailure(
exampleLengthMismatchFailure(
getDisplayUri(groupMember),
testName,
Failure.buildExampleLengthMismatchFailure(
getDisplayUri(groupMember),
testName,
expectedGroup.getLength(),
group.getLength()))
.countAssert();
expectedGroup.getLength(),
group.getLength()));
testResults.add(testResultBuilder.build());
return true;
}
var groupSucceeded = new MutableBoolean(true);
group.iterateMembers(
((exampleIndex, exampleMember) -> {
if (exampleMember.isLocalOrExternalOrHidden()) {
return true;
}
var exampleName =
group.getLength() == 1 ? testName : testName + " #" + exampleIndex;
Object exampleValue;
try {
exampleValue = VmUtils.readMember(group, exampleIndex);
} catch (VmException err) {
errored.set(true);
results
.newResult(
exampleName,
new Error(err.getMessage(), err.toPklException(stackFrameTransformer)))
.countAssert();
groupSucceeded.set(false);
testResultBuilder.addError(
new TestResults.Error(
err.getMessage(), err.toPklException(stackFrameTransformer)));
return true;
}
var expectedValue = VmUtils.readMember(expectedGroup, exampleIndex);
@@ -236,8 +225,6 @@ public final class TestRunner {
actualExampleOutputs.set(loadExampleOutputs(actualOutputFile));
}
groupSucceeded.set(false);
var expectedMember = VmUtils.findMember(expectedGroup, exampleIndex);
assert expectedMember != null;
@@ -253,26 +240,22 @@ public final class TestRunner {
.build();
}
results
.newResult(
exampleName,
Failure.buildExampleFailure(
getDisplayUri(exampleMember),
getDisplayUri(expectedMember),
expectedValuePcf,
getDisplayUri(actualMember),
exampleValuePcf))
.countAssert();
testResultBuilder.addFailure(
exampleFailure(
getDisplayUri(exampleMember),
getDisplayUri(expectedMember),
expectedValuePcf,
getDisplayUri(actualMember),
exampleValuePcf,
testResultBuilder.getCount()));
} else {
results.newResult(exampleName).countAssert();
testResultBuilder.addSuccess();
}
return true;
}));
if (!groupSucceeded.get()) {
allGroupsSucceeded.set(false);
}
testResults.add(testResultBuilder.build());
return true;
});
@@ -285,12 +268,12 @@ public final class TestRunner {
if (examples.getCachedValue(groupKey) == null) {
var testName = String.valueOf(groupKey);
allGroupsSucceeded.set(false);
results
.newResult(
testName,
Failure.buildExamplePropertyMismatchFailure(
getDisplayUri(groupMember), testName, false))
.countAssert();
var result =
new TestResult.Builder(testName)
.addFailure(
examplePropertyMismatchFailure(getDisplayUri(groupMember), testName, false))
.build();
testResults.add(result);
}
return true;
});
@@ -298,49 +281,50 @@ public final class TestRunner {
if (!allGroupsSucceeded.get() && actualExampleOutputs.isNull() && !errored.get()) {
writeExampleOutputs(actualOutputFile, examples);
}
return new TestSectionResults(
TestSectionName.EXAMPLES, Collections.unmodifiableList(testResults));
}
private void doRunAndWriteExamples(
VmMapping examples, Path outputFile, TestSectionResults results) {
var allSucceeded =
examples.forceAndIterateMemberValues(
(groupKey, groupMember, groupValue) -> {
var testName = String.valueOf(groupKey);
var listing = (VmListing) groupValue;
var success =
listing.iterateMembers(
(idx, member) -> {
if (member.isLocalOrExternalOrHidden()) {
return true;
}
var exampleName =
listing.getLength() == 1 ? testName : testName + " #" + idx;
try {
VmUtils.readMember(listing, idx);
return true;
} catch (VmException err) {
results
.newResult(
exampleName,
new Error(
err.getMessage(), err.toPklException(stackFrameTransformer)))
.countAssert();
return false;
}
});
if (!success) {
return false;
}
var result = results.newResult(testName);
result.countAssert();
result.setExampleWritten(true);
return true;
});
if (allSucceeded) {
private TestSectionResults doRunAndWriteExamples(VmMapping examples, Path outputFile) {
var testResults = new ArrayList<TestResult>();
var allSucceeded = new MutableBoolean(true);
examples.forceAndIterateMemberValues(
(groupKey, groupMember, groupValue) -> {
var testName = String.valueOf(groupKey);
var listing = (VmListing) groupValue;
var testResultBuilder = new TestResult.Builder(testName);
var success = new MutableBoolean(true);
listing.iterateMembers(
(idx, member) -> {
if (member.isLocalOrExternalOrHidden()) {
return true;
}
try {
VmUtils.readMember(listing, idx);
return true;
} catch (VmException err) {
testResultBuilder.addError(
new TestResults.Error(
err.getMessage(), err.toPklException(stackFrameTransformer)));
allSucceeded.set(false);
success.set(false);
return true;
}
});
if (success.get()) {
// treat writing an example as a message
testResultBuilder.setExampleWritten(true);
testResultBuilder.addFailure(
writtenExampleOutputFailure(testName, getDisplayUri(groupMember)));
}
testResults.add(testResultBuilder.build());
return true;
});
if (allSucceeded.get()) {
writeExampleOutputs(outputFile, examples);
}
return new TestSectionResults(
TestSectionName.EXAMPLES, Collections.unmodifiableList(testResults));
}
private void writeExampleOutputs(Path outputFile, VmMapping examples) {
@@ -402,4 +386,87 @@ public final class TestRunner {
return VmUtils.getDisplayUri(
moduleInfo.getModuleKey().getUri(), VmContext.get(null).getFrameTransformer());
}
private static Failure factFailure(SourceSection sourceSection, String location) {
String message = sourceSection.getCharacters().toString() + " " + renderLocation(location);
return new Failure("Fact Failure", message);
}
private static Failure exampleLengthMismatchFailure(
String location, String property, int expectedLength, int actualLength) {
String msg =
renderLocation(location)
+ "\n"
+ "Output mismatch: Expected \""
+ property
+ "\" to contain "
+ expectedLength
+ " examples, but found "
+ actualLength;
return new Failure("Output Mismatch (Length)", msg);
}
private static Failure examplePropertyMismatchFailure(
String location, String property, boolean isMissingInExpected) {
String existsIn;
String missingIn;
if (isMissingInExpected) {
existsIn = "actual";
missingIn = "expected";
} else {
existsIn = "expected";
missingIn = "actual";
}
String message =
renderLocation(location)
+ "\n"
+ "Output mismatch: \""
+ property
+ "\" exists in "
+ existsIn
+ " but not in "
+ missingIn
+ " output";
return new Failure("Output Mismatch", message);
}
private static Failure exampleFailure(
String location,
String expectedLocation,
String expectedValue,
String actualLocation,
String actualValue,
int exampleNumber) {
String err =
"#"
+ exampleNumber
+ " "
+ renderLocation(location)
+ ":\n "
+ "Expected: "
+ renderLocation(expectedLocation)
+ "\n "
+ expectedValue.replaceAll("\n", "\n ")
+ "\n "
+ "Actual: "
+ renderLocation(actualLocation)
+ "\n "
+ actualValue.replaceAll("\n", "\n ");
return new Failure("Example Failure", err);
}
private static String renderLocation(String location) {
return "(" + location + ")";
}
private static Failure writtenExampleOutputFailure(String testName, String location) {
var message = renderLocation(location) + "\n" + "Wrote expected output for test " + testName;
return new Failure("Example Output Written", message);
}
}

68
pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java
View File

@@ -20,13 +20,13 @@ import java.io.Writer;
import java.util.ArrayList;
import java.util.function.Consumer;
import org.graalvm.collections.EconomicMap;
import org.pkl.core.TestResults;
import org.pkl.core.TestResults.Error;
import org.pkl.core.TestResults.TestResult;
import org.pkl.core.TestResults.TestSectionResults;
import org.pkl.core.ast.member.ObjectMember;
import org.pkl.core.runtime.BaseModule;
import org.pkl.core.runtime.Identifier;
import org.pkl.core.runtime.TestResults;
import org.pkl.core.runtime.TestResults.TestSectionResults;
import org.pkl.core.runtime.TestResults.TestSectionResults.Error;
import org.pkl.core.runtime.TestResults.TestSectionResults.TestResult;
import org.pkl.core.runtime.VmDynamic;
import org.pkl.core.runtime.VmMapping;
import org.pkl.core.runtime.VmTyped;
@@ -44,44 +44,51 @@ public final class JUnitReport implements TestReport {
}
private VmDynamic buildSuite(TestResults results) {
var testCases = testCases(results.moduleName, results.facts);
testCases.addAll(testCases(results.moduleName, results.examples));
if (results.error() != null) {
var testCase = rootTestCase(results, results.error());
var attrs =
buildAttributes(
"name", results.moduleName(),
"tests", 1,
"failures", 1);
return buildXmlElement("testsuite", attrs, testCase);
}
if (!results.getErr().isBlank()) {
var testCases = testCases(results.moduleName(), results.facts());
testCases.addAll(testCases(results.moduleName(), results.examples()));
if (!results.logs().isBlank()) {
var err =
buildXmlElement(
"system-err",
VmMapping.empty(),
members -> members.put("body", syntheticElement(makeCdata(results.getErr()))));
members -> members.put("body", syntheticElement(makeCdata(results.logs()))));
testCases.add(err);
}
var attrs =
buildAttributes(
"name", results.moduleName,
"name", results.moduleName(),
"tests", (long) results.totalTests(),
"failures", (long) results.totalFailures());
return buildXmlElement("testsuite", attrs, testCases.toArray(new VmDynamic[0]));
}
private VmDynamic rootTestCase(TestResults results, TestResults.Error error) {
var testCaseAttrs =
buildAttributes("classname", results.moduleName(), "name", results.moduleName());
var err = error(error);
return buildXmlElement("testcase", testCaseAttrs, err.toArray(new VmDynamic[0]));
}
private ArrayList<VmDynamic> testCases(String moduleName, TestSectionResults testSectionResults) {
var elements = new ArrayList<VmDynamic>(testSectionResults.totalTests());
if (testSectionResults.hasError()) {
var error = error(testSectionResults.getError());
var attrs =
buildAttributes("classname", moduleName + "." + testSectionResults.name, "name", "error");
var element = buildXmlElement("testcase", attrs, error.toArray(new VmDynamic[0]));
elements.add(element);
}
for (var res : testSectionResults.getResults()) {
for (var res : testSectionResults.results()) {
var attrs =
buildAttributes(
"classname", moduleName + "." + testSectionResults.name, "name", res.name);
"classname", moduleName + "." + testSectionResults.name(), "name", res.name());
var failures = failures(res);
failures.addAll(errors(res));
var element = buildXmlElement("testcase", attrs, failures.toArray(new VmDynamic[0]));
@@ -93,14 +100,12 @@ public final class JUnitReport implements TestReport {
private ArrayList<VmDynamic> failures(TestResult res) {
var list = new ArrayList<VmDynamic>();
long i = 0;
for (var fail : res.getFailures()) {
var attrs = buildAttributes("message", fail.getKind());
for (var fail : res.failures()) {
var attrs = buildAttributes("message", fail.kind());
long element = i++;
list.add(
buildXmlElement(
"failure",
attrs,
members -> members.put(element, syntheticElement(fail.getRendered()))));
"failure", attrs, members -> members.put(element, syntheticElement(fail.message()))));
}
return list;
}
@@ -108,27 +113,26 @@ public final class JUnitReport implements TestReport {
private ArrayList<VmDynamic> errors(TestResult res) {
var list = new ArrayList<VmDynamic>();
long i = 0;
for (var error : res.getErrors()) {
var attrs = buildAttributes("message", error.getMessage());
for (var error : res.errors()) {
var attrs = buildAttributes("message", error.message());
long element = i++;
list.add(
buildXmlElement(
"error",
attrs,
members ->
members.put(element, syntheticElement(error.getException().getMessage()))));
members -> members.put(element, syntheticElement(error.exception().getMessage()))));
}
return list;
}
private ArrayList<VmDynamic> error(Error error) {
var list = new ArrayList<VmDynamic>();
var attrs = buildAttributes("message", error.getMessage());
var attrs = buildAttributes("message", error.message());
list.add(
buildXmlElement(
"error",
attrs,
members -> members.put(1, syntheticElement("\n" + error.getRendered()))));
members -> members.put(1, syntheticElement("\n" + error.exception().getMessage()))));
return list;
}

54
pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java
View File

@@ -18,9 +18,9 @@ package org.pkl.core.stdlib.test.report;
import java.io.IOException;
import java.io.Writer;
import java.util.stream.Collectors;
import org.pkl.core.runtime.TestResults;
import org.pkl.core.runtime.TestResults.TestSectionResults;
import org.pkl.core.runtime.TestResults.TestSectionResults.TestResult;
import org.pkl.core.TestResults;
import org.pkl.core.TestResults.TestResult;
import org.pkl.core.TestResults.TestSectionResults;
import org.pkl.core.util.StringUtils;
public final class SimpleReport implements TestReport {
@@ -29,10 +29,22 @@ public final class SimpleReport implements TestReport {
public void report(TestResults results, Writer writer) throws IOException {
var builder = new StringBuilder();
builder.append("module ").append(results.moduleName).append("\n");
builder.append("module ").append(results.moduleName()).append("\n");
reportResults(results.facts, builder);
reportResults(results.examples, builder);
if (results.error() != null) {
var rendered = results.error().exception().getMessage();
appendPadded(builder, rendered, " ");
builder.append('\n');
} else {
reportResults(results.facts(), builder);
reportResults(results.examples(), builder);
}
if (results.isExampleWrittenFailure()) {
builder.append(results.examples().totalFailures()).append(" examples written\n");
writer.append(builder);
return;
}
builder.append(results.failed() ? "❌ " : "✅ ");
@@ -51,16 +63,11 @@ public final class SimpleReport implements TestReport {
}
private void reportResults(TestSectionResults section, StringBuilder builder) {
if (!section.getResults().isEmpty()) {
builder.append(" ").append(section.name).append("\n");
if (!section.results().isEmpty()) {
builder.append(" ").append(section.name()).append("\n");
StringUtils.joinToStringBuilder(
builder, section.getResults(), "\n", res -> reportResult(res, builder));
builder.append("\n");
} else if (section.hasError()) {
builder.append(" ").append(section.name).append("\n");
var error = section.getError().getRendered();
appendPadded(builder, error, " ");
builder, section.results(), "\n", res -> reportResult(res, builder));
builder.append("\n");
}
}
@@ -69,23 +76,22 @@ public final class SimpleReport implements TestReport {
builder.append(" ");
if (result.isExampleWritten()) {
builder.append(result.name).append(" ✍️");
builder.append("✍️ ").append(result.name());
} else {
builder.append(result.isFailure() ? "❌ " : "✅ ").append(result.name);
builder.append(result.isFailure() ? "❌ " : "✅ ").append(result.name());
if (result.isFailure()) {
var failurePadding = " ";
builder.append("\n");
StringUtils.joinToStringBuilder(
builder,
result.getFailures(),
result.failures(),
"\n",
failure -> appendPadded(builder, failure.getRendered(), failurePadding));
failure -> appendPadded(builder, failure.message(), failurePadding));
StringUtils.joinToStringBuilder(
builder,
result.getErrors(),
result.errors(),
"\n",
error -> appendPadded(builder, error.getException().getMessage(), failurePadding));
error -> appendPadded(builder, error.exception().getMessage(), failurePadding));
}
}
}
@@ -96,15 +102,15 @@ public final class SimpleReport implements TestReport {
lines.lines().collect(Collectors.toList()),
"\n",
str -> {
if (str.length() > 0) builder.append(padding).append(str);
if (!str.isEmpty()) builder.append(padding).append(str);
});
}
private String makeStatsLine(String kind, int total, int failed, boolean isFailed) {
var passed = total - failed;
var pct_passed = total > 0 ? 100.0 * passed / total : 0.0;
var passRate = total > 0 ? 100.0 * passed / total : 0.0;
String line = String.format("%.1f%% %s pass", pct_passed, kind);
String line = String.format("%.1f%% %s pass", passRate, kind);
if (isFailed) {
line += String.format(" [%d/%d failed]", failed, total);

2
pkl-core/src/main/java/org/pkl/core/stdlib/test/report/TestReport.java
View File

@@ -21,7 +21,7 @@ import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import org.pkl.core.PklBugException;
import org.pkl.core.runtime.TestResults;
import org.pkl.core.TestResults;
import org.pkl.core.util.StringBuilderWriter;
public interface TestReport {

56
pkl-core/src/test/kotlin/org/pkl/core/EvaluateTestsTest.kt
View File

@@ -54,8 +54,8 @@ class EvaluateTestsTest {
assertThat(results.displayUri).isEqualTo("repl:text")
assertThat(results.totalTests()).isEqualTo(1)
assertThat(results.failed()).isFalse
assertThat(results.facts.results[0].name).isEqualTo("should pass")
assertThat(results.err.isBlank()).isTrue
assertThat(results.facts().results[0].name).isEqualTo("should pass")
assertThat(results.logs().isBlank()).isTrue
}
@Test
@@ -82,16 +82,16 @@ class EvaluateTestsTest {
assertThat(results.totalFailures()).isEqualTo(1)
assertThat(results.failed()).isTrue
val res = results.facts.results[0]
val res = results.facts().results[0]
assertThat(res.name).isEqualTo("should fail")
assertThat(results.facts.hasError()).isFalse
assertThat(results.facts().hasError()).isFalse
assertThat(res.failures.size).isEqualTo(2)
val fail1 = res.failures[0]
assertThat(fail1.rendered).isEqualTo("1 == 2 (repl:text)")
assertThat(fail1.message).isEqualTo("1 == 2 (repl:text)")
val fail2 = res.failures[1]
assertThat(fail2.rendered).isEqualTo(""""foo" == "bar" (repl:text)""")
assertThat(fail2.message).isEqualTo(""""foo" == "bar" (repl:text)""")
}
@Test
@@ -118,7 +118,7 @@ class EvaluateTestsTest {
assertThat(results.totalFailures()).isEqualTo(1)
assertThat(results.failed()).isTrue
val res = results.facts.results[0]
val res = results.facts().results[0]
assertThat(res.name).isEqualTo("should fail")
assertThat(res.failures).hasSize(1)
assertThat(res.errors).hasSize(1)
@@ -180,7 +180,7 @@ class EvaluateTestsTest {
assertThat(results.displayUri).startsWith("file:///").endsWith(".pkl")
assertThat(results.totalTests()).isEqualTo(1)
assertThat(results.failed()).isFalse
assertThat(results.examples.results[0].name).isEqualTo("user")
assertThat(results.examples().results[0].name).isEqualTo("user")
}
@Test
@@ -232,9 +232,9 @@ class EvaluateTestsTest {
assertThat(results.totalFailures()).isEqualTo(1)
assertThat(results.failed()).isTrue
assertThat(results.facts.results[0].name).isEqualTo("should fail")
assertThat(results.facts.results[0].failures.size).isEqualTo(2)
assertThat(results.examples.results[0].name).isEqualTo("user")
assertThat(results.facts().results[0].name).isEqualTo("should fail")
assertThat(results.facts().results[0].failures.size).isEqualTo(2)
assertThat(results.examples().results[0].name).isEqualTo("user")
}
@Test
@@ -286,7 +286,7 @@ class EvaluateTestsTest {
assertThat(results.totalFailures()).isEqualTo(1)
assertThat(results.failed()).isTrue
val res = results.facts.results[0]
val res = results.facts().results[0]
assertThat(res.name).isEqualTo("should fail")
assertThat(res.failures).hasSize(0)
assertThat(res.errors).hasSize(1)
@@ -294,7 +294,7 @@ class EvaluateTestsTest {
val error = res.errors[0]
assertThat(error.message).isEqualTo("exception")
assertThat(results.examples.results[0].name).isEqualTo("user")
assertThat(results.examples().results[0].name).isEqualTo("user")
}
@Test
@@ -339,26 +339,26 @@ class EvaluateTestsTest {
assertThat(results.failed()).isTrue
assertThat(results.totalFailures()).isEqualTo(1)
val res = results.examples.results[0]
val res = results.examples().results[0]
assertThat(res.name).isEqualTo("user")
assertFalse(results.examples.hasError())
assertFalse(results.examples().hasError())
val fail1 = res.failures[0]
assertThat(fail1.rendered.stripFileAndLines(tempDir))
assertThat(fail1.message.stripFileAndLines(tempDir))
.isEqualTo(
"""
(/tempDir/example.pkl)
Expected: (/tempDir/example.pkl-expected.pcf)
new {
name = "Alice"
age = 45
}
Actual: (/tempDir/example.pkl-actual.pcf)
new {
name = "Bob"
age = 33
}
"""
#0 (/tempDir/example.pkl):
Expected: (/tempDir/example.pkl-expected.pcf)
new {
name = "Alice"
age = 45
}
Actual: (/tempDir/example.pkl-actual.pcf)
new {
name = "Bob"
age = 33
}
"""
.trimIndent()
)
}