mirror of
https://github.com/apple/pkl.git
synced 2026-07-03 11:41:43 +02:00
Polish test result running and reporting (#738)
Changes: * Move class `TestResults` to package `org.pkl.core`, because it is a public class (it's the result of `Evaluator#evaluateTest`) * Change examples to treat individual examples as assertions in the same test. Previously, they were considered different tests with an incrementing number. This better aligns with how facts are treated. * Change `TestResults` to be a record, and introduce builders. * Remove "module" test result section (it is not really a section). * Add javadoc to `TestResults` * Formatting fix: prefix ✍️ emoji just like we do the ❌ and ✅ emojis * Consider writing examples as failures, not successes. `pkl test` will exit with code 10 if the only failing tests are due to writing examples.
This commit is contained in:
@@ -54,8 +54,8 @@ class EvaluateTestsTest {
|
||||
assertThat(results.displayUri).isEqualTo("repl:text")
|
||||
assertThat(results.totalTests()).isEqualTo(1)
|
||||
assertThat(results.failed()).isFalse
|
||||
assertThat(results.facts.results[0].name).isEqualTo("should pass")
|
||||
assertThat(results.err.isBlank()).isTrue
|
||||
assertThat(results.facts().results[0].name).isEqualTo("should pass")
|
||||
assertThat(results.logs().isBlank()).isTrue
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -82,16 +82,16 @@ class EvaluateTestsTest {
|
||||
assertThat(results.totalFailures()).isEqualTo(1)
|
||||
assertThat(results.failed()).isTrue
|
||||
|
||||
val res = results.facts.results[0]
|
||||
val res = results.facts().results[0]
|
||||
assertThat(res.name).isEqualTo("should fail")
|
||||
assertThat(results.facts.hasError()).isFalse
|
||||
assertThat(results.facts().hasError()).isFalse
|
||||
assertThat(res.failures.size).isEqualTo(2)
|
||||
|
||||
val fail1 = res.failures[0]
|
||||
assertThat(fail1.rendered).isEqualTo("1 == 2 (repl:text)")
|
||||
assertThat(fail1.message).isEqualTo("1 == 2 (repl:text)")
|
||||
|
||||
val fail2 = res.failures[1]
|
||||
assertThat(fail2.rendered).isEqualTo(""""foo" == "bar" (repl:text)""")
|
||||
assertThat(fail2.message).isEqualTo(""""foo" == "bar" (repl:text)""")
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -118,7 +118,7 @@ class EvaluateTestsTest {
|
||||
assertThat(results.totalFailures()).isEqualTo(1)
|
||||
assertThat(results.failed()).isTrue
|
||||
|
||||
val res = results.facts.results[0]
|
||||
val res = results.facts().results[0]
|
||||
assertThat(res.name).isEqualTo("should fail")
|
||||
assertThat(res.failures).hasSize(1)
|
||||
assertThat(res.errors).hasSize(1)
|
||||
@@ -180,7 +180,7 @@ class EvaluateTestsTest {
|
||||
assertThat(results.displayUri).startsWith("file:///").endsWith(".pkl")
|
||||
assertThat(results.totalTests()).isEqualTo(1)
|
||||
assertThat(results.failed()).isFalse
|
||||
assertThat(results.examples.results[0].name).isEqualTo("user")
|
||||
assertThat(results.examples().results[0].name).isEqualTo("user")
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -232,9 +232,9 @@ class EvaluateTestsTest {
|
||||
assertThat(results.totalFailures()).isEqualTo(1)
|
||||
assertThat(results.failed()).isTrue
|
||||
|
||||
assertThat(results.facts.results[0].name).isEqualTo("should fail")
|
||||
assertThat(results.facts.results[0].failures.size).isEqualTo(2)
|
||||
assertThat(results.examples.results[0].name).isEqualTo("user")
|
||||
assertThat(results.facts().results[0].name).isEqualTo("should fail")
|
||||
assertThat(results.facts().results[0].failures.size).isEqualTo(2)
|
||||
assertThat(results.examples().results[0].name).isEqualTo("user")
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -286,7 +286,7 @@ class EvaluateTestsTest {
|
||||
assertThat(results.totalFailures()).isEqualTo(1)
|
||||
assertThat(results.failed()).isTrue
|
||||
|
||||
val res = results.facts.results[0]
|
||||
val res = results.facts().results[0]
|
||||
assertThat(res.name).isEqualTo("should fail")
|
||||
assertThat(res.failures).hasSize(0)
|
||||
assertThat(res.errors).hasSize(1)
|
||||
@@ -294,7 +294,7 @@ class EvaluateTestsTest {
|
||||
val error = res.errors[0]
|
||||
assertThat(error.message).isEqualTo("exception")
|
||||
|
||||
assertThat(results.examples.results[0].name).isEqualTo("user")
|
||||
assertThat(results.examples().results[0].name).isEqualTo("user")
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -339,26 +339,26 @@ class EvaluateTestsTest {
|
||||
assertThat(results.failed()).isTrue
|
||||
assertThat(results.totalFailures()).isEqualTo(1)
|
||||
|
||||
val res = results.examples.results[0]
|
||||
val res = results.examples().results[0]
|
||||
assertThat(res.name).isEqualTo("user")
|
||||
assertFalse(results.examples.hasError())
|
||||
assertFalse(results.examples().hasError())
|
||||
|
||||
val fail1 = res.failures[0]
|
||||
assertThat(fail1.rendered.stripFileAndLines(tempDir))
|
||||
assertThat(fail1.message.stripFileAndLines(tempDir))
|
||||
.isEqualTo(
|
||||
"""
|
||||
(/tempDir/example.pkl)
|
||||
Expected: (/tempDir/example.pkl-expected.pcf)
|
||||
new {
|
||||
name = "Alice"
|
||||
age = 45
|
||||
}
|
||||
Actual: (/tempDir/example.pkl-actual.pcf)
|
||||
new {
|
||||
name = "Bob"
|
||||
age = 33
|
||||
}
|
||||
"""
|
||||
#0 (/tempDir/example.pkl):
|
||||
Expected: (/tempDir/example.pkl-expected.pcf)
|
||||
new {
|
||||
name = "Alice"
|
||||
age = 45
|
||||
}
|
||||
Actual: (/tempDir/example.pkl-actual.pcf)
|
||||
new {
|
||||
name = "Bob"
|
||||
age = 33
|
||||
}
|
||||
"""
|
||||
.trimIndent()
|
||||
)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user