Improve testing with stats and errors per test section (#498)

* Emojis are moves to the left to be aligned
* A summary line is added with test counts
* Facts and Examples are grouped under their own section
This commit is contained in:
Javier Maestro
2024-10-24 07:00:35 +01:00
committed by GitHub
parent 2040f14b07
commit 86d870ba09
8 changed files with 858 additions and 393 deletions
@@ -54,7 +54,7 @@ class EvaluateTestsTest {
assertThat(results.displayUri).isEqualTo("repl:text")
assertThat(results.totalTests()).isEqualTo(1)
assertThat(results.failed()).isFalse
assertThat(results.results[0].name).isEqualTo("should pass")
assertThat(results.facts.results[0].name).isEqualTo("should pass")
assertThat(results.err.isBlank()).isTrue
}
@@ -79,18 +79,19 @@ class EvaluateTestsTest {
)
assertThat(results.totalTests()).isEqualTo(1)
assertThat(results.totalFailures()).isEqualTo(2)
assertThat(results.totalFailures()).isEqualTo(1)
assertThat(results.failed()).isTrue
val res = results.results[0]
val res = results.facts.results[0]
assertThat(res.name).isEqualTo("should fail")
assertThat(res.errors).isEmpty()
assertThat(results.facts.hasError()).isFalse
assertThat(res.failures.size).isEqualTo(2)
val fail1 = res.failures[0]
assertThat(fail1.rendered).isEqualTo("1 == 2 (repl:text)")
assertThat(fail1.rendered).isEqualTo("1 == 2 (repl:text)")
val fail2 = res.failures[1]
assertThat(fail2.rendered).isEqualTo(""""foo" == "bar" (repl:text)""")
assertThat(fail2.rendered).isEqualTo(""""foo" == "bar" (repl:text)""")
}
@Test
@@ -117,7 +118,7 @@ class EvaluateTestsTest {
assertThat(results.totalFailures()).isEqualTo(1)
assertThat(results.failed()).isTrue
val res = results.results[0]
val res = results.facts.results[0]
assertThat(res.name).isEqualTo("should fail")
assertThat(res.failures).hasSize(1)
assertThat(res.errors).hasSize(1)
@@ -179,7 +180,121 @@ class EvaluateTestsTest {
assertThat(results.displayUri).startsWith("file:///").endsWith(".pkl")
assertThat(results.totalTests()).isEqualTo(1)
assertThat(results.failed()).isFalse
assertThat(results.results[0].name).isEqualTo("user")
assertThat(results.examples.results[0].name).isEqualTo("user")
}
@Test
fun `test fact failures with successful example`(@TempDir tempDir: Path) {
val file = tempDir.createTempFile(prefix = "example", suffix = ".pkl")
Files.writeString(
file,
"""
amends "pkl:test"
facts {
["should fail"] {
1 == 2
"foo" == "bar"
}
}
examples {
["user"] {
new {
name = "Bob"
age = 33
}
}
}
"""
.trimIndent()
)
Files.writeString(
createExpected(file),
"""
examples {
["user"] {
new {
name = "Bob"
age = 33
}
}
}
"""
.trimIndent()
)
val results = evaluator.evaluateTest(path(file), false)
assertThat(results.moduleName).startsWith("example")
assertThat(results.displayUri).startsWith("file:///").endsWith(".pkl")
assertThat(results.totalTests()).isEqualTo(2)
assertThat(results.totalFailures()).isEqualTo(1)
assertThat(results.failed()).isTrue
assertThat(results.facts.results[0].name).isEqualTo("should fail")
assertThat(results.facts.results[0].failures.size).isEqualTo(2)
assertThat(results.examples.results[0].name).isEqualTo("user")
}
@Test
fun `test fact error with successful example`(@TempDir tempDir: Path) {
val file = tempDir.createTempFile(prefix = "example", suffix = ".pkl")
Files.writeString(
file,
"""
amends "pkl:test"
facts {
["should fail"] {
throw("exception")
}
}
examples {
["user"] {
new {
name = "Bob"
age = 33
}
}
}
"""
.trimIndent()
)
Files.writeString(
createExpected(file),
"""
examples {
["user"] {
new {
name = "Bob"
age = 33
}
}
}
"""
.trimIndent()
)
val results = evaluator.evaluateTest(path(file), false)
assertThat(results.moduleName).startsWith("example")
assertThat(results.displayUri).startsWith("file:///").endsWith(".pkl")
assertThat(results.totalTests()).isEqualTo(2)
assertThat(results.totalFailures()).isEqualTo(1)
assertThat(results.failed()).isTrue
val res = results.facts.results[0]
assertThat(res.name).isEqualTo("should fail")
assertThat(res.failures).hasSize(0)
assertThat(res.errors).hasSize(1)
val error = res.errors[0]
assertThat(error.message).isEqualTo("exception")
assertThat(results.examples.results[0].name).isEqualTo("user")
}
@Test
@@ -224,9 +339,9 @@ class EvaluateTestsTest {
assertThat(results.failed()).isTrue
assertThat(results.totalFailures()).isEqualTo(1)
val res = results.results[0]
val res = results.examples.results[0]
assertThat(res.name).isEqualTo("user")
assertThat(res.errors.isEmpty()).isTrue
assertFalse(results.examples.hasError())
val fail1 = res.failures[0]
assertThat(fail1.rendered.stripFileAndLines(tempDir))