Assert that formatter snippet test output is stable (#1270)

pkl-commons-test/src/main/kotlin/org/pkl/commons/test/FileTestUtils.kt

@@ -111,7 +111,10 @@ data class SnippetOutcome(val expectedOutFile: Path, val actual: String, val suc
     }
   }
 
-  private fun failWithDiff(message: String): Nothing =
+  private fun failWithDiff(message: String): Nothing = failWithDiff(message, expected, actual)
+}
+
+fun failWithDiff(message: String, expected: String, actual: String): Nothing =
   if (System.getProperty("sun.java.command", "").contains("intellij")) {
     // IntelliJ only shows diffs for AssertionFailedError
     throw AssertionFailedError(message, expected, actual)
@@ -119,4 +122,3 @@ data class SnippetOutcome(val expectedOutFile: Path, val actual: String, val suc
     // Gradle test logging/report only shows diffs for PklAssertionFailedError
     throw PklAssertionFailedError(message, expected, actual)
   }
-}

pkl-formatter/src/test/files/FormatterSnippetTests/input/line-breaks.pkl

@@ -62,3 +62,12 @@ local const function someFunction(param1: String, param2: String(!isBlank)): Boo
 local function render(currentIndent: String) =
   "\(currentIndent)@\(identifier.render(currentIndent))" +
     if (body == null) "" else " " + body.render(currentIndent)
+
+// FIXME: output is currently unstable
+// items: List<Item> =
+//   allItems
+//     .filter((item) ->
+//     badItems.containsKey(item) // some line comment 
+//     || item.tags.toList().findOrNull((it) -> it.type == "bookmark") != null // some other line comment
+//   )
+//     .sortBy((item) -> item.name)

pkl-formatter/src/test/files/FormatterSnippetTests/output/line-breaks.pkl

@@ -73,3 +73,12 @@ local const function someFunction(param1: String, param2: String(!isBlank)): Boo
 local function render(currentIndent: String) =
   "\(currentIndent)@\(identifier.render(currentIndent))"
     + if (body == null) "" else " " + body.render(currentIndent)
+
+// FIXME: output is currently unstable
+// items: List<Item> =
+//   allItems
+//     .filter((item) ->
+//     badItems.containsKey(item) // some line comment 
+//     || item.tags.toList().findOrNull((it) -> it.type == "bookmark") != null // some other line comment
+//   )
+//     .sortBy((item) -> item.name)

pkl-formatter/src/test/kotlin/org/pkl/formatter/FormatterTest.kt

@@ -15,9 +15,16 @@
  */
 package org.pkl.formatter
 
+import java.nio.file.Path
+import kotlin.io.path.isRegularFile
+import kotlin.io.path.relativeTo
+import kotlin.io.path.useDirectoryEntries
 import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
+import org.pkl.commons.readString
+import org.pkl.commons.test.FileTestUtils
+import org.pkl.commons.test.failWithDiff
 import org.pkl.parser.GenericParserError
 
 class FormatterTest {
@@ -76,4 +83,34 @@ class FormatterTest {
   private fun format(code: String): String {
     return Formatter().format(code.trimIndent())
   }
+
+  @Test
+  fun `snippet test output must be stable`() {
+    val outputDir =
+      FileTestUtils.rootProjectDir.resolve(
+        "pkl-formatter/src/test/files/FormatterSnippetTests/output"
+      )
+    val formatter = Formatter()
+    fun walkDir(dir: Path) {
+      dir.useDirectoryEntries { children ->
+        for (child in children) {
+          if (child.isRegularFile()) {
+            val expected = child.readString()
+            val formatted = formatter.format(expected)
+            if (expected != formatted) {
+              failWithDiff(
+                "Formatter output not stable: ${child.relativeTo(outputDir)}",
+                expected,
+                formatted,
+              )
+            }
+          } else {
+            walkDir(child)
+          }
+        }
+      }
+    }
+
+    walkDir(outputDir)
+  }
 }