Introduces Bytes class (#1019)

This introduces a new `Bytes` standard library class, for working with binary data. * Add Bytes class to the standard library * Change CLI to eval `output.bytes` * Change code generators to map Bytes to respective underlying type * Add subscript and concat operator support * Add binary encoding for Bytes * Add PCF and Plist rendering for Bytes Co-authored-by: Kushal Pisavadia <kushi.p@gmail.com>
2026-04-10 10:53:40 +02:00 · 2025-06-11 16:23:55 -07:00
parent 3bd8a88506
commit e9320557b7
104 changed files with 2210 additions and 545 deletions
--- a/pkl-cli/src/main/kotlin/org/pkl/cli/CliEvaluator.kt
+++ b/pkl-cli/src/main/kotlin/org/pkl/cli/CliEvaluator.kt
@@ -16,20 +16,23 @@
 package org.pkl.cli

 import java.io.File
-import java.io.Reader
-import java.io.Writer
+import java.io.InputStream
+import java.io.OutputStream
 import java.net.URI
+import java.nio.charset.StandardCharsets
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.StandardOpenOption
 import kotlin.io.path.createParentDirectories
 import kotlin.io.path.exists
 import kotlin.io.path.isDirectory
+import kotlin.io.path.writeBytes
 import org.pkl.commons.cli.CliCommand
 import org.pkl.commons.cli.CliException
 import org.pkl.commons.currentWorkingDir
 import org.pkl.commons.writeString
 import org.pkl.core.Closeables
+import org.pkl.core.Evaluator
 import org.pkl.core.EvaluatorBuilder
 import org.pkl.core.ModuleSource
 import org.pkl.core.PklException
@@ -48,8 +51,8 @@ constructor(
  private val options: CliEvaluatorOptions,
  // use System.{in,out}() rather than System.console()
  // because the latter returns null when output is sent through a unix pipe
-  private val consoleReader: Reader = System.`in`.reader(),
-  private val consoleWriter: Writer = System.out.writer(),
+  private val inputStream: InputStream = System.`in`,
+  private val outputStream: OutputStream = System.out,
 ) : CliCommand(options.base) {
  /**
   * Output files for the modules to be evaluated. Returns `null` if `options.outputPath` is `null`
@@ -84,11 +87,12 @@ constructor(
  /**
   * Evaluates source modules according to [options].
   *
-   * If [CliEvaluatorOptions.outputPath] is set, each module's `output.text` is written to the
-   * module's [output file][outputFiles]. If [CliEvaluatorOptions.multipleFileOutputPath] is set,
-   * each module's `output.files` are written to the module's [output directory][outputDirectories].
-   * Otherwise, each module's `output.text` is written to [consoleWriter] (which defaults to
-   * standard out).
+   * If [CliEvaluatorOptions.outputPath] is set, each module's `output.bytes` is written to the
+   * module's [output file][outputFiles].
+   *
+   * If [CliEvaluatorOptions.multipleFileOutputPath] is set, each module's `output.files` are
+   * written to the module's [output directory][outputDirectories]. Otherwise, each module's
+   * `output.bytes` is written to [outputStream] (which defaults to standard out).
   *
   * Throws [CliException] in case of an error.
   */
@@ -139,10 +143,29 @@ constructor(
    }
  }

-  /** Renders each module's `output.text`, writing it to the specified output file. */
+  private fun Evaluator.writeOutput(moduleSource: ModuleSource, writeTo: Path): Boolean {
+    if (options.expression == null) {
+      val bytes = evaluateOutputBytes(moduleSource)
+      writeTo.writeBytes(bytes)
+      return bytes.isNotEmpty()
+    }
+    val text = evaluateExpressionString(moduleSource, options.expression)
+    writeTo.writeString(text)
+    return text.isNotEmpty()
+  }
+
+  private fun Evaluator.evalOutput(moduleSource: ModuleSource): ByteArray {
+    if (options.expression == null) {
+      return evaluateOutputBytes(moduleSource)
+    }
+    return evaluateExpressionString(moduleSource, options.expression)
+      .toByteArray(StandardCharsets.UTF_8)
+  }
+
+  /** Renders each module's `output.bytes`, writing it to the specified output file. */
  private fun writeOutput(builder: EvaluatorBuilder) {
    val evaluator = builder.setOutputFormat(options.outputFormat).build()
-    evaluator.use {
+    evaluator.use { ev ->
      val outputFiles = fileOutputPaths
      if (outputFiles != null) {
        // files that we've written non-empty output to
@@ -151,18 +174,18 @@ constructor(
        val writtenFiles = mutableSetOf<Path>()

        for ((moduleUri, outputFile) in outputFiles) {
-          val moduleSource = toModuleSource(moduleUri, consoleReader)
-          val output = evaluator.evaluateExpressionString(moduleSource, options.expression)
+          val moduleSource = toModuleSource(moduleUri, inputStream)
          if (Files.isDirectory(outputFile)) {
            throw CliException(
              "Output file `$outputFile` is a directory. " +
                "Did you mean `--multiple-file-output-path`?"
            )
          }
+          val output = ev.evalOutput(moduleSource)
          outputFile.createParentDirectories()
          if (!writtenFiles.contains(outputFile)) {
            // write file even if output is empty to overwrite output from previous runs
-            outputFile.writeString(output)
+            outputFile.writeBytes(output)
            if (output.isNotEmpty()) {
              writtenFiles.add(outputFile)
            }
@@ -174,34 +197,49 @@ constructor(
                StandardOpenOption.WRITE,
                StandardOpenOption.APPEND,
              )
-              outputFile.writeString(
-                output,
-                Charsets.UTF_8,
-                StandardOpenOption.WRITE,
-                StandardOpenOption.APPEND,
-              )
+              outputFile.writeBytes(output, StandardOpenOption.WRITE, StandardOpenOption.APPEND)
            }
          }
        }
      } else {
        var outputWritten = false
        for (moduleUri in options.base.normalizedSourceModules) {
-          val moduleSource = toModuleSource(moduleUri, consoleReader)
-          val output = evaluator.evaluateExpressionString(moduleSource, options.expression)
-          if (output.isNotEmpty()) {
-            if (outputWritten) consoleWriter.appendLine(options.moduleOutputSeparator)
-            consoleWriter.write(output)
-            consoleWriter.flush()
-            outputWritten = true
+          val moduleSource = toModuleSource(moduleUri, inputStream)
+          if (options.expression != null) {
+            val output = evaluator.evaluateExpressionString(moduleSource, options.expression)
+            if (output.isNotEmpty()) {
+              if (outputWritten) outputStream.writeLine(options.moduleOutputSeparator)
+              outputStream.writeText(output)
+              outputStream.flush()
+              outputWritten = true
+            }
+          } else {
+            val outputBytes = evaluator.evaluateOutputBytes(moduleSource)
+            if (outputBytes.isNotEmpty()) {
+              if (outputWritten) outputStream.writeLine(options.moduleOutputSeparator)
+              outputStream.write(outputBytes)
+              outputStream.flush()
+              outputWritten = true
+            }
          }
        }
      }
    }
  }

-  private fun toModuleSource(uri: URI, reader: Reader) =
-    if (uri == VmUtils.REPL_TEXT_URI) ModuleSource.create(uri, reader.readText())
-    else ModuleSource.uri(uri)
+  private fun OutputStream.writeText(text: String) = write(text.toByteArray())
+
+  private fun OutputStream.writeLine(text: String) {
+    writeText(text)
+    writeText("\n")
+  }
+
+  private fun toModuleSource(uri: URI, reader: InputStream) =
+    if (uri == VmUtils.REPL_TEXT_URI) {
+      ModuleSource.create(uri, reader.readAllBytes().toString(StandardCharsets.UTF_8))
+    } else {
+      ModuleSource.uri(uri)
+    }

  private fun checkPathSpec(pathSpec: String) {
    val illegal = pathSpec.indexOfFirst { IoUtils.isReservedFilenameChar(it) && it != '/' }
@@ -223,7 +261,7 @@ constructor(
      if (outputDir.exists() && !outputDir.isDirectory()) {
        throw CliException("Output path `$outputDir` exists and is not a directory.")
      }
-      val moduleSource = toModuleSource(moduleUri, consoleReader)
+      val moduleSource = toModuleSource(moduleUri, inputStream)
      val output = evaluator.evaluateOutputFiles(moduleSource)
      for ((pathSpec, fileOutput) in output) {
        checkPathSpec(pathSpec)
@@ -247,12 +285,12 @@ constructor(
        }
        writtenFiles[realPath] = OutputFile(pathSpec, moduleUri)
        realPath.createParentDirectories()
-        realPath.writeString(fileOutput.text)
-        consoleWriter.write(
+        realPath.writeBytes(fileOutput.bytes)
+        outputStream.writeText(
          IoUtils.relativize(resolvedPath, currentWorkingDir).toString() +
            IoUtils.getLineSeparator()
        )
-        consoleWriter.flush()
+        outputStream.flush()
      }
    }
  }
--- a/pkl-cli/src/main/kotlin/org/pkl/cli/CliEvaluatorOptions.kt
+++ b/pkl-cli/src/main/kotlin/org/pkl/cli/CliEvaluatorOptions.kt
@@ -74,9 +74,9 @@ data class CliEvaluatorOptions(
   *
   * If set, the said expression is evaluated under the context of the enclosing module.
   *
-   * If unset, the module's `output.text` property evaluated.
+   * If unset, the module's `output.bytes` property is evaluated.
   */
-  val expression: String = "output.text",
+  val expression: String? = null,
 ) {

  companion object {
--- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt
+++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt
@@ -18,10 +18,11 @@ package org.pkl.cli
 import com.github.tomakehurst.wiremock.client.WireMock.*
 import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo
 import com.github.tomakehurst.wiremock.junit5.WireMockTest
-import java.io.StringReader
-import java.io.StringWriter
+import java.io.ByteArrayInputStream
+import java.io.ByteArrayOutputStream
 import java.net.ServerSocket
 import java.net.URI
+import java.nio.charset.StandardCharsets
 import java.nio.file.Files
 import java.nio.file.Path
 import java.time.Duration
@@ -29,22 +30,20 @@ import java.util.regex.Pattern
 import kotlin.io.path.*
 import org.assertj.core.api.Assertions.assertThat
 import org.assertj.core.api.Assertions.assertThatCode
-import org.junit.jupiter.api.AfterAll
-import org.junit.jupiter.api.AfterEach
-import org.junit.jupiter.api.Disabled
-import org.junit.jupiter.api.Test
-import org.junit.jupiter.api.assertThrows
+import org.junit.jupiter.api.*
 import org.junit.jupiter.api.condition.DisabledOnOs
 import org.junit.jupiter.api.condition.EnabledOnOs
 import org.junit.jupiter.api.condition.OS
 import org.junit.jupiter.api.io.TempDir
 import org.junit.jupiter.params.ParameterizedTest
 import org.junit.jupiter.params.provider.EnumSource
-import org.pkl.commons.*
 import org.pkl.commons.cli.CliBaseOptions
 import org.pkl.commons.cli.CliException
+import org.pkl.commons.readString
 import org.pkl.commons.test.FileTestUtils
 import org.pkl.commons.test.PackageServer
+import org.pkl.commons.toPath
+import org.pkl.commons.writeString
 import org.pkl.core.OutputFormat
 import org.pkl.core.SecurityManagers
 import org.pkl.core.util.IoUtils
@@ -157,7 +156,6 @@ person:
      )

    assertThat(outputFiles).hasSize(1)
-    @Suppress("HttpUrlsUsage")
    checkOutputFile(
      outputFiles[0],
      "test.plist",
@@ -469,8 +467,8 @@ result = someLib.x

  @Test
  fun `take input from stdin`() {
-    val stdin = StringReader(defaultContents)
-    val stdout = StringWriter()
+    val stdin = ByteArrayInputStream(defaultContents.toByteArray(StandardCharsets.UTF_8))
+    val stdout = ByteArrayOutputStream()
    val evaluator =
      CliEvaluator(
        CliEvaluatorOptions(
@@ -481,7 +479,8 @@ result = someLib.x
        stdout,
      )
    evaluator.run()
-    assertThat(stdout.toString().trim()).isEqualTo(defaultContents.replace("20 + 10", "30").trim())
+    assertThat(stdout.toString(StandardCharsets.UTF_8).trim())
+      .isEqualTo(defaultContents.replace("20 + 10", "30").trim())
  }

  @Test
@@ -1101,9 +1100,9 @@ result = someLib.x
        CliBaseOptions(sourceModules = listOf(moduleUri), workingDir = tempDir),
        expression = "foo",
      )
-    val buffer = StringWriter()
-    CliEvaluator(options, consoleWriter = buffer).run()
-    assertThat(buffer.toString())
+    val buffer = ByteArrayOutputStream()
+    CliEvaluator(options, outputStream = buffer).run()
+    assertThat(buffer.toString(StandardCharsets.UTF_8))
      .isEqualTo(
        """
      new Dynamic { bar = 1 }
@@ -1132,9 +1131,9 @@ result = someLib.x
        CliBaseOptions(sourceModules = listOf(moduleUri), workingDir = tempDir),
        expression = "person",
      )
-    val buffer = StringWriter()
-    CliEvaluator(options, consoleWriter = buffer).run()
-    assertThat(buffer.toString()).isEqualTo("Person(Frodo)")
+    val buffer = ByteArrayOutputStream()
+    CliEvaluator(options, outputStream = buffer).run()
+    assertThat(buffer.toString(StandardCharsets.UTF_8)).isEqualTo("Person(Frodo)")
  }

  @Test
@@ -1154,9 +1153,10 @@ result = someLib.x
        CliBaseOptions(sourceModules = listOf(moduleUri), workingDir = tempDir),
        expression = "person",
      )
-    val buffer = StringWriter()
-    CliEvaluator(options, consoleWriter = buffer).run()
-    assertThat(buffer.toString()).isEqualTo("new Dynamic { friend { name = \"Bilbo\" } }")
+    val buffer = ByteArrayOutputStream()
+    CliEvaluator(options, outputStream = buffer).run()
+    assertThat(buffer.toString(StandardCharsets.UTF_8))
+      .isEqualTo("new Dynamic { friend { name = \"Bilbo\" } }")
  }

  @Test
@@ -1182,9 +1182,9 @@ result = someLib.x
      CliEvaluatorOptions(
        CliBaseOptions(sourceModules = listOf(moduleUri), workingDir = tempDir, noProject = true)
      )
-    val buffer = StringWriter()
-    CliEvaluator(options, consoleWriter = buffer).run()
-    assertThat(buffer.toString()).isEqualTo("res = 1\n")
+    val buffer = ByteArrayOutputStream()
+    CliEvaluator(options, outputStream = buffer).run()
+    assertThat(buffer.toString(StandardCharsets.UTF_8)).isEqualTo("res = 1\n")
  }

  @Test
@@ -1215,9 +1215,9 @@ result = someLib.x
    )
    val options =
      CliEvaluatorOptions(CliBaseOptions(sourceModules = listOf(moduleUri), workingDir = tempDir))
-    val buffer = StringWriter()
-    CliEvaluator(options, consoleWriter = buffer).run()
-    assertThat(buffer.toString())
+    val buffer = ByteArrayOutputStream()
+    CliEvaluator(options, outputStream = buffer).run()
+    assertThat(buffer.toString(StandardCharsets.UTF_8))
      .isEqualTo(
        """
      res {
@@ -1242,7 +1242,7 @@ result = someLib.x
    """
          .trimIndent(),
      )
-    val buffer = StringWriter()
+    val buffer = ByteArrayOutputStream()
    val options =
      CliEvaluatorOptions(
        CliBaseOptions(
@@ -1254,8 +1254,8 @@ result = someLib.x
          testPort = packageServer.port,
        )
      )
-    CliEvaluator(options, consoleWriter = buffer).run()
-    assertThat(buffer.toString())
+    CliEvaluator(options, outputStream = buffer).run()
+    assertThat(buffer.toString(StandardCharsets.UTF_8))
      .isEqualTo(
        """
      res {
@@ -1590,10 +1590,10 @@ result = someLib.x
  }

  private fun evalToConsole(options: CliEvaluatorOptions): String {
-    val reader = StringReader("")
-    val writer = StringWriter()
+    val reader = ByteArrayInputStream(byteArrayOf())
+    val writer = ByteArrayOutputStream()
    CliEvaluator(options, reader, writer).run()
-    return writer.toString()
+    return writer.toString(StandardCharsets.UTF_8)
  }

  private fun checkOutputFile(file: Path, name: String, contents: String) {