Introduce C library for Pkl (#1238)

This uses native-image to generate a C library for Pkl.
This generated library from native-image is wrapped with our own library,
in `pkl.h`.

This produces a static and a dynamic library for each os/arch variant
that Pkl currently supports.

Co-authored-by: Kushal Pisavadia <kushal.p@apple.com>
Co-authored-by: Jen Basch <jbasch94@gmail.com>
Co-authored-by: Islon Scherer <i_desouzascherer@apple.com>

.github/workflows/release.yml

@@ -239,10 +239,10 @@ jobs:
     - name: gradle buildNative
       shell: bash
       run: ./gradlew $DEBUG_ARGS --no-daemon -DpklMultiJdkTesting=true -DreleaseBuild=true pkl-cli:buildNative
-    - name: Upload executable artifacts
+    - name: Upload built artifacts
       uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
       with:
-        name: executable-pkl-cli-linux-amd64
+        name: artifacts-pkl-cli-linux-amd64
         path: pkl-cli*/build/executable/**/*
     - name: Upload Test Result XML
       if: '!cancelled()'
@@ -290,10 +290,10 @@ jobs:
     - name: gradle buildNative
       shell: bash
       run: ./gradlew $DEBUG_ARGS --no-daemon -DpklMultiJdkTesting=true -DreleaseBuild=true pkl-cli:buildNative
-    - name: Upload executable artifacts
+    - name: Upload built artifacts
       uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
       with:
-        name: executable-pkl-cli-linux-aarch64
+        name: artifacts-pkl-cli-linux-aarch64
         path: pkl-cli*/build/executable/**/*
     - name: Upload Test Result XML
       if: '!cancelled()'
@@ -340,10 +340,10 @@ jobs:
     - name: gradle buildNative
       shell: bash
       run: ./gradlew $DEBUG_ARGS --no-daemon -DpklMultiJdkTesting=true -DreleaseBuild=true pkl-cli:buildNative
-    - name: Upload executable artifacts
+    - name: Upload built artifacts
       uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
       with:
-        name: executable-pkl-cli-macOS-aarch64
+        name: artifacts-pkl-cli-macOS-aarch64
         path: pkl-cli*/build/executable/**/*
     - name: Upload Test Result XML
       if: '!cancelled()'
@@ -458,10 +458,10 @@ jobs:
     - name: gradle buildNative
       shell: bash
       run: ./gradlew $DEBUG_ARGS --no-daemon -DpklMultiJdkTesting=true -DreleaseBuild=true -Dpkl.musl=true pkl-cli:buildNative
-    - name: Upload executable artifacts
+    - name: Upload built artifacts
       uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
       with:
-        name: executable-pkl-cli-alpine-linux-amd64
+        name: artifacts-pkl-cli-alpine-linux-amd64
         path: pkl-cli*/build/executable/**/*
     - name: Upload Test Result XML
       if: '!cancelled()'
@@ -504,10 +504,10 @@ jobs:
     - name: gradle buildNative
       shell: bash
       run: ./gradlew $DEBUG_ARGS --no-daemon -DpklMultiJdkTesting=true -DreleaseBuild=true pkl-cli:buildNative
-    - name: Upload executable artifacts
+    - name: Upload built artifacts
       uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
       with:
-        name: executable-pkl-cli-windows-amd64
+        name: artifacts-pkl-cli-windows-amd64
         path: pkl-cli*/build/executable/**/*
     - name: Upload Test Result XML
       if: '!cancelled()'
@@ -553,10 +553,10 @@ jobs:
     - name: gradle buildNative
       shell: bash
       run: ./gradlew $DEBUG_ARGS --no-daemon -DpklMultiJdkTesting=true -DreleaseBuild=true pkl-doc:buildNative
-    - name: Upload executable artifacts
+    - name: Upload built artifacts
       uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
       with:
-        name: executable-pkl-doc-linux-amd64
+        name: artifacts-pkl-doc-linux-amd64
         path: pkl-doc*/build/executable/**/*
     - name: Upload Test Result XML
       if: '!cancelled()'
@@ -604,10 +604,10 @@ jobs:
     - name: gradle buildNative
       shell: bash
       run: ./gradlew $DEBUG_ARGS --no-daemon -DpklMultiJdkTesting=true -DreleaseBuild=true pkl-doc:buildNative
-    - name: Upload executable artifacts
+    - name: Upload built artifacts
       uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
       with:
-        name: executable-pkl-doc-linux-aarch64
+        name: artifacts-pkl-doc-linux-aarch64
         path: pkl-doc*/build/executable/**/*
     - name: Upload Test Result XML
       if: '!cancelled()'
@@ -654,10 +654,10 @@ jobs:
     - name: gradle buildNative
       shell: bash
       run: ./gradlew $DEBUG_ARGS --no-daemon -DpklMultiJdkTesting=true -DreleaseBuild=true pkl-doc:buildNative
-    - name: Upload executable artifacts
+    - name: Upload built artifacts
       uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
       with:
-        name: executable-pkl-doc-macOS-aarch64
+        name: artifacts-pkl-doc-macOS-aarch64
         path: pkl-doc*/build/executable/**/*
     - name: Upload Test Result XML
       if: '!cancelled()'
@@ -772,10 +772,10 @@ jobs:
     - name: gradle buildNative
       shell: bash
       run: ./gradlew $DEBUG_ARGS --no-daemon -DpklMultiJdkTesting=true -DreleaseBuild=true -Dpkl.musl=true pkl-doc:buildNative
-    - name: Upload executable artifacts
+    - name: Upload built artifacts
       uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
       with:
-        name: executable-pkl-doc-alpine-linux-amd64
+        name: artifacts-pkl-doc-alpine-linux-amd64
         path: pkl-doc*/build/executable/**/*
     - name: Upload Test Result XML
       if: '!cancelled()'
@@ -818,10 +818,10 @@ jobs:
     - name: gradle buildNative
       shell: bash
       run: ./gradlew $DEBUG_ARGS --no-daemon -DpklMultiJdkTesting=true -DreleaseBuild=true pkl-doc:buildNative
-    - name: Upload executable artifacts
+    - name: Upload built artifacts
       uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
       with:
-        name: executable-pkl-doc-windows-amd64
+        name: artifacts-pkl-doc-windows-amd64
         path: pkl-doc*/build/executable/**/*
     - name: Upload Test Result XML
       if: '!cancelled()'
@@ -836,6 +836,322 @@ jobs:
         name: test-results-html-pkl-doc-windows-amd64-release
         path: '**/build/reports/tests/**/*'
         if-no-files-found: ignore
+  libpkl-linux-amd64-release:
+    runs-on: ubuntu-latest
+    env:
+      LANG: en_US.UTF-8
+    steps:
+    - name: Install deps
+      run: dnf install -y git binutils gcc glibc-devel zlib-devel libstdc++-static glibc-langpack-en
+    - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+      with:
+        persist-credentials: false
+    - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5
+      with:
+        java-version: '25'
+        distribution: temurin
+        architecture: x64
+    - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5
+      with:
+        cache-disabled: true
+    - name: Set DEBUG_ARGS env var
+      env:
+        RUNNER_DEBUG: ${{ runner.debug }}
+      shell: bash
+      run: |-
+        if [[ "$RUNNER_DEBUG" -eq 1 ]]; then
+          echo "DEBUG_ARGS=--info --stacktrace" >> $GITHUB_ENV
+        fi
+    - name: Fix git ownership
+      run: git status || git config --system --add safe.directory "$GITHUB_WORKSPACE"
+    - name: gradle buildNative
+      shell: bash
+      run: ./gradlew $DEBUG_ARGS --no-daemon -DpklMultiJdkTesting=true -DreleaseBuild=true libpkl:buildNative
+    - name: Upload built artifacts
+      uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
+      with:
+        name: artifacts-libpkl-linux-amd64
+        path: libpkl/build/native-libs/**/*
+    - name: Upload Test Result XML
+      if: '!cancelled()'
+      uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
+      with:
+        name: test-results-xml-libpkl-linux-amd64-release
+        path: '**/build/test-results/**/*.xml'
+    - name: Upload Test Result HTML
+      if: '!cancelled()'
+      uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
+      with:
+        name: test-results-html-libpkl-linux-amd64-release
+        path: '**/build/reports/tests/**/*'
+        if-no-files-found: ignore
+    container:
+      image: redhat/ubi8:8.10
+  libpkl-linux-aarch64-release:
+    runs-on: ubuntu-24.04-arm
+    env:
+      LANG: en_US.UTF-8
+    steps:
+    - name: Install deps
+      run: dnf install -y git binutils gcc glibc-devel zlib-devel libstdc++-static glibc-langpack-en
+    - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+      with:
+        persist-credentials: false
+    - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5
+      with:
+        java-version: '25'
+        distribution: temurin
+        architecture: aarch64
+    - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5
+      with:
+        cache-disabled: true
+    - name: Set DEBUG_ARGS env var
+      env:
+        RUNNER_DEBUG: ${{ runner.debug }}
+      shell: bash
+      run: |-
+        if [[ "$RUNNER_DEBUG" -eq 1 ]]; then
+          echo "DEBUG_ARGS=--info --stacktrace" >> $GITHUB_ENV
+        fi
+    - name: Fix git ownership
+      run: git status || git config --system --add safe.directory "$GITHUB_WORKSPACE"
+    - name: gradle buildNative
+      shell: bash
+      run: ./gradlew $DEBUG_ARGS --no-daemon -DpklMultiJdkTesting=true -DreleaseBuild=true libpkl:buildNative
+    - name: Upload built artifacts
+      uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
+      with:
+        name: artifacts-libpkl-linux-aarch64
+        path: libpkl/build/native-libs/**/*
+    - name: Upload Test Result XML
+      if: '!cancelled()'
+      uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
+      with:
+        name: test-results-xml-libpkl-linux-aarch64-release
+        path: '**/build/test-results/**/*.xml'
+    - name: Upload Test Result HTML
+      if: '!cancelled()'
+      uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
+      with:
+        name: test-results-html-libpkl-linux-aarch64-release
+        path: '**/build/reports/tests/**/*'
+        if-no-files-found: ignore
+    container:
+      image: redhat/ubi8:8.10
+  libpkl-macOS-aarch64-release:
+    if: github.repository_owner == 'apple'
+    runs-on:
+    - self-hosted
+    - macos
+    env:
+      LANG: en_US.UTF-8
+    steps:
+    - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+      with:
+        persist-credentials: false
+    - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5
+      with:
+        java-version: '25'
+        distribution: temurin
+        architecture: aarch64
+    - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5
+      with:
+        cache-disabled: true
+    - name: Set DEBUG_ARGS env var
+      env:
+        RUNNER_DEBUG: ${{ runner.debug }}
+      shell: bash
+      run: |-
+        if [[ "$RUNNER_DEBUG" -eq 1 ]]; then
+          echo "DEBUG_ARGS=--info --stacktrace" >> $GITHUB_ENV
+        fi
+    - name: gradle buildNative
+      shell: bash
+      run: ./gradlew $DEBUG_ARGS --no-daemon -DpklMultiJdkTesting=true -DreleaseBuild=true libpkl:buildNative
+    - name: Upload built artifacts
+      uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
+      with:
+        name: artifacts-libpkl-macOS-aarch64
+        path: libpkl*/build/executable/**/*
+    - name: Upload Test Result XML
+      if: '!cancelled()'
+      uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
+      with:
+        name: test-results-xml-libpkl-macOS-aarch64-release
+        path: '**/build/test-results/**/*.xml'
+    - name: Upload Test Result HTML
+      if: '!cancelled()'
+      uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
+      with:
+        name: test-results-html-libpkl-macOS-aarch64-release
+        path: '**/build/reports/tests/**/*'
+        if-no-files-found: ignore
+  libpkl-alpine-linux-amd64-release:
+    runs-on: ubuntu-latest
+    env:
+      LANG: en_US.UTF-8
+    steps:
+    - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+      with:
+        persist-credentials: false
+    - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5
+      with:
+        java-version: '25'
+        distribution: temurin
+        architecture: x64
+    - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5
+      with:
+        cache-disabled: true
+    - name: Set DEBUG_ARGS env var
+      env:
+        RUNNER_DEBUG: ${{ runner.debug }}
+      shell: bash
+      run: |-
+        if [[ "$RUNNER_DEBUG" -eq 1 ]]; then
+          echo "DEBUG_ARGS=--info --stacktrace" >> $GITHUB_ENV
+        fi
+    - name: Install musl and zlib
+      run: |
+        set -e
+        mkdir -p ~/staticdeps/
+
+        ZLIB_VERSION="1.2.13"
+        MUSL_VERSION="1.2.5"
+
+        # install zlib
+        if [[ ! -f ~/staticdeps/include/zlib.h ]]; then
+          # Download zlib tarball and signature
+          curl -Lf "https://github.com/madler/zlib/releases/download/v${ZLIB_VERSION}/zlib-${ZLIB_VERSION}.tar.gz" -o /tmp/zlib.tar.gz
+          curl -Lf "https://github.com/madler/zlib/releases/download/v${ZLIB_VERSION}/zlib-${ZLIB_VERSION}.tar.gz.asc" -o /tmp/zlib.tar.gz.asc
+
+          # Import zlib GPG key
+          gpg --batch --keyserver keyserver.ubuntu.com --recv-keys 5ED46A6721D365587791E2AA783FCD8E58BCAFBA
+
+          # Verify GPG signature
+          echo "Verifying zlib GPG signature..."
+          gpg --verify /tmp/zlib.tar.gz.asc /tmp/zlib.tar.gz
+
+          mkdir -p "/tmp/dep_zlib-${ZLIB_VERSION}"
+          cd "/tmp/dep_zlib-${ZLIB_VERSION}"
+          # shellcheck disable=SC2002
+          cat /tmp/zlib.tar.gz | tar --strip-components=1 -xzC .
+
+          echo "zlib-${ZLIB_VERSION}: configure..." 
+          ./configure --static --prefix="$HOME"/staticdeps > /dev/null
+
+          echo "zlib-${ZLIB_VERSION}: make..." 
+          make -s -j4
+
+          echo "zlib-${ZLIB_VERSION}: make install..." 
+          make -s install
+
+          rm -rf /tmp/dep_zlib-${ZLIB_VERSION}
+        fi
+
+        # install musl
+        if [[ ! -f ~/staticdeps/bin/x86_64-linux-musl-gcc ]]; then
+          # Download musl tarball and signature
+          curl -Lf "https://musl.libc.org/releases/musl-${MUSL_VERSION}.tar.gz" -o /tmp/musl.tar.gz
+          curl -Lf "https://musl.libc.org/releases/musl-${MUSL_VERSION}.tar.gz.asc" -o /tmp/musl.tar.gz.asc
+
+          # Import musl GPG key
+          gpg --batch --keyserver keyserver.ubuntu.com --recv-keys 836489290BB6B70F99FFDA0556BCDB593020450F
+
+          # Verify GPG signature
+          echo "Verifying musl GPG signature..."
+          gpg --verify /tmp/musl.tar.gz.asc /tmp/musl.tar.gz
+
+          mkdir -p "/tmp/dep_musl-${MUSL_VERSION}"
+          cd "/tmp/dep_musl-${MUSL_VERSION}"
+
+          # shellcheck disable=SC2002
+          cat /tmp/musl.tar.gz | tar --strip-components=1 -xzC .
+
+          echo "musl-${MUSL_VERSION}: configure..." 
+          ./configure --disable-shared --prefix="$HOME"/staticdeps > /dev/null
+
+          echo "musl-${MUSL_VERSION}: make..." 
+          make -s -j4
+
+          echo "musl-${MUSL_VERSION}: make install..." 
+          make -s install
+
+          rm -rf "/tmp/dep_musl-${MUSL_VERSION}"
+
+          # native-image expects to find an executable at this path.
+          ln -s ~/staticdeps/bin/musl-gcc ~/staticdeps/bin/x86_64-linux-musl-gcc
+        fi
+
+        echo "${HOME}/staticdeps/bin" >> "$GITHUB_PATH"
+    - name: gradle buildNative
+      shell: bash
+      run: ./gradlew $DEBUG_ARGS --no-daemon -DpklMultiJdkTesting=true -DreleaseBuild=true -Dpkl.musl=true libpkl:buildNative
+    - name: Upload built artifacts
+      uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
+      with:
+        name: artifacts-libpkl-alpine-linux-amd64
+        path: libpkl*/build/distributions/**/*
+    - name: Upload Test Result XML
+      if: '!cancelled()'
+      uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
+      with:
+        name: test-results-xml-libpkl-alpine-linux-amd64-release
+        path: '**/build/test-results/**/*.xml'
+    - name: Upload Test Result HTML
+      if: '!cancelled()'
+      uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
+      with:
+        name: test-results-html-libpkl-alpine-linux-amd64-release
+        path: '**/build/reports/tests/**/*'
+        if-no-files-found: ignore
+  libpkl-windows-amd64-release:
+    runs-on: windows-latest
+    env:
+      LANG: en_US.UTF-8
+      JAVA_HOME: /jdk
+    steps:
+    - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+      with:
+        persist-credentials: false
+    - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5
+      with:
+        java-version: '25'
+        distribution: temurin
+        architecture: x64
+    - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5
+      with:
+        cache-disabled: true
+    - name: Set DEBUG_ARGS env var
+      env:
+        RUNNER_DEBUG: ${{ runner.debug }}
+      shell: bash
+      run: |-
+        if [[ "$RUNNER_DEBUG" -eq 1 ]]; then
+          echo "DEBUG_ARGS=--info --stacktrace" >> $GITHUB_ENV
+        fi
+    - name: Set up MSVC
+      uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1
+    - name: gradle buildNative
+      shell: bash
+      run: ./gradlew $DEBUG_ARGS --no-daemon -DpklMultiJdkTesting=true -DreleaseBuild=true libpkl:buildNative
+    - name: Upload built artifacts
+      uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
+      with:
+        name: artifacts-libpkl-windows-amd64
+        path: libpkl*/build/distributions/**/*
+    - name: Upload Test Result XML
+      if: '!cancelled()'
+      uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
+      with:
+        name: test-results-xml-libpkl-windows-amd64-release
+        path: '**/build/test-results/**/*.xml'
+    - name: Upload Test Result HTML
+      if: '!cancelled()'
+      uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
+      with:
+        name: test-results-html-libpkl-windows-amd64-release
+        path: '**/build/reports/tests/**/*'
+        if-no-files-found: ignore
   deploy-release:
     needs:
     - gradle-check
@@ -853,6 +1169,11 @@ jobs:
     - pkl-doc-macOS-aarch64-release
     - pkl-doc-alpine-linux-amd64-release
     - pkl-doc-windows-amd64-release
+    - libpkl-linux-amd64-release
+    - libpkl-linux-aarch64-release
+    - libpkl-macOS-aarch64-release
+    - libpkl-alpine-linux-amd64-release
+    - libpkl-windows-amd64-release
     runs-on: ubuntu-latest
     env:
       LANG: en_US.UTF-8
@@ -934,6 +1255,11 @@ jobs:
     - pkl-doc-macOS-aarch64-release
     - pkl-doc-alpine-linux-amd64-release
     - pkl-doc-windows-amd64-release
+    - libpkl-linux-amd64-release
+    - libpkl-linux-aarch64-release
+    - libpkl-macOS-aarch64-release
+    - libpkl-alpine-linux-amd64-release
+    - libpkl-windows-amd64-release
     permissions:
       checks: write
     runs-on: ubuntu-latest
@@ -972,6 +1298,11 @@ jobs:
     - pkl-doc-macOS-aarch64-release
     - pkl-doc-alpine-linux-amd64-release
     - pkl-doc-windows-amd64-release
+    - libpkl-linux-amd64-release
+    - libpkl-linux-aarch64-release
+    - libpkl-macOS-aarch64-release
+    - libpkl-alpine-linux-amd64-release
+    - libpkl-windows-amd64-release
     - deploy-release
     - github-release
     - publish-test-results