.github/workflows: prebuilt integration test artifacts (#2954)

This PR restructures the integration tests and prebuilds all common assets used in all tests:

Headscale and Tailscale HEAD image
hi binary that is used to run tests
go cache is warmed up for compilation of the test
This essentially means we spend 6-10 minutes building assets before any tests starts, when that is done, all tests can just sprint through.

It looks like we are saving 3-9 minutes per test, and since we are limited to running max 20 concurrent tests across the repo, that means we had a lot of double work.

There is currently 113 checks, so we have to do five runs of 20, and the saving should be quite noticeable! I think the "worst case" saving would be 20+min and "best case" probably towards an hour.

.github/workflows/test-integration.yaml

@@ -7,7 +7,117 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 jobs:
+  # build: Builds binaries and Docker images once, uploads as artifacts for reuse.
+  # build-postgres: Pulls postgres image separately to avoid Docker Hub rate limits.
+  # sqlite: Runs all integration tests with SQLite backend.
+  # postgres: Runs a subset of tests with PostgreSQL to verify database compatibility.
+  build:
+    runs-on: ubuntu-latest
+    outputs:
+      files-changed: ${{ steps.changed-files.outputs.files }}
+    steps:
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          fetch-depth: 2
+      - name: Get changed files
+        id: changed-files
+        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
+        with:
+          filters: |
+            files:
+              - '*.nix'
+              - 'go.*'
+              - '**/*.go'
+              - 'integration/**'
+              - 'config-example.yaml'
+              - '.github/workflows/test-integration.yaml'
+              - '.github/workflows/integration-test-template.yml'
+              - 'Dockerfile.*'
+      - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34
+        if: steps.changed-files.outputs.files == 'true'
+      - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3
+        if: steps.changed-files.outputs.files == 'true'
+        with:
+          primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', '**/flake.lock') }}
+          restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }}
+      - name: Build binaries and warm Go cache
+        if: steps.changed-files.outputs.files == 'true'
+        run: |
+          # Build all Go binaries in one nix shell to maximize cache reuse
+          nix develop --command -- bash -c '
+            go build -o hi ./cmd/hi
+            CGO_ENABLED=0 GOOS=linux go build -o headscale ./cmd/headscale
+            # Build integration test binary to warm the cache with all dependencies
+            go test -c ./integration -o /dev/null 2>/dev/null || true
+          '
+      - name: Upload hi binary
+        if: steps.changed-files.outputs.files == 'true'
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
+        with:
+          name: hi-binary
+          path: hi
+          retention-days: 10
+      - name: Package Go cache
+        if: steps.changed-files.outputs.files == 'true'
+        run: |
+          # Package Go module cache and build cache
+          tar -czf go-cache.tar.gz -C ~ go .cache/go-build
+      - name: Upload Go cache
+        if: steps.changed-files.outputs.files == 'true'
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
+        with:
+          name: go-cache
+          path: go-cache.tar.gz
+          retention-days: 10
+      - name: Build headscale image
+        if: steps.changed-files.outputs.files == 'true'
+        run: |
+          docker build \
+            --file Dockerfile.integration-ci \
+            --tag headscale:${{ github.sha }} \
+            .
+          docker save headscale:${{ github.sha }} | gzip > headscale-image.tar.gz
+      - name: Build tailscale HEAD image
+        if: steps.changed-files.outputs.files == 'true'
+        run: |
+          docker build \
+            --file Dockerfile.tailscale-HEAD \
+            --tag tailscale-head:${{ github.sha }} \
+            .
+          docker save tailscale-head:${{ github.sha }} | gzip > tailscale-head-image.tar.gz
+      - name: Upload headscale image
+        if: steps.changed-files.outputs.files == 'true'
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
+        with:
+          name: headscale-image
+          path: headscale-image.tar.gz
+          retention-days: 10
+      - name: Upload tailscale HEAD image
+        if: steps.changed-files.outputs.files == 'true'
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
+        with:
+          name: tailscale-head-image
+          path: tailscale-head-image.tar.gz
+          retention-days: 10
+  build-postgres:
+    runs-on: ubuntu-latest
+    needs: build
+    if: needs.build.outputs.files-changed == 'true'
+    steps:
+      - name: Pull and save postgres image
+        run: |
+          docker pull postgres:latest
+          docker tag postgres:latest postgres:${{ github.sha }}
+          docker save postgres:${{ github.sha }} | gzip > postgres-image.tar.gz
+      - name: Upload postgres image
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
+        with:
+          name: postgres-image
+          path: postgres-image.tar.gz
+          retention-days: 10
   sqlite:
+    needs: build
+    if: needs.build.outputs.files-changed == 'true'
     strategy:
       fail-fast: false
       matrix:
@@ -121,11 +231,14 @@ jobs:
           - TestTagsAdminAPICannotRemoveAllTags
           - TestTagsAdminAPICannotSetInvalidFormat
     uses: ./.github/workflows/integration-test-template.yml
+    secrets: inherit
     with:
       test: ${{ matrix.test }}
       postgres_flag: "--postgres=0"
       database_name: "sqlite"
   postgres:
+    needs: [build, build-postgres]
+    if: needs.build.outputs.files-changed == 'true'
     strategy:
       fail-fast: false
       matrix:
@@ -136,6 +249,7 @@ jobs:
           - TestPingAllByIPManyUpDown
           - TestSubnetRouterMultiNetwork
     uses: ./.github/workflows/integration-test-template.yml
+    secrets: inherit
     with:
       test: ${{ matrix.test }}
       postgres_flag: "--postgres=1"