.github/workflows: prebuilt integration test artifacts (#2954)

This PR restructures the integration tests and prebuilds all common assets used in all tests:

Headscale and Tailscale HEAD image
hi binary that is used to run tests
go cache is warmed up for compilation of the test
This essentially means we spend 6-10 minutes building assets before any tests starts, when that is done, all tests can just sprint through.

It looks like we are saving 3-9 minutes per test, and since we are limited to running max 20 concurrent tests across the repo, that means we had a lot of double work.

There is currently 113 checks, so we have to do five runs of 20, and the saving should be quite noticeable! I think the "worst case" saving would be 20+min and "best case" probably towards an hour.

.github/workflows/build.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     permissions: write-all
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           fetch-depth: 2
       - name: Get changed files
@@ -29,13 +29,12 @@ jobs:
               - '**/*.go'
               - 'integration_test/'
               - 'config-example.yaml'
-      - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31
+      - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34
         if: steps.changed-files.outputs.files == 'true'
       - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3
         if: steps.changed-files.outputs.files == 'true'
         with:
-          primary-key:
+          primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix',
-            nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix',
             '**/flake.lock') }}
           restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }}
 
@@ -55,7 +54,7 @@ jobs:
           exit $BUILD_STATUS
 
       - name: Nix gosum diverging
-        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
         if: failure() && steps.build.outcome == 'failure'
         with:
           github-token: ${{secrets.GITHUB_TOKEN}}
@@ -67,7 +66,7 @@ jobs:
               body: 'Nix build failed with wrong gosum, please update "vendorSha256" (${{ steps.build.outputs.OLD_HASH }}) for the "headscale" package in flake.nix with the new SHA: ${{ steps.build.outputs.NEW_HASH }}'
             })
 
-      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+      - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
         if: steps.changed-files.outputs.files == 'true'
         with:
           name: headscale-linux
@@ -82,22 +81,20 @@ jobs:
           - "GOARCH=arm64 GOOS=darwin"
           - "GOARCH=amd64 GOOS=darwin"
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
-      - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31
+      - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34
       - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3
         with:
-          primary-key:
+          primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix',
-            nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix',
             '**/flake.lock') }}
           restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }}
 
       - name: Run go cross compile
         env:
           CGO_ENABLED: 0
-        run:
+        run: env ${{ matrix.env }} nix develop --command -- go build -o "headscale"
-          env ${{ matrix.env }} nix develop --command -- go build -o "headscale"
           ./cmd/headscale
-      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+      - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
         with:
           name: "headscale-${{ matrix.env }}"
           path: "headscale"

.github/workflows/check-generated.yml

@@ -16,7 +16,7 @@ jobs:
   check-generated:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           fetch-depth: 2
       - name: Get changed files
@@ -31,7 +31,7 @@ jobs:
               - '**/*.proto'
               - 'buf.gen.yaml'
               - 'tools/**'
-      - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31
+      - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34
         if: steps.changed-files.outputs.files == 'true'
       - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3
         if: steps.changed-files.outputs.files == 'true'

.github/workflows/check-tests.yaml

@@ -10,7 +10,7 @@ jobs:
   check-tests:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           fetch-depth: 2
       - name: Get changed files
@@ -24,13 +24,12 @@ jobs:
               - '**/*.go'
               - 'integration_test/'
               - 'config-example.yaml'
-      - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31
+      - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34
         if: steps.changed-files.outputs.files == 'true'
       - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3
         if: steps.changed-files.outputs.files == 'true'
         with:
-          primary-key:
+          primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix',
-            nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix',
             '**/flake.lock') }}
           restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }}
 

.github/workflows/docs-deploy.yml

@@ -21,15 +21,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           fetch-depth: 0
       - name: Install python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
         with:
           python-version: 3.x
       - name: Setup cache
-        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        uses: actions/cache@a7833574556fa59680c1b7cb190c1735db73ebf0 # v5.0.0
         with:
           key: ${{ github.ref }}
           path: .cache

.github/workflows/docs-test.yml

@@ -11,13 +11,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
       - name: Install python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
         with:
           python-version: 3.x
       - name: Setup cache
-        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        uses: actions/cache@a7833574556fa59680c1b7cb190c1735db73ebf0 # v5.0.0
         with:
           key: ${{ github.ref }}
           path: .cache

.github/workflows/gh-actions-updater.yaml

@@ -11,13 +11,13 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           # [Required] Access token with `workflow` scope.
           token: ${{ secrets.WORKFLOW_SECRET }}
 
       - name: Run GitHub Actions Version Updater
-        uses: saadmk11/github-actions-version-updater@64be81ba69383f81f2be476703ea6570c4c8686e # v0.8.1
+        uses: saadmk11/github-actions-version-updater@d8781caf11d11168579c8e5e94f62b068038f442 # v0.9.0
         with:
           # [Required] Access token with `workflow` scope.
           token: ${{ secrets.WORKFLOW_SECRET }}

.github/workflows/integration-test-template.yml

@@ -28,23 +28,12 @@ jobs:
       # that triggered the build.
       HAS_TAILSCALE_SECRET: ${{ secrets.TS_OAUTH_CLIENT_ID }}
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           fetch-depth: 2
-      - name: Get changed files
-        id: changed-files
-        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
-        with:
-          filters: |
-            files:
-              - '*.nix'
-              - 'go.*'
-              - '**/*.go'
-              - 'integration_test/'
-              - 'config-example.yaml'
       - name: Tailscale
         if: ${{ env.HAS_TAILSCALE_SECRET }}
-        uses: tailscale/github-action@6986d2c82a91fbac2949fe01f5bab95cf21b5102 # v3.2.2
+        uses: tailscale/github-action@a392da0a182bba0e9613b6243ebd69529b1878aa # v4.1.0
         with:
           oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
           oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
@@ -52,28 +41,62 @@ jobs:
       - name: Setup SSH server for Actor
         if: ${{ env.HAS_TAILSCALE_SECRET }}
         uses: alexellis/setup-sshd-actor@master
-      - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31
+      - name: Download headscale image
-        if: steps.changed-files.outputs.files == 'true'
+        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
-      - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3
-        if: steps.changed-files.outputs.files == 'true'
         with:
-          primary-key:
+          name: headscale-image
-            nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix',
+          path: /tmp/artifacts
-            '**/flake.lock') }}
+      - name: Download tailscale HEAD image
-          restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }}
+        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
+        with:
+          name: tailscale-head-image
+          path: /tmp/artifacts
+      - name: Download hi binary
+        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
+        with:
+          name: hi-binary
+          path: /tmp/artifacts
+      - name: Download Go cache
+        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
+        with:
+          name: go-cache
+          path: /tmp/artifacts
+      - name: Download postgres image
+        if: ${{ inputs.postgres_flag == '--postgres=1' }}
+        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
+        with:
+          name: postgres-image
+          path: /tmp/artifacts
+      - name: Load Docker images, Go cache, and prepare binary
+        run: |
+          gunzip -c /tmp/artifacts/headscale-image.tar.gz | docker load
+          gunzip -c /tmp/artifacts/tailscale-head-image.tar.gz | docker load
+          if [ -f /tmp/artifacts/postgres-image.tar.gz ]; then
+            gunzip -c /tmp/artifacts/postgres-image.tar.gz | docker load
+          fi
+          chmod +x /tmp/artifacts/hi
+          docker images
+          # Extract Go cache to host directories for bind mounting
+          mkdir -p /tmp/go-cache
+          tar -xzf /tmp/artifacts/go-cache.tar.gz -C /tmp/go-cache
+          ls -la /tmp/go-cache/ /tmp/go-cache/.cache/
       - name: Run Integration Test
-        if: always() && steps.changed-files.outputs.files == 'true'
+        env:
-        run:
+          HEADSCALE_INTEGRATION_HEADSCALE_IMAGE: headscale:${{ github.sha }}
-          nix develop --command -- hi run --stats --ts-memory-limit=300 --hs-memory-limit=1500 "^${{ inputs.test }}$" \
+          HEADSCALE_INTEGRATION_TAILSCALE_IMAGE: tailscale-head:${{ github.sha }}
+          HEADSCALE_INTEGRATION_POSTGRES_IMAGE: ${{ inputs.postgres_flag == '--postgres=1' && format('postgres:{0}', github.sha) || '' }}
+          HEADSCALE_INTEGRATION_GO_CACHE: /tmp/go-cache/go
+          HEADSCALE_INTEGRATION_GO_BUILD_CACHE: /tmp/go-cache/.cache/go-build
+        run: /tmp/artifacts/hi run --stats --ts-memory-limit=300 --hs-memory-limit=1500 "^${{ inputs.test }}$" \
           --timeout=120m \
           ${{ inputs.postgres_flag }}
-      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+      - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
-        if: always() && steps.changed-files.outputs.files == 'true'
+        if: always()
         with:
           name: ${{ inputs.database_name }}-${{ inputs.test }}-logs
           path: "control_logs/*/*.log"
-      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+      - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
-        if: always() && steps.changed-files.outputs.files == 'true'
+        if: always()
         with:
           name: ${{ inputs.database_name }}-${{ inputs.test }}-artifacts
           path: control_logs/

.github/workflows/lint.yml

@@ -10,7 +10,7 @@ jobs:
   golangci-lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           fetch-depth: 2
       - name: Get changed files
@@ -24,13 +24,12 @@ jobs:
               - '**/*.go'
               - 'integration_test/'
               - 'config-example.yaml'
-      - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31
+      - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34
         if: steps.changed-files.outputs.files == 'true'
       - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3
         if: steps.changed-files.outputs.files == 'true'
         with:
-          primary-key:
+          primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix',
-            nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix',
             '**/flake.lock') }}
           restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }}
 
@@ -46,7 +45,7 @@ jobs:
   prettier-lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           fetch-depth: 2
       - name: Get changed files
@@ -65,13 +64,12 @@ jobs:
               - '**/*.css'
               - '**/*.scss'
               - '**/*.html'
-      - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31
+      - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34
         if: steps.changed-files.outputs.files == 'true'
       - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3
         if: steps.changed-files.outputs.files == 'true'
         with:
-          primary-key:
+          primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix',
-            nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix',
             '**/flake.lock') }}
           restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }}
 
@@ -83,12 +81,11 @@ jobs:
   proto-lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
-      - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31
+      - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34
       - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3
         with:
-          primary-key:
+          primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix',
-            nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix',
             '**/flake.lock') }}
           restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }}
 

.github/workflows/nix-module-test.yml

@@ -19,7 +19,7 @@ jobs:
       contents: read
 
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           fetch-depth: 2
 
@@ -38,14 +38,13 @@ jobs:
               - 'cmd/**'
               - 'hscontrol/**'
 
-      - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31
+      - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34
         if: steps.changed-files.outputs.nix == 'true' || steps.changed-files.outputs.go == 'true'
 
       - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3
         if: steps.changed-files.outputs.nix == 'true' || steps.changed-files.outputs.go == 'true'
         with:
-          primary-key:
+          primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix',
-            nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix',
             '**/flake.lock') }}
           restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }}
 

.github/workflows/release.yml

@@ -13,28 +13,27 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           fetch-depth: 0
 
       - name: Login to DockerHub
-        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Login to GHCR
-        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31
+      - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34
       - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3
         with:
-          primary-key:
+          primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix',
-            nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix',
             '**/flake.lock') }}
           restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }}
 

.github/workflows/stale.yml

@@ -12,16 +12,14 @@ jobs:
       issues: write
       pull-requests: write
     steps:
-      - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0
+      - uses: actions/stale@997185467fa4f803885201cee163a9f38240193d # v10.1.1
         with:
           days-before-issue-stale: 90
           days-before-issue-close: 7
           stale-issue-label: "stale"
-          stale-issue-message:
+          stale-issue-message: "This issue is stale because it has been open for 90 days with no
-            "This issue is stale because it has been open for 90 days with no
             activity."
-          close-issue-message:
+          close-issue-message: "This issue was closed because it has been inactive for 14 days
-            "This issue was closed because it has been inactive for 14 days
             since being marked as stale."
           days-before-pr-stale: -1
           days-before-pr-close: -1

.github/workflows/test-integration.yaml

@@ -7,7 +7,117 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 jobs:
+  # build: Builds binaries and Docker images once, uploads as artifacts for reuse.
+  # build-postgres: Pulls postgres image separately to avoid Docker Hub rate limits.
+  # sqlite: Runs all integration tests with SQLite backend.
+  # postgres: Runs a subset of tests with PostgreSQL to verify database compatibility.
+  build:
+    runs-on: ubuntu-latest
+    outputs:
+      files-changed: ${{ steps.changed-files.outputs.files }}
+    steps:
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          fetch-depth: 2
+      - name: Get changed files
+        id: changed-files
+        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
+        with:
+          filters: |
+            files:
+              - '*.nix'
+              - 'go.*'
+              - '**/*.go'
+              - 'integration/**'
+              - 'config-example.yaml'
+              - '.github/workflows/test-integration.yaml'
+              - '.github/workflows/integration-test-template.yml'
+              - 'Dockerfile.*'
+      - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34
+        if: steps.changed-files.outputs.files == 'true'
+      - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3
+        if: steps.changed-files.outputs.files == 'true'
+        with:
+          primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', '**/flake.lock') }}
+          restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }}
+      - name: Build binaries and warm Go cache
+        if: steps.changed-files.outputs.files == 'true'
+        run: |
+          # Build all Go binaries in one nix shell to maximize cache reuse
+          nix develop --command -- bash -c '
+            go build -o hi ./cmd/hi
+            CGO_ENABLED=0 GOOS=linux go build -o headscale ./cmd/headscale
+            # Build integration test binary to warm the cache with all dependencies
+            go test -c ./integration -o /dev/null 2>/dev/null || true
+          '
+      - name: Upload hi binary
+        if: steps.changed-files.outputs.files == 'true'
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
+        with:
+          name: hi-binary
+          path: hi
+          retention-days: 10
+      - name: Package Go cache
+        if: steps.changed-files.outputs.files == 'true'
+        run: |
+          # Package Go module cache and build cache
+          tar -czf go-cache.tar.gz -C ~ go .cache/go-build
+      - name: Upload Go cache
+        if: steps.changed-files.outputs.files == 'true'
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
+        with:
+          name: go-cache
+          path: go-cache.tar.gz
+          retention-days: 10
+      - name: Build headscale image
+        if: steps.changed-files.outputs.files == 'true'
+        run: |
+          docker build \
+            --file Dockerfile.integration-ci \
+            --tag headscale:${{ github.sha }} \
+            .
+          docker save headscale:${{ github.sha }} | gzip > headscale-image.tar.gz
+      - name: Build tailscale HEAD image
+        if: steps.changed-files.outputs.files == 'true'
+        run: |
+          docker build \
+            --file Dockerfile.tailscale-HEAD \
+            --tag tailscale-head:${{ github.sha }} \
+            .
+          docker save tailscale-head:${{ github.sha }} | gzip > tailscale-head-image.tar.gz
+      - name: Upload headscale image
+        if: steps.changed-files.outputs.files == 'true'
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
+        with:
+          name: headscale-image
+          path: headscale-image.tar.gz
+          retention-days: 10
+      - name: Upload tailscale HEAD image
+        if: steps.changed-files.outputs.files == 'true'
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
+        with:
+          name: tailscale-head-image
+          path: tailscale-head-image.tar.gz
+          retention-days: 10
+  build-postgres:
+    runs-on: ubuntu-latest
+    needs: build
+    if: needs.build.outputs.files-changed == 'true'
+    steps:
+      - name: Pull and save postgres image
+        run: |
+          docker pull postgres:latest
+          docker tag postgres:latest postgres:${{ github.sha }}
+          docker save postgres:${{ github.sha }} | gzip > postgres-image.tar.gz
+      - name: Upload postgres image
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
+        with:
+          name: postgres-image
+          path: postgres-image.tar.gz
+          retention-days: 10
   sqlite:
+    needs: build
+    if: needs.build.outputs.files-changed == 'true'
     strategy:
       fail-fast: false
       matrix:
@@ -121,11 +231,14 @@ jobs:
           - TestTagsAdminAPICannotRemoveAllTags
           - TestTagsAdminAPICannotSetInvalidFormat
     uses: ./.github/workflows/integration-test-template.yml
+    secrets: inherit
     with:
       test: ${{ matrix.test }}
       postgres_flag: "--postgres=0"
       database_name: "sqlite"
   postgres:
+    needs: [build, build-postgres]
+    if: needs.build.outputs.files-changed == 'true'
     strategy:
       fail-fast: false
       matrix:
@@ -136,6 +249,7 @@ jobs:
           - TestPingAllByIPManyUpDown
           - TestSubnetRouterMultiNetwork
     uses: ./.github/workflows/integration-test-template.yml
+    secrets: inherit
     with:
       test: ${{ matrix.test }}
       postgres_flag: "--postgres=1"

.github/workflows/test.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           fetch-depth: 2
 
@@ -27,13 +27,12 @@ jobs:
               - 'integration_test/'
               - 'config-example.yaml'
 
-      - uses: nixbuild/nix-quick-install-action@889f3180bb5f064ee9e3201428d04ae9e41d54ad # v31
+      - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34
         if: steps.changed-files.outputs.files == 'true'
       - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3
         if: steps.changed-files.outputs.files == 'true'
         with:
-          primary-key:
+          primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix',
-            nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix',
             '**/flake.lock') }}
           restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }}
 

Dockerfile.integration

@@ -2,27 +2,43 @@
 # and are in no way endorsed by Headscale's maintainers as an
 # official nor supported release or distribution.
 
-FROM docker.io/golang:1.25-trixie
+FROM docker.io/golang:1.25-trixie AS builder
 ARG VERSION=dev
 ENV GOPATH /go
 WORKDIR /go/src/headscale
 
-RUN apt-get --update install --no-install-recommends --yes less jq sqlite3 dnsutils \
+# Install delve debugger first - rarely changes, good cache candidate
-  && apt-get dist-clean
-RUN mkdir -p /var/run/headscale
-
-# Install delve debugger
 RUN go install github.com/go-delve/delve/cmd/dlv@latest
 
+# Download dependencies - only invalidated when go.mod/go.sum change
 COPY go.mod go.sum /go/src/headscale/
 RUN go mod download
 
+# Copy source and build - invalidated on any source change
 COPY . .
 
 # Build debug binary with debug symbols for delve
 RUN CGO_ENABLED=0 GOOS=linux go build -gcflags="all=-N -l" -o /go/bin/headscale ./cmd/headscale
 
+# Runtime stage
+FROM debian:trixie-slim
+
+RUN apt-get --update install --no-install-recommends --yes \
+    less jq sqlite3 dnsutils ca-certificates procps bash findutils curl traceroute python3 \
+  && apt-get dist-clean
+
+RUN mkdir -p /var/run/headscale
+
+# Copy binaries from builder
+COPY --from=builder /go/bin/headscale /usr/local/bin/headscale
+COPY --from=builder /go/bin/dlv /usr/local/bin/dlv
+
+# Copy source code for delve source-level debugging
+COPY --from=builder /go/src/headscale /go/src/headscale
+
+WORKDIR /go/src/headscale
+
 # Need to reset the entrypoint or everything will run as a busybox script
 ENTRYPOINT []
 EXPOSE 8080/tcp 40000/tcp
-CMD ["/go/bin/dlv", "--listen=0.0.0.0:40000", "--headless=true", "--api-version=2", "--accept-multiclient", "exec", "/go/bin/headscale", "--"]
+CMD ["dlv", "--listen=0.0.0.0:40000", "--headless=true", "--api-version=2", "--accept-multiclient", "exec", "/usr/local/bin/headscale", "--"]

Dockerfile.integration-ci

@@ -0,0 +1,17 @@
+# Minimal CI image - expects pre-built headscale binary in build context
+# For local development with delve debugging, use Dockerfile.integration instead
+
+FROM debian:trixie-slim
+
+RUN apt-get --update install --no-install-recommends --yes \
+    less jq sqlite3 dnsutils ca-certificates procps bash findutils curl traceroute \
+  && apt-get dist-clean
+
+RUN mkdir -p /var/run/headscale
+
+# Copy pre-built headscale binary from build context
+COPY headscale /usr/local/bin/headscale
+
+ENTRYPOINT []
+EXPOSE 8080/tcp
+CMD ["/usr/local/bin/headscale"]

Dockerfile.tailscale-HEAD

@@ -37,7 +37,7 @@ RUN GOARCH=$TARGETARCH go install -tags="${BUILD_TAGS}" -ldflags="\
       -v ./cmd/tailscale ./cmd/tailscaled ./cmd/containerboot
 
 FROM alpine:3.22
-RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables curl
+RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables curl traceroute
 
 COPY --from=build-env /go/bin/* /usr/local/bin/
 # For compat with the previous run.sh, although ideally you should be

cmd/hi/docker.go

@@ -301,6 +301,11 @@ func createGoTestContainer(ctx context.Context, cli *client.Client, config *RunC
 		"HEADSCALE_INTEGRATION_RUN_ID=" + runID,
 	}
 
+	// Pass through CI environment variable for CI detection
+	if ci := os.Getenv("CI"); ci != "" {
+		env = append(env, "CI="+ci)
+	}
+
 	// Pass through all HEADSCALE_INTEGRATION_* environment variables
 	for _, e := range os.Environ() {
 		if strings.HasPrefix(e, "HEADSCALE_INTEGRATION_") {
@@ -313,6 +318,10 @@ func createGoTestContainer(ctx context.Context, cli *client.Client, config *RunC
 			env = append(env, e)
 		}
 	}
+
+	// Set GOCACHE to a known location (used by both bind mount and volume cases)
+	env = append(env, "GOCACHE=/cache/go-build")
+
 	containerConfig := &container.Config{
 		Image:      "golang:" + config.GoVersion,
 		Cmd:        goTestCmd,
@@ -332,20 +341,43 @@ func createGoTestContainer(ctx context.Context, cli *client.Client, config *RunC
 		log.Printf("Using Docker socket: %s", dockerSocketPath)
 	}
 
+	binds := []string{
+		fmt.Sprintf("%s:%s", projectRoot, projectRoot),
+		dockerSocketPath + ":/var/run/docker.sock",
+		logsDir + ":/tmp/control",
+	}
+
+	// Use bind mounts for Go cache if provided via environment variables,
+	// otherwise fall back to Docker volumes for local development
+	var mounts []mount.Mount
+
+	goCache := os.Getenv("HEADSCALE_INTEGRATION_GO_CACHE")
+	goBuildCache := os.Getenv("HEADSCALE_INTEGRATION_GO_BUILD_CACHE")
+
+	if goCache != "" {
+		binds = append(binds, goCache+":/go")
+	} else {
+		mounts = append(mounts, mount.Mount{
+			Type:   mount.TypeVolume,
+			Source: "hs-integration-go-cache",
+			Target: "/go",
+		})
+	}
+
+	if goBuildCache != "" {
+		binds = append(binds, goBuildCache+":/cache/go-build")
+	} else {
+		mounts = append(mounts, mount.Mount{
+			Type:   mount.TypeVolume,
+			Source: "hs-integration-go-build-cache",
+			Target: "/cache/go-build",
+		})
+	}
+
 	hostConfig := &container.HostConfig{
 		AutoRemove: false, // We'll remove manually for better control
-		Binds: []string{
+		Binds:      binds,
-			fmt.Sprintf("%s:%s", projectRoot, projectRoot),
+		Mounts:     mounts,
-			dockerSocketPath + ":/var/run/docker.sock",
-			logsDir + ":/tmp/control",
-		},
-		Mounts: []mount.Mount{
-			{
-				Type:   mount.TypeVolume,
-				Source: "hs-integration-go-cache",
-				Target: "/go",
-			},
-		},
 	}
 
 	return cli.ContainerCreate(ctx, containerConfig, hostConfig, nil, nil, containerName)

integration/hsic/hsic.go

@@ -33,7 +33,6 @@ import (
 	"github.com/ory/dockertest/v3"
 	"github.com/ory/dockertest/v3/docker"
 	"gopkg.in/yaml.v3"
-	"tailscale.com/envknob"
 	"tailscale.com/tailcfg"
 	"tailscale.com/util/mak"
 )
@@ -49,7 +48,12 @@ const (
 	IntegrationTestDockerFileName = "Dockerfile.integration"
 )
 
-var errHeadscaleStatusCodeNotOk = errors.New("headscale status code not ok")
+var (
+	errHeadscaleStatusCodeNotOk    = errors.New("headscale status code not ok")
+	errInvalidHeadscaleImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_HEADSCALE_IMAGE format, expected repository:tag")
+	errHeadscaleImageRequiredInCI  = errors.New("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE must be set in CI")
+	errInvalidPostgresImageFormat  = errors.New("invalid HEADSCALE_INTEGRATION_POSTGRES_IMAGE format, expected repository:tag")
+)
 
 type fileInContainer struct {
 	path     string
@@ -70,7 +74,6 @@ type HeadscaleInContainer struct {
 	// optional config
 	port             int
 	extraPorts       []string
-	debugPort        int
 	caCerts          [][]byte
 	hostPortBindings map[string][]string
 	aclPolicy        *policyv2.Policy
@@ -281,24 +284,9 @@ func WithDERPAsIP() Option {
 	}
 }
 
-// WithDebugPort sets the debug port for delve debugging.
-func WithDebugPort(port int) Option {
-	return func(hsic *HeadscaleInContainer) {
-		hsic.debugPort = port
-	}
-}
-
 // buildEntrypoint builds the container entrypoint command based on configuration.
 func (hsic *HeadscaleInContainer) buildEntrypoint() []string {
-	debugCmd := fmt.Sprintf(
+	entrypoint := "/bin/sleep 3 ; update-ca-certificates ; /usr/local/bin/headscale serve ; /bin/sleep 30"
-		"/go/bin/dlv --listen=0.0.0.0:%d --headless=true --api-version=2 --accept-multiclient --allow-non-terminal-interactive=true exec /go/bin/headscale --continue -- serve",
-		hsic.debugPort,
-	)
-
-	entrypoint := fmt.Sprintf(
-		"/bin/sleep 3 ; update-ca-certificates ; %s ; /bin/sleep 30",
-		debugCmd,
-	)
 
 	return []string{"/bin/bash", "-c", entrypoint}
 }
@@ -316,18 +304,9 @@ func New(
 
 	hostname := "hs-" + hash
 
-	// Get debug port from environment or use default
-	debugPort := 40000
-	if envDebugPort := envknob.String("HEADSCALE_DEBUG_PORT"); envDebugPort != "" {
-		if port, err := strconv.Atoi(envDebugPort); err == nil {
-			debugPort = port
-		}
-	}
-
 	hsic := &HeadscaleInContainer{
-		hostname:  hostname,
+		hostname: hostname,
-		port:      headscaleDefaultPort,
+		port:     headscaleDefaultPort,
-		debugPort: debugPort,
 
 		pool:     pool,
 		networks: networks,
@@ -344,7 +323,6 @@ func New(
 	log.Println("NAME: ", hsic.hostname)
 
 	portProto := fmt.Sprintf("%d/tcp", hsic.port)
-	debugPortProto := fmt.Sprintf("%d/tcp", hsic.debugPort)
 
 	headscaleBuildOptions := &dockertest.BuildOptions{
 		Dockerfile: IntegrationTestDockerFileName,
@@ -359,10 +337,24 @@ func New(
 		hsic.env["HEADSCALE_DATABASE_POSTGRES_NAME"] = "headscale"
 		delete(hsic.env, "HEADSCALE_DATABASE_SQLITE_PATH")
 
+		// Determine postgres image - use prebuilt if available, otherwise pull from registry
+		pgRepo := "postgres"
+		pgTag := "latest"
+
+		if prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_POSTGRES_IMAGE"); prebuiltImage != "" {
+			repo, tag, found := strings.Cut(prebuiltImage, ":")
+			if !found {
+				return nil, errInvalidPostgresImageFormat
+			}
+
+			pgRepo = repo
+			pgTag = tag
+		}
+
 		pgRunOptions := &dockertest.RunOptions{
 			Name:       "postgres-" + hash,
-			Repository: "postgres",
+			Repository: pgRepo,
-			Tag:        "latest",
+			Tag:        pgTag,
 			Networks:   networks,
 			Env: []string{
 				"POSTGRES_USER=headscale",
@@ -409,7 +401,7 @@ func New(
 
 	runOptions := &dockertest.RunOptions{
 		Name:         hsic.hostname,
-		ExposedPorts: append([]string{portProto, debugPortProto, "9090/tcp"}, hsic.extraPorts...),
+		ExposedPorts: append([]string{portProto, "9090/tcp"}, hsic.extraPorts...),
 		Networks:     networks,
 		// Cmd:          []string{"headscale", "serve"},
 		// TODO(kradalby): Get rid of this hack, we currently need to give us some
@@ -418,13 +410,10 @@ func New(
 		Env:        env,
 	}
 
-	// Always bind debug port and metrics port to predictable host ports
+	// Bind metrics port to predictable host port
 	if runOptions.PortBindings == nil {
 		runOptions.PortBindings = map[docker.Port][]docker.PortBinding{}
 	}
-	runOptions.PortBindings[docker.Port(debugPortProto)] = []docker.PortBinding{
-		{HostPort: strconv.Itoa(hsic.debugPort)},
-	}
 	runOptions.PortBindings["9090/tcp"] = []docker.PortBinding{
 		{HostPort: "49090"},
 	}
@@ -451,52 +440,80 @@ func New(
 	// Add integration test labels if running under hi tool
 	dockertestutil.DockerAddIntegrationLabels(runOptions, "headscale")
 
-	container, err := pool.BuildAndRunWithBuildOptions(
+	var container *dockertest.Resource
-		headscaleBuildOptions,
-		runOptions,
-		dockertestutil.DockerRestartPolicy,
-		dockertestutil.DockerAllowLocalIPv6,
-		dockertestutil.DockerAllowNetworkAdministration,
-	)
-	if err != nil {
-		// Try to get more detailed build output
-		log.Printf("Docker build failed, attempting to get detailed output...")
 
-		buildOutput, buildErr := dockertestutil.RunDockerBuildForDiagnostics(dockerContextPath, IntegrationTestDockerFileName)
+	// Check if a pre-built image is available via environment variable
+	prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE")
 
-		// Show the last 100 lines of build output to avoid overwhelming the logs
+	if prebuiltImage != "" {
-		lines := strings.Split(buildOutput, "\n")
+		log.Printf("Using pre-built headscale image: %s", prebuiltImage)
-
+		// Parse image into repository and tag
-		const maxLines = 100
+		repo, tag, ok := strings.Cut(prebuiltImage, ":")
-
+		if !ok {
-		startLine := 0
+			return nil, errInvalidHeadscaleImageFormat
-		if len(lines) > maxLines {
-			startLine = len(lines) - maxLines
 		}
 
-		relevantOutput := strings.Join(lines[startLine:], "\n")
+		runOptions.Repository = repo
+		runOptions.Tag = tag
 
-		if buildErr != nil {
+		container, err = pool.RunWithOptions(
-			// The diagnostic build also failed - this is the real error
+			runOptions,
-			return nil, fmt.Errorf("could not start headscale container: %w\n\nDocker build failed. Last %d lines of output:\n%s", err, maxLines, relevantOutput)
+			dockertestutil.DockerRestartPolicy,
+			dockertestutil.DockerAllowLocalIPv6,
+			dockertestutil.DockerAllowNetworkAdministration,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("could not run pre-built headscale container %q: %w", prebuiltImage, err)
 		}
+	} else if util.IsCI() {
+		return nil, errHeadscaleImageRequiredInCI
+	} else {
+		container, err = pool.BuildAndRunWithBuildOptions(
+			headscaleBuildOptions,
+			runOptions,
+			dockertestutil.DockerRestartPolicy,
+			dockertestutil.DockerAllowLocalIPv6,
+			dockertestutil.DockerAllowNetworkAdministration,
+		)
+		if err != nil {
+			// Try to get more detailed build output
+			log.Printf("Docker build/run failed, attempting to get detailed output...")
 
-		if buildOutput != "" {
+			buildOutput, buildErr := dockertestutil.RunDockerBuildForDiagnostics(dockerContextPath, IntegrationTestDockerFileName)
-			// Build succeeded on retry but container creation still failed
+
-			return nil, fmt.Errorf("could not start headscale container: %w\n\nDocker build succeeded on retry, but container creation failed. Last %d lines of build output:\n%s", err, maxLines, relevantOutput)
+			// Show the last 100 lines of build output to avoid overwhelming the logs
+			lines := strings.Split(buildOutput, "\n")
+
+			const maxLines = 100
+
+			startLine := 0
+			if len(lines) > maxLines {
+				startLine = len(lines) - maxLines
+			}
+
+			relevantOutput := strings.Join(lines[startLine:], "\n")
+
+			if buildErr != nil {
+				// The diagnostic build also failed - this is the real error
+				return nil, fmt.Errorf("could not start headscale container: %w\n\nDocker build failed. Last %d lines of output:\n%s", err, maxLines, relevantOutput)
+			}
+
+			if buildOutput != "" {
+				// Build succeeded on retry but container creation still failed
+				return nil, fmt.Errorf("could not start headscale container: %w\n\nDocker build succeeded on retry, but container creation failed. Last %d lines of build output:\n%s", err, maxLines, relevantOutput)
+			}
+
+			// No output at all - diagnostic build command may have failed
+			return nil, fmt.Errorf("could not start headscale container: %w\n\nUnable to get diagnostic build output (command may have failed silently)", err)
 		}
-
-		// No output at all - diagnostic build command may have failed
-		return nil, fmt.Errorf("could not start headscale container: %w\n\nUnable to get diagnostic build output (command may have failed silently)", err)
 	}
 	log.Printf("Created %s container\n", hsic.hostname)
 
 	hsic.container = container
 
 	log.Printf(
-		"Debug ports for %s: delve=%s, metrics/pprof=49090\n",
+		"Ports for %s: metrics/pprof=49090\n",
 		hsic.hostname,
-		hsic.GetHostDebugPort(),
 	)
 
 	// Write the CA certificates to the container
@@ -886,16 +903,6 @@ func (t *HeadscaleInContainer) GetPort() string {
 	return strconv.Itoa(t.port)
 }
 
-// GetDebugPort returns the debug port as a string.
-func (t *HeadscaleInContainer) GetDebugPort() string {
-	return strconv.Itoa(t.debugPort)
-}
-
-// GetHostDebugPort returns the host port mapped to the debug port.
-func (t *HeadscaleInContainer) GetHostDebugPort() string {
-	return strconv.Itoa(t.debugPort)
-}
-
 // GetHealthEndpoint returns a health endpoint for the HeadscaleInContainer
 // instance.
 func (t *HeadscaleInContainer) GetHealthEndpoint() string {

integration/tsic/tsic.go

@@ -54,6 +54,8 @@ var (
 	errTailscaleNotConnected           = errors.New("tailscale not connected")
 	errTailscaledNotReadyForLogin      = errors.New("tailscaled not ready for login")
 	errInvalidClientConfig             = errors.New("verifiably invalid client config requested")
+	errInvalidTailscaleImageFormat     = errors.New("invalid HEADSCALE_INTEGRATION_TAILSCALE_IMAGE format, expected repository:tag")
+	errTailscaleImageRequiredInCI      = errors.New("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE must be set in CI for HEAD version")
 )
 
 const (
@@ -299,80 +301,119 @@ func New(
 
 	switch version {
 	case VersionHead:
-		buildOptions := &dockertest.BuildOptions{
+		// Check if a pre-built image is available via environment variable
-			Dockerfile: "Dockerfile.tailscale-HEAD",
+		prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE")
-			ContextDir: dockerContextPath,
+
-			BuildArgs:  []docker.BuildArg{},
+		// If custom build tags are required (e.g., for websocket DERP), we cannot use
+		// the pre-built image as it won't have the necessary code compiled in.
+		hasBuildTags := len(tsic.buildConfig.tags) > 0
+		if hasBuildTags && prebuiltImage != "" {
+			log.Printf("Ignoring pre-built image %s because custom build tags are required: %v",
+				prebuiltImage, tsic.buildConfig.tags)
+			prebuiltImage = ""
 		}
 
-		buildTags := strings.Join(tsic.buildConfig.tags, ",")
+		if prebuiltImage != "" {
-		if len(buildTags) > 0 {
+			log.Printf("Using pre-built tailscale image: %s", prebuiltImage)
-			buildOptions.BuildArgs = append(
-				buildOptions.BuildArgs,
-				docker.BuildArg{
-					Name:  "BUILD_TAGS",
-					Value: buildTags,
-				},
-			)
-		}
 
-		container, err = pool.BuildAndRunWithBuildOptions(
+			// Parse image into repository and tag
-			buildOptions,
+			repo, tag, ok := strings.Cut(prebuiltImage, ":")
-			tailscaleOptions,
+			if !ok {
-			dockertestutil.DockerRestartPolicy,
+				return nil, errInvalidTailscaleImageFormat
-			dockertestutil.DockerAllowLocalIPv6,
-			dockertestutil.DockerAllowNetworkAdministration,
-			dockertestutil.DockerMemoryLimit,
-		)
-		if err != nil {
-			// Try to get more detailed build output
-			log.Printf("Docker build failed for %s, attempting to get detailed output...", hostname)
-
-			buildOutput, buildErr := dockertestutil.RunDockerBuildForDiagnostics(dockerContextPath, "Dockerfile.tailscale-HEAD")
-
-			// Show the last 100 lines of build output to avoid overwhelming the logs
-			lines := strings.Split(buildOutput, "\n")
-
-			const maxLines = 100
-
-			startLine := 0
-			if len(lines) > maxLines {
-				startLine = len(lines) - maxLines
 			}
 
-			relevantOutput := strings.Join(lines[startLine:], "\n")
+			tailscaleOptions.Repository = repo
+			tailscaleOptions.Tag = tag
 
-			if buildErr != nil {
+			container, err = pool.RunWithOptions(
-				// The diagnostic build also failed - this is the real error
+				tailscaleOptions,
-				return nil, fmt.Errorf(
+				dockertestutil.DockerRestartPolicy,
-					"%s could not start tailscale container (version: %s): %w\n\nDocker build failed. Last %d lines of output:\n%s",
+				dockertestutil.DockerAllowLocalIPv6,
-					hostname,
+				dockertestutil.DockerAllowNetworkAdministration,
-					version,
+				dockertestutil.DockerMemoryLimit,
-					err,
+			)
-					maxLines,
+			if err != nil {
-					relevantOutput,
+				return nil, fmt.Errorf("could not run pre-built tailscale container %q: %w", prebuiltImage, err)
+			}
+		} else if util.IsCI() && !hasBuildTags {
+			// In CI, we require a pre-built image unless custom build tags are needed
+			return nil, errTailscaleImageRequiredInCI
+		} else {
+			buildOptions := &dockertest.BuildOptions{
+				Dockerfile: "Dockerfile.tailscale-HEAD",
+				ContextDir: dockerContextPath,
+				BuildArgs:  []docker.BuildArg{},
+			}
+
+			buildTags := strings.Join(tsic.buildConfig.tags, ",")
+			if len(buildTags) > 0 {
+				buildOptions.BuildArgs = append(
+					buildOptions.BuildArgs,
+					docker.BuildArg{
+						Name:  "BUILD_TAGS",
+						Value: buildTags,
+					},
 				)
 			}
 
-			if buildOutput != "" {
+			container, err = pool.BuildAndRunWithBuildOptions(
-				// Build succeeded on retry but container creation still failed
+				buildOptions,
+				tailscaleOptions,
+				dockertestutil.DockerRestartPolicy,
+				dockertestutil.DockerAllowLocalIPv6,
+				dockertestutil.DockerAllowNetworkAdministration,
+				dockertestutil.DockerMemoryLimit,
+			)
+			if err != nil {
+				// Try to get more detailed build output
+				log.Printf("Docker build failed for %s, attempting to get detailed output...", hostname)
+
+				buildOutput, buildErr := dockertestutil.RunDockerBuildForDiagnostics(dockerContextPath, "Dockerfile.tailscale-HEAD")
+
+				// Show the last 100 lines of build output to avoid overwhelming the logs
+				lines := strings.Split(buildOutput, "\n")
+
+				const maxLines = 100
+
+				startLine := 0
+				if len(lines) > maxLines {
+					startLine = len(lines) - maxLines
+				}
+
+				relevantOutput := strings.Join(lines[startLine:], "\n")
+
+				if buildErr != nil {
+					// The diagnostic build also failed - this is the real error
+					return nil, fmt.Errorf(
+						"%s could not start tailscale container (version: %s): %w\n\nDocker build failed. Last %d lines of output:\n%s",
+						hostname,
+						version,
+						err,
+						maxLines,
+						relevantOutput,
+					)
+				}
+
+				if buildOutput != "" {
+					// Build succeeded on retry but container creation still failed
+					return nil, fmt.Errorf(
+						"%s could not start tailscale container (version: %s): %w\n\nDocker build succeeded on retry, but container creation failed. Last %d lines of build output:\n%s",
+						hostname,
+						version,
+						err,
+						maxLines,
+						relevantOutput,
+					)
+				}
+
+				// No output at all - diagnostic build command may have failed
 				return nil, fmt.Errorf(
-					"%s could not start tailscale container (version: %s): %w\n\nDocker build succeeded on retry, but container creation failed. Last %d lines of build output:\n%s",
+					"%s could not start tailscale container (version: %s): %w\n\nUnable to get diagnostic build output (command may have failed silently)",
 					hostname,
 					version,
 					err,
-					maxLines,
-					relevantOutput,
 				)
 			}
-
-			// No output at all - diagnostic build command may have failed
-			return nil, fmt.Errorf(
-				"%s could not start tailscale container (version: %s): %w\n\nUnable to get diagnostic build output (command may have failed silently)",
-				hostname,
-				version,
-				err,
-			)
 		}
 	case "unstable":
 		tailscaleOptions.Repository = "tailscale/tailscale"