docs(README): update contributors

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,65 @@
+---
+name: "Bug report"
+about: "Create a bug report to help us improve"
+title: ""
+labels: ["bug"]
+assignees: ""
+---
+
+<!--
+Before posting a bug report, discuss the behaviour you are expecting with the Discord community
+to make sure that it is truly a bug.
+The issue tracker is not the place to ask for support or how to set up Headscale.
+
+Bug reports without the sufficient information will be closed.
+
+Headscale is a multinational community across the globe. Our language is English.
+All bug reports needs to be in English.
+-->
+
+## Bug description
+
+<!-- A clear and concise description of what the bug is. Describe the expected bahavior
+  and how it is currently different. If you are unsure if it is a bug, consider discussing
+  it on our Discord server first. -->
+
+## Environment
+
+<!-- Please add relevant information about your system. For example:
+- Version of headscale used
+- Version of tailscale client
+- OS (e.g. Linux, Mac, Cygwin, WSL, etc.) and version
+- Kernel version
+- The relevant config parameters you used
+- Log output
+-->
+
+- OS:
+- Headscale version:
+- Tailscale version:
+
+<!--
+We do not support running Headscale in a container nor behind a (reverse) proxy.
+If either of these are true for your environment, ask the community in Discord
+instead of filing a bug report.
+-->
+
+- [ ] Headscale is behind a (reverse) proxy
+- [ ] Headscale runs in a container
+
+## To Reproduce
+
+<!-- Steps to reproduce the behavior. -->
+
+## Logs and attachments
+
+<!-- Please attach files with:
+- Client netmap dump (see below)
+- ACL configuration
+- Headscale configuration
+
+Dump the netmap of tailscale clients:
+`tailscale debug netmap > DESCRIPTIVE_NAME.json`
+
+Please provide information describing the netmap, which client, which headscale version etc.
+-->

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -1,83 +0,0 @@
-name: 🐞 Bug
-description: File a bug/issue
-title: "[Bug] <title>"
-labels: ["bug", "needs triage"]
-body:
-  - type: checkboxes
-    attributes:
-      label: Is this a support request?
-      description: This issue tracker is for bugs and feature requests only. If you need help, please use ask in our Discord community
-      options:
-        - label: This is not a support request
-          required: true
-  - type: checkboxes
-    attributes:
-      label: Is there an existing issue for this?
-      description: Please search to see if an issue already exists for the bug you encountered.
-      options:
-        - label: I have searched the existing issues
-          required: true
-  - type: textarea
-    attributes:
-      label: Current Behavior
-      description: A concise description of what you're experiencing.
-    validations:
-      required: true
-  - type: textarea
-    attributes:
-      label: Expected Behavior
-      description: A concise description of what you expected to happen.
-    validations:
-      required: true
-  - type: textarea
-    attributes:
-      label: Steps To Reproduce
-      description: Steps to reproduce the behavior.
-      placeholder: |
-        1. In this environment...
-        1. With this config...
-        1. Run '...'
-        1. See error...
-    validations:
-      required: true
-  - type: textarea
-    attributes:
-      label: Environment
-      description: |
-        examples:
-          - **OS**: Ubuntu 20.04
-          - **Headscale version**: 0.22.3
-          - **Tailscale version**: 1.64.0
-      value: |
-        - OS:
-        - Headscale version:
-        - Tailscale version:
-      render: markdown
-    validations:
-      required: true
-  - type: checkboxes
-    attributes:
-      label: Runtime environment
-      options:
-        - label: Headscale is behind a (reverse) proxy
-          required: false
-        - label: Headscale runs in a container
-          required: false
-  - type: textarea
-    attributes:
-      label: Anything else?
-      description: |
-        Links? References? Anything that will give us more context about the issue you are encountering!
-
-        - Client netmap dump (see below)
-        - ACL configuration
-        - Headscale configuration
-
-        Dump the netmap of tailscale clients:
-        `tailscale debug netmap > DESCRIPTIVE_NAME.json`
-
-        Please provide information describing the netmap, which client, which headscale version etc.
-
-        Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in.
-    validations:
-      required: false

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,26 @@
+---
+name: "Feature request"
+about: "Suggest an idea for headscale"
+title: ""
+labels: ["enhancement"]
+assignees: ""
+---
+
+<!--
+We typically have a clear roadmap for what we want to improve and reserve the right
+to close feature requests that does not fit in the roadmap, or fit with the scope
+of the project, or we actually want to implement ourselves.
+
+Headscale is a multinational community across the globe. Our language is English.
+All bug reports needs to be in English.
+-->
+
+## Why
+
+<!-- Include the reason, why you would need the feature. E.g. what problem
+  does it solve? Or which workflow is currently frustrating and will be improved by
+  this? -->
+
+## Description
+
+<!-- A clear and precise description of what new or changed feature you want. -->

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -1,36 +0,0 @@
-name: 🚀 Feature Request
-description: Suggest an idea for Headscale
-title: "[Feature] <title>"
-labels: [enhancement]
-body:
-  - type: textarea
-    attributes:
-      label: Use case
-      description: Please describe the use case for this feature.
-      placeholder: |
-        <!-- Include the reason, why you would need the feature. E.g. what problem
-          does it solve? Or which workflow is currently frustrating and will be improved by
-          this? -->
-    validations:
-      required: true
-  - type: textarea
-    attributes:
-      label: Description
-      description: A clear and precise description of what new or changed feature you want.
-    validations:
-      required: true
-  - type: checkboxes
-    attributes:
-      label: Contribution
-      description: Are you willing to contribute to the implementation of this feature?
-      options:
-        - label: I can write the design doc for this feature
-          required: true
-        - label: I can contribute this feature
-          required: true
-  - type: textarea
-    attributes:
-      label: How can it be implemented?
-      description: Free text for your ideas on how this feature could be implemented.
-    validations:
-      required: false

.github/workflows/contributors.yml

@@ -0,0 +1,36 @@
+name: Contributors
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  add-contributors:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Delete upstream contributor branch
+        # Allow continue on failure to account for when the
+        # upstream branch is deleted or does not exist.
+        continue-on-error: true
+        run: git push origin --delete update-contributors
+      - name: Create up-to-date contributors branch
+        run: git checkout -B update-contributors
+      - name: Push empty contributors branch
+        run: git push origin update-contributors
+      - name: Switch back to main
+        run: git checkout main
+      - uses: BobAnkh/add-contributors@v0.2.2
+        with:
+          CONTRIBUTOR: "## Contributors"
+          COLUMN_PER_ROW: "6"
+          ACCESS_TOKEN: ${{secrets.GITHUB_TOKEN}}
+          IMG_WIDTH: "100"
+          FONT_SIZE: "14"
+          PATH: "/README.md"
+          COMMIT_MESSAGE: "docs(README): update contributors"
+          AVATAR_SHAPE: "round"
+          BRANCH: "update-contributors"
+          PULL_REQUEST: "main"

.github/workflows/docs-test.yml

@@ -24,4 +24,4 @@ jobs:
       - name: Setup dependencies
         run: pip install -r docs/requirements.txt
       - name: Build docs
-        run: mkdocs build --strict
+        run: mkdocs build --strict

examples/README.md

@@ -0,0 +1,5 @@
+# Examples
+
+This directory contains examples on how to run `headscale` on different platforms.
+
+All examples are provided by the community and they are not verified by the `headscale` authors.

examples/kustomize/.gitignore

@@ -0,0 +1,2 @@
+/**/site
+/**/secrets

examples/kustomize/README.md

@@ -0,0 +1,100 @@
+# Deploying headscale on Kubernetes
+
+**Note:** This is contributed by the community and not verified by the headscale authors.
+
+This directory contains [Kustomize](https://kustomize.io) templates that deploy
+headscale in various configurations.
+
+These templates currently support Rancher k3s. Other clusters may require
+adaptation, especially around volume claims and ingress.
+
+Commands below assume this directory is your current working directory.
+
+# Generate secrets and site configuration
+
+Run `./init.bash` to generate keys, passwords, and site configuration files.
+
+Edit `base/site/public.env`, changing `public-hostname` to the public DNS name
+that will be used for your headscale deployment.
+
+Set `public-proto` to "https" if you're planning to use TLS & Let's Encrypt.
+
+Configure DERP servers by editing `base/site/derp.yaml` if needed.
+
+# Add the image to the registry
+
+You'll somehow need to get `headscale:latest` into your cluster image registry.
+
+An easy way to do this with k3s:
+
+- Reconfigure k3s to use docker instead of containerd (`k3s server --docker`)
+- `docker build -t headscale:latest ..` from here
+
+# Create the namespace
+
+If it doesn't already exist, `kubectl create ns headscale`.
+
+# Deploy headscale
+
+## sqlite
+
+`kubectl -n headscale apply -k ./sqlite`
+
+## postgres
+
+`kubectl -n headscale apply -k ./postgres`
+
+# TLS & Let's Encrypt
+
+Test a staging certificate with your configured DNS name and Let's Encrypt.
+
+`kubectl -n headscale apply -k ./staging-tls`
+
+Replace with a production certificate.
+
+`kubectl -n headscale apply -k ./production-tls`
+
+## Static / custom TLS certificates
+
+Only Let's Encrypt is supported. If you need other TLS settings, modify or patch the ingress.
+
+# Administration
+
+Use the wrapper script to remotely operate headscale to perform administrative
+tasks like creating namespaces, authkeys, etc.
+
+```
+[c@nix-slate:~/Projects/headscale/k8s]$ ./headscale.bash
+
+headscale is an open source implementation of the Tailscale control server
+
+https://github.com/juanfont/headscale
+
+Usage:
+  headscale [command]
+
+Available Commands:
+  help        Help about any command
+  namespace   Manage the namespaces of headscale
+  node        Manage the nodes of headscale
+  preauthkey  Handle the preauthkeys in headscale
+  routes      Manage the routes of headscale
+  serve       Launches the headscale server
+  version     Print the version.
+
+Flags:
+  -h, --help            help for headscale
+  -o, --output string   Output format. Empty for human-readable, 'json' or 'json-line'
+
+Use "headscale [command] --help" for more information about a command.
+
+```
+
+# TODO / Ideas
+
+- Interpolate `email:` option to the ClusterIssuer from site configuration.
+  This probably needs to be done with a transformer, kustomize vars don't seem to work.
+- Add kustomize examples for cloud-native ingress, load balancer
+- CockroachDB for the backend
+- DERP server deployment
+- Tor hidden service

examples/kustomize/base/configmap.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: headscale-config
+data:
+  server_url: $(PUBLIC_PROTO)://$(PUBLIC_HOSTNAME)
+  listen_addr: "0.0.0.0:8080"
+  metrics_listen_addr: "127.0.0.1:9090"
+  ephemeral_node_inactivity_timeout: "30m"

examples/kustomize/base/ingress.yaml

@@ -0,0 +1,18 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: headscale
+  annotations:
+    kubernetes.io/ingress.class: traefik
+spec:
+  rules:
+    - host: $(PUBLIC_HOSTNAME)
+      http:
+        paths:
+          - backend:
+              service:
+                name: headscale
+                port:
+                  number: 8080
+            path: /
+            pathType: Prefix

examples/kustomize/base/kustomization.yaml

@@ -0,0 +1,42 @@
+namespace: headscale
+resources:
+  - configmap.yaml
+  - ingress.yaml
+  - service.yaml
+generatorOptions:
+  disableNameSuffixHash: true
+configMapGenerator:
+  - name: headscale-site
+    files:
+      - derp.yaml=site/derp.yaml
+    envs:
+      - site/public.env
+  - name: headscale-etc
+    literals:
+      - config.json={}
+secretGenerator:
+  - name: headscale
+    files:
+      - secrets/private-key
+vars:
+  - name: PUBLIC_PROTO
+    objRef:
+      kind: ConfigMap
+      name: headscale-site
+      apiVersion: v1
+    fieldRef:
+      fieldPath: data.public-proto
+  - name: PUBLIC_HOSTNAME
+    objRef:
+      kind: ConfigMap
+      name: headscale-site
+      apiVersion: v1
+    fieldRef:
+      fieldPath: data.public-hostname
+  - name: CONTACT_EMAIL
+    objRef:
+      kind: ConfigMap
+      name: headscale-site
+      apiVersion: v1
+    fieldRef:
+      fieldPath: data.contact-email

examples/kustomize/base/service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: headscale
+  labels:
+    app: headscale
+spec:
+  selector:
+    app: headscale
+  ports:
+    - name: http
+      targetPort: http
+      port: 8080

examples/kustomize/headscale.bash

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+set -eu
+exec kubectl -n headscale exec -ti pod/headscale-0 -- /go/bin/headscale "$@"

examples/kustomize/init.bash

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -eux
+cd $(dirname $0)
+
+umask 022
+mkdir -p base/site/
+[ ! -e base/site/public.env ] && (
+    cat >base/site/public.env <<EOF
+public-hostname=localhost
+public-proto=http
+contact-email=headscale@example.com
+EOF
+)
+[ ! -e base/site/derp.yaml ] && cp ../derp.yaml base/site/derp.yaml
+
+umask 077
+mkdir -p base/secrets/
+[ ! -e base/secrets/private-key ] && (
+    wg genkey > base/secrets/private-key
+)
+mkdir -p postgres/secrets/
+[ ! -e postgres/secrets/password ] && (head -c 32 /dev/urandom | base64 -w0 > postgres/secrets/password)

examples/kustomize/install-cert-manager.bash

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+set -eux
+kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.4.0/cert-manager.yaml

examples/kustomize/postgres/deployment.yaml

@@ -0,0 +1,81 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: headscale
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: headscale
+  template:
+    metadata:
+      labels:
+        app: headscale
+    spec:
+      containers:
+        - name: headscale
+          image: "headscale:latest"
+          imagePullPolicy: IfNotPresent
+          command: ["/go/bin/headscale", "serve"]
+          env:
+            - name: SERVER_URL
+              value: $(PUBLIC_PROTO)://$(PUBLIC_HOSTNAME)
+            - name: LISTEN_ADDR
+              valueFrom:
+                configMapKeyRef:
+                  name: headscale-config
+                  key: listen_addr
+            - name: METRICS_LISTEN_ADDR
+              valueFrom:
+                configMapKeyRef:
+                  name: headscale-config
+                  key: metrics_listen_addr
+            - name: DERP_MAP_PATH
+              value: /vol/config/derp.yaml
+            - name: EPHEMERAL_NODE_INACTIVITY_TIMEOUT
+              valueFrom:
+                configMapKeyRef:
+                  name: headscale-config
+                  key: ephemeral_node_inactivity_timeout
+            - name: DB_TYPE
+              value: postgres
+            - name: DB_HOST
+              value: postgres.headscale.svc.cluster.local
+            - name: DB_PORT
+              value: "5432"
+            - name: DB_USER
+              value: headscale
+            - name: DB_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: postgresql
+                  key: password
+            - name: DB_NAME
+              value: headscale
+          ports:
+            - name: http
+              protocol: TCP
+              containerPort: 8080
+          livenessProbe:
+            tcpSocket:
+              port: http
+            initialDelaySeconds: 30
+            timeoutSeconds: 5
+            periodSeconds: 15
+          volumeMounts:
+            - name: config
+              mountPath: /vol/config
+            - name: secret
+              mountPath: /vol/secret
+            - name: etc
+              mountPath: /etc/headscale
+      volumes:
+        - name: config
+          configMap:
+            name: headscale-site
+        - name: etc
+          configMap:
+            name: headscale-etc
+        - name: secret
+          secret:
+            secretName: headscale

examples/kustomize/postgres/kustomization.yaml

@@ -0,0 +1,13 @@
+namespace: headscale
+bases:
+  - ../base
+resources:
+  - deployment.yaml
+  - postgres-service.yaml
+  - postgres-statefulset.yaml
+generatorOptions:
+  disableNameSuffixHash: true
+secretGenerator:
+  - name: postgresql
+    files:
+      - secrets/password

examples/kustomize/postgres/postgres-service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres
+  labels:
+    app: postgres
+spec:
+  selector:
+    app: postgres
+  ports:
+    - name: postgres
+      targetPort: postgres
+      port: 5432

examples/kustomize/postgres/postgres-statefulset.yaml

@@ -0,0 +1,49 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: postgres
+spec:
+  serviceName: postgres
+  replicas: 1
+  selector:
+    matchLabels:
+      app: postgres
+  template:
+    metadata:
+      labels:
+        app: postgres
+    spec:
+      containers:
+        - name: postgres
+          image: "postgres:13"
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: postgresql
+                  key: password
+            - name: POSTGRES_USER
+              value: headscale
+          ports:
+            - name: postgres
+              protocol: TCP
+              containerPort: 5432
+          livenessProbe:
+            tcpSocket:
+              port: 5432
+            initialDelaySeconds: 30
+            timeoutSeconds: 5
+            periodSeconds: 15
+          volumeMounts:
+            - name: pgdata
+              mountPath: /var/lib/postgresql/data
+  volumeClaimTemplates:
+    - metadata:
+        name: pgdata
+      spec:
+        storageClassName: local-path
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 1Gi

examples/kustomize/production-tls/ingress-patch.yaml

@@ -0,0 +1,11 @@
+kind: Ingress
+metadata:
+  name: headscale
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-production
+    traefik.ingress.kubernetes.io/router.tls: "true"
+spec:
+  tls:
+    - hosts:
+        - $(PUBLIC_HOSTNAME)
+      secretName: production-cert

examples/kustomize/production-tls/kustomization.yaml

@@ -0,0 +1,9 @@
+namespace: headscale
+bases:
+  - ../base
+resources:
+  - production-issuer.yaml
+patches:
+  - path: ingress-patch.yaml
+    target:
+      kind: Ingress

examples/kustomize/production-tls/production-issuer.yaml

@@ -0,0 +1,16 @@
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-production
+spec:
+  acme:
+    # TODO: figure out how to get kustomize to interpolate this, or use a transformer
+    #email: $(CONTACT_EMAIL)
+    server: https://acme-v02.api.letsencrypt.org/directory
+    privateKeySecretRef:
+      # Secret resource used to store the account's private key.
+      name: letsencrypt-production-acc-key
+    solvers:
+      - http01:
+          ingress:
+            class: traefik

examples/kustomize/sqlite/kustomization.yaml

@@ -0,0 +1,5 @@
+namespace: headscale
+bases:
+  - ../base
+resources:
+  - statefulset.yaml

examples/kustomize/sqlite/statefulset.yaml

@@ -0,0 +1,82 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: headscale
+spec:
+  serviceName: headscale
+  replicas: 1
+  selector:
+    matchLabels:
+      app: headscale
+  template:
+    metadata:
+      labels:
+        app: headscale
+    spec:
+      containers:
+        - name: headscale
+          image: "headscale:latest"
+          imagePullPolicy: IfNotPresent
+          command: ["/go/bin/headscale", "serve"]
+          env:
+            - name: SERVER_URL
+              value: $(PUBLIC_PROTO)://$(PUBLIC_HOSTNAME)
+            - name: LISTEN_ADDR
+              valueFrom:
+                configMapKeyRef:
+                  name: headscale-config
+                  key: listen_addr
+            - name: METRICS_LISTEN_ADDR
+              valueFrom:
+                configMapKeyRef:
+                  name: headscale-config
+                  key: metrics_listen_addr
+            - name: DERP_MAP_PATH
+              value: /vol/config/derp.yaml
+            - name: EPHEMERAL_NODE_INACTIVITY_TIMEOUT
+              valueFrom:
+                configMapKeyRef:
+                  name: headscale-config
+                  key: ephemeral_node_inactivity_timeout
+            - name: DB_TYPE
+              value: sqlite3
+            - name: DB_PATH
+              value: /vol/data/db.sqlite
+          ports:
+            - name: http
+              protocol: TCP
+              containerPort: 8080
+          livenessProbe:
+            tcpSocket:
+              port: http
+            initialDelaySeconds: 30
+            timeoutSeconds: 5
+            periodSeconds: 15
+          volumeMounts:
+            - name: config
+              mountPath: /vol/config
+            - name: data
+              mountPath: /vol/data
+            - name: secret
+              mountPath: /vol/secret
+            - name: etc
+              mountPath: /etc/headscale
+      volumes:
+        - name: config
+          configMap:
+            name: headscale-site
+        - name: etc
+          configMap:
+            name: headscale-etc
+        - name: secret
+          secret:
+            secretName: headscale
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        storageClassName: local-path
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 1Gi

examples/kustomize/staging-tls/ingress-patch.yaml

@@ -0,0 +1,11 @@
+kind: Ingress
+metadata:
+  name: headscale
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-staging
+    traefik.ingress.kubernetes.io/router.tls: "true"
+spec:
+  tls:
+    - hosts:
+        - $(PUBLIC_HOSTNAME)
+      secretName: staging-cert

examples/kustomize/staging-tls/kustomization.yaml

@@ -0,0 +1,9 @@
+namespace: headscale
+bases:
+  - ../base
+resources:
+  - staging-issuer.yaml
+patches:
+  - path: ingress-patch.yaml
+    target:
+      kind: Ingress

examples/kustomize/staging-tls/staging-issuer.yaml

@@ -0,0 +1,16 @@
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-staging
+spec:
+  acme:
+    # TODO: figure out how to get kustomize to interpolate this, or use a transformer
+    #email: $(CONTACT_EMAIL)
+    server: https://acme-staging-v02.api.letsencrypt.org/directory
+    privateKeySecretRef:
+      # Secret resource used to store the account's private key.
+      name: letsencrypt-staging-acc-key
+    solvers:
+      - http01:
+          ingress:
+            class: traefik

hscontrol/app.go

@@ -225,7 +225,7 @@ func (h *Headscale) deleteExpireEphemeralNodes(milliSeconds int64) {
 	for range ticker.C {
 		var removed []types.NodeID
 		var changed []types.NodeID
-		if err := h.db.Write(func(tx *gorm.DB) error {
+		if err := h.db.DB.Transaction(func(tx *gorm.DB) error {
 			removed, changed = db.DeleteExpiredEphemeralNodes(tx, h.cfg.EphemeralNodeInactivityTimeout)
 
 			return nil
@@ -263,7 +263,7 @@ func (h *Headscale) expireExpiredMachines(intervalMs int64) {
 	var changed bool
 
 	for range ticker.C {
-		if err := h.db.Write(func(tx *gorm.DB) error {
+		if err := h.db.DB.Transaction(func(tx *gorm.DB) error {
 			lastCheck, update, changed = db.ExpireExpiredNodes(tx, lastCheck)
 
 			return nil
@@ -452,7 +452,7 @@ func (h *Headscale) ensureUnixSocketIsAbsent() error {
 
 func (h *Headscale) createRouter(grpcMux *grpcRuntime.ServeMux) *mux.Router {
 	router := mux.NewRouter()
-	router.Use(prometheusMiddleware)
+	router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux)
 
 	router.HandleFunc(ts2021UpgradePath, h.NoiseUpgradeHandler).Methods(http.MethodPost)
 
@@ -508,7 +508,7 @@ func (h *Headscale) Serve() error {
 
 	// Fetch an initial DERP Map before we start serving
 	h.DERPMap = derp.GetDERPMap(h.cfg.DERP)
-	h.mapper = mapper.NewMapper(h.db, h.cfg, h.DERPMap, h.nodeNotifier)
+	h.mapper = mapper.NewMapper(h.db, h.cfg, h.DERPMap, h.nodeNotifier.ConnectedMap())
 
 	if h.cfg.DERP.ServerEnabled {
 		// When embedded DERP is enabled we always need a STUN server
@@ -680,7 +680,7 @@ func (h *Headscale) Serve() error {
 	// HTTP setup
 	//
 	// This is the regular router that we expose
-	// over our main Addr
+	// over our main Addr. It also serves the legacy Tailcale API
 	router := h.createRouter(grpcGatewayMux)
 
 	httpServer := &http.Server{
@@ -710,10 +710,11 @@ func (h *Headscale) Serve() error {
 		Msgf("listening and serving HTTP on: %s", h.cfg.Addr)
 
 	debugMux := http.NewServeMux()
-	debugMux.Handle("/debug/pprof/", http.DefaultServeMux)
 	debugMux.HandleFunc("/debug/notifier", func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
 		w.Write([]byte(h.nodeNotifier.String()))
+
+		return
 	})
 	debugMux.HandleFunc("/debug/mapresp", func(w http.ResponseWriter, r *http.Request) {
 		h.mapSessionMu.Lock()
@@ -727,6 +728,8 @@ func (h *Headscale) Serve() error {
 
 		w.WriteHeader(http.StatusOK)
 		w.Write([]byte(b.String()))
+
+		return
 	})
 	debugMux.Handle("/metrics", promhttp.Handler())
 

hscontrol/auth.go

@@ -273,6 +273,8 @@ func (h *Headscale) handleAuthKey(
 				Err(err).
 				Msg("Cannot encode message")
 			http.Error(writer, "Internal server error", http.StatusInternalServerError)
+			nodeRegistrations.WithLabelValues("new", util.RegisterMethodAuthKey, "error", pak.User.Name).
+				Inc()
 
 			return
 		}
@@ -292,6 +294,13 @@ func (h *Headscale) handleAuthKey(
 			Str("node", registerRequest.Hostinfo.Hostname).
 			Msg("Failed authentication via AuthKey")
 
+		if pak != nil {
+			nodeRegistrations.WithLabelValues("new", util.RegisterMethodAuthKey, "error", pak.User.Name).
+				Inc()
+		} else {
+			nodeRegistrations.WithLabelValues("new", util.RegisterMethodAuthKey, "error", "unknown").Inc()
+		}
+
 		return
 	}
 
@@ -395,13 +404,15 @@ func (h *Headscale) handleAuthKey(
 				Caller().
 				Err(err).
 				Msg("could not register node")
+			nodeRegistrations.WithLabelValues("new", util.RegisterMethodAuthKey, "error", pak.User.Name).
+				Inc()
 			http.Error(writer, "Internal server error", http.StatusInternalServerError)
 
 			return
 		}
 	}
 
-	h.db.Write(func(tx *gorm.DB) error {
+	err = h.db.DB.Transaction(func(tx *gorm.DB) error {
 		return db.UsePreAuthKey(tx, pak)
 	})
 	if err != nil {
@@ -409,6 +420,8 @@ func (h *Headscale) handleAuthKey(
 			Caller().
 			Err(err).
 			Msg("Failed to use pre-auth key")
+		nodeRegistrations.WithLabelValues("new", util.RegisterMethodAuthKey, "error", pak.User.Name).
+			Inc()
 		http.Error(writer, "Internal server error", http.StatusInternalServerError)
 
 		return
@@ -427,10 +440,14 @@ func (h *Headscale) handleAuthKey(
 			Str("node", registerRequest.Hostinfo.Hostname).
 			Err(err).
 			Msg("Cannot encode message")
+		nodeRegistrations.WithLabelValues("new", util.RegisterMethodAuthKey, "error", pak.User.Name).
+			Inc()
 		http.Error(writer, "Internal server error", http.StatusInternalServerError)
 
 		return
 	}
+	nodeRegistrations.WithLabelValues("new", util.RegisterMethodAuthKey, "success", pak.User.Name).
+		Inc()
 	writer.Header().Set("Content-Type", "application/json; charset=utf-8")
 	writer.WriteHeader(http.StatusOK)
 	_, err = writer.Write(respBody)
@@ -546,7 +563,7 @@ func (h *Headscale) handleNodeLogOut(
 	}
 
 	if node.IsEphemeral() {
-		changedNodes, err := h.db.DeleteNode(&node, h.nodeNotifier.LikelyConnectedMap())
+		changedNodes, err := h.db.DeleteNode(&node, h.nodeNotifier.ConnectedMap())
 		if err != nil {
 			log.Error().
 				Err(err).
@@ -599,10 +616,14 @@ func (h *Headscale) handleNodeWithValidRegistration(
 			Caller().
 			Err(err).
 			Msg("Cannot encode message")
+		nodeRegistrations.WithLabelValues("update", "web", "error", node.User.Name).
+			Inc()
 		http.Error(writer, "Internal server error", http.StatusInternalServerError)
 
 		return
 	}
+	nodeRegistrations.WithLabelValues("update", "web", "success", node.User.Name).
+		Inc()
 
 	writer.Header().Set("Content-Type", "application/json; charset=utf-8")
 	writer.WriteHeader(http.StatusOK)
@@ -633,7 +654,7 @@ func (h *Headscale) handleNodeKeyRefresh(
 		Str("node", node.Hostname).
 		Msg("We have the OldNodeKey in the database. This is a key refresh")
 
-	err := h.db.Write(func(tx *gorm.DB) error {
+	err := h.db.DB.Transaction(func(tx *gorm.DB) error {
 		return db.NodeSetNodeKey(tx, &node, registerRequest.NodeKey)
 	})
 	if err != nil {
@@ -716,10 +737,14 @@ func (h *Headscale) handleNodeExpiredOrLoggedOut(
 			Caller().
 			Err(err).
 			Msg("Cannot encode message")
+		nodeRegistrations.WithLabelValues("reauth", "web", "error", node.User.Name).
+			Inc()
 		http.Error(writer, "Internal server error", http.StatusInternalServerError)
 
 		return
 	}
+	nodeRegistrations.WithLabelValues("reauth", "web", "success", node.User.Name).
+		Inc()
 
 	writer.Header().Set("Content-Type", "application/json; charset=utf-8")
 	writer.WriteHeader(http.StatusOK)

hscontrol/auth_noise.go

@@ -33,6 +33,7 @@ func (ns *noiseServer) NoiseRegistrationHandler(
 			Caller().
 			Err(err).
 			Msg("Cannot parse RegisterRequest")
+		nodeRegistrations.WithLabelValues("unknown", "web", "error", "unknown").Inc()
 		http.Error(writer, "Internal error", http.StatusInternalServerError)
 
 		return

hscontrol/db/node.go

@@ -10,7 +10,6 @@ import (
 	"github.com/juanfont/headscale/hscontrol/types"
 	"github.com/juanfont/headscale/hscontrol/util"
 	"github.com/patrickmn/go-cache"
-	"github.com/puzpuzpuz/xsync/v3"
 	"github.com/rs/zerolog/log"
 	"gorm.io/gorm"
 	"tailscale.com/tailcfg"
@@ -261,9 +260,9 @@ func NodeSetExpiry(tx *gorm.DB,
 	return tx.Model(&types.Node{}).Where("id = ?", nodeID).Update("expiry", expiry).Error
 }
 
-func (hsdb *HSDatabase) DeleteNode(node *types.Node, isLikelyConnected *xsync.MapOf[types.NodeID, bool]) ([]types.NodeID, error) {
+func (hsdb *HSDatabase) DeleteNode(node *types.Node, isConnected types.NodeConnectedMap) ([]types.NodeID, error) {
 	return Write(hsdb.DB, func(tx *gorm.DB) ([]types.NodeID, error) {
-		return DeleteNode(tx, node, isLikelyConnected)
+		return DeleteNode(tx, node, isConnected)
 	})
 }
 
@@ -271,9 +270,9 @@ func (hsdb *HSDatabase) DeleteNode(node *types.Node, isLikelyConnected *xsync.Ma
 // Caller is responsible for notifying all of change.
 func DeleteNode(tx *gorm.DB,
 	node *types.Node,
-	isLikelyConnected *xsync.MapOf[types.NodeID, bool],
+	isConnected types.NodeConnectedMap,
 ) ([]types.NodeID, error) {
-	changed, err := deleteNodeRoutes(tx, node, isLikelyConnected)
+	changed, err := deleteNodeRoutes(tx, node, isConnected)
 	if err != nil {
 		return changed, err
 	}

hscontrol/db/node_test.go

@@ -11,7 +11,6 @@ import (
 	"github.com/juanfont/headscale/hscontrol/policy"
 	"github.com/juanfont/headscale/hscontrol/types"
 	"github.com/juanfont/headscale/hscontrol/util"
-	"github.com/puzpuzpuz/xsync/v3"
 	"gopkg.in/check.v1"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
@@ -121,7 +120,7 @@ func (s *Suite) TestHardDeleteNode(c *check.C) {
 	}
 	db.DB.Save(&node)
 
-	_, err = db.DeleteNode(&node, xsync.NewMapOf[types.NodeID, bool]())
+	_, err = db.DeleteNode(&node, types.NodeConnectedMap{})
 	c.Assert(err, check.IsNil)
 
 	_, err = db.getNode(user.Name, "testnode3")

hscontrol/db/preauth_keys_test.go

@@ -147,7 +147,7 @@ func (*Suite) TestEphemeralKeyReusable(c *check.C) {
 	_, err = db.getNode("test7", "testest")
 	c.Assert(err, check.IsNil)
 
-	db.Write(func(tx *gorm.DB) error {
+	db.DB.Transaction(func(tx *gorm.DB) error {
 		DeleteExpiredEphemeralNodes(tx, time.Second*20)
 		return nil
 	})
@@ -181,7 +181,7 @@ func (*Suite) TestEphemeralKeyNotReusable(c *check.C) {
 	_, err = db.getNode("test7", "testest")
 	c.Assert(err, check.IsNil)
 
-	db.Write(func(tx *gorm.DB) error {
+	db.DB.Transaction(func(tx *gorm.DB) error {
 		DeleteExpiredEphemeralNodes(tx, time.Second*20)
 		return nil
 	})

hscontrol/db/routes.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/juanfont/headscale/hscontrol/policy"
 	"github.com/juanfont/headscale/hscontrol/types"
-	"github.com/puzpuzpuz/xsync/v3"
 	"github.com/rs/zerolog/log"
 	"gorm.io/gorm"
 	"tailscale.com/util/set"
@@ -127,7 +126,7 @@ func EnableRoute(tx *gorm.DB, id uint64) (*types.StateUpdate, error) {
 
 func DisableRoute(tx *gorm.DB,
 	id uint64,
-	isLikelyConnected *xsync.MapOf[types.NodeID, bool],
+	isConnected types.NodeConnectedMap,
 ) ([]types.NodeID, error) {
 	route, err := GetRoute(tx, id)
 	if err != nil {
@@ -148,7 +147,7 @@ func DisableRoute(tx *gorm.DB,
 			return nil, err
 		}
 
-		update, err = failoverRouteTx(tx, isLikelyConnected, route)
+		update, err = failoverRouteTx(tx, isConnected, route)
 		if err != nil {
 			return nil, err
 		}
@@ -183,17 +182,17 @@ func DisableRoute(tx *gorm.DB,
 
 func (hsdb *HSDatabase) DeleteRoute(
 	id uint64,
-	isLikelyConnected *xsync.MapOf[types.NodeID, bool],
+	isConnected types.NodeConnectedMap,
 ) ([]types.NodeID, error) {
 	return Write(hsdb.DB, func(tx *gorm.DB) ([]types.NodeID, error) {
-		return DeleteRoute(tx, id, isLikelyConnected)
+		return DeleteRoute(tx, id, isConnected)
 	})
 }
 
 func DeleteRoute(
 	tx *gorm.DB,
 	id uint64,
-	isLikelyConnected *xsync.MapOf[types.NodeID, bool],
+	isConnected types.NodeConnectedMap,
 ) ([]types.NodeID, error) {
 	route, err := GetRoute(tx, id)
 	if err != nil {
@@ -208,7 +207,7 @@ func DeleteRoute(
 	// https://github.com/juanfont/headscale/issues/804#issuecomment-1399314002
 	var update []types.NodeID
 	if !route.IsExitRoute() {
-		update, err = failoverRouteTx(tx, isLikelyConnected, route)
+		update, err = failoverRouteTx(tx, isConnected, route)
 		if err != nil {
 			return nil, nil
 		}
@@ -253,7 +252,7 @@ func DeleteRoute(
 	return update, nil
 }
 
-func deleteNodeRoutes(tx *gorm.DB, node *types.Node, isLikelyConnected *xsync.MapOf[types.NodeID, bool]) ([]types.NodeID, error) {
+func deleteNodeRoutes(tx *gorm.DB, node *types.Node, isConnected types.NodeConnectedMap) ([]types.NodeID, error) {
 	routes, err := GetNodeRoutes(tx, node)
 	if err != nil {
 		return nil, fmt.Errorf("getting node routes: %w", err)
@@ -267,7 +266,7 @@ func deleteNodeRoutes(tx *gorm.DB, node *types.Node, isLikelyConnected *xsync.Ma
 
 		// TODO(kradalby): This is a bit too aggressive, we could probably
 		// figure out which routes needs to be failed over rather than all.
-		chn, err := failoverRouteTx(tx, isLikelyConnected, &routes[i])
+		chn, err := failoverRouteTx(tx, isConnected, &routes[i])
 		if err != nil {
 			return changed, fmt.Errorf("failing over route after delete: %w", err)
 		}
@@ -410,7 +409,7 @@ func SaveNodeRoutes(tx *gorm.DB, node *types.Node) (bool, error) {
 // If needed, the failover will be attempted.
 func FailoverNodeRoutesIfNeccessary(
 	tx *gorm.DB,
-	isLikelyConnected *xsync.MapOf[types.NodeID, bool],
+	isConnected types.NodeConnectedMap,
 	node *types.Node,
 ) (*types.StateUpdate, error) {
 	nodeRoutes, err := GetNodeRoutes(tx, node)
@@ -431,12 +430,12 @@ nodeRouteLoop:
 			if route.IsPrimary {
 				// if we have a primary route, and the node is connected
 				// nothing needs to be done.
-				if val, ok := isLikelyConnected.Load(route.Node.ID); ok && val {
+				if conn, ok := isConnected[route.Node.ID]; conn && ok {
 					continue nodeRouteLoop
 				}
 
 				// if not, we need to failover the route
-				failover := failoverRoute(isLikelyConnected, &route, routes)
+				failover := failoverRoute(isConnected, &route, routes)
 				if failover != nil {
 					err := failover.save(tx)
 					if err != nil {
@@ -478,7 +477,7 @@ nodeRouteLoop:
 // If the given route was not primary, it returns early.
 func failoverRouteTx(
 	tx *gorm.DB,
-	isLikelyConnected *xsync.MapOf[types.NodeID, bool],
+	isConnected types.NodeConnectedMap,
 	r *types.Route,
 ) ([]types.NodeID, error) {
 	if r == nil {
@@ -501,7 +500,7 @@ func failoverRouteTx(
 		return nil, fmt.Errorf("getting routes by prefix: %w", err)
 	}
 
-	fo := failoverRoute(isLikelyConnected, r, routes)
+	fo := failoverRoute(isConnected, r, routes)
 	if fo == nil {
 		return nil, nil
 	}
@@ -539,7 +538,7 @@ func (f *failover) save(tx *gorm.DB) error {
 }
 
 func failoverRoute(
-	isLikelyConnected *xsync.MapOf[types.NodeID, bool],
+	isConnected types.NodeConnectedMap,
 	routeToReplace *types.Route,
 	altRoutes types.Routes,
 
@@ -571,11 +570,9 @@ func failoverRoute(
 			continue
 		}
 
-		if isLikelyConnected != nil {
-			if val, ok := isLikelyConnected.Load(route.Node.ID); ok && val {
-				newPrimary = &altRoutes[idx]
-				break
-			}
+		if isConnected != nil && isConnected[route.Node.ID] {
+			newPrimary = &altRoutes[idx]
+			break
 		}
 	}
 

hscontrol/db/routes_test.go

@@ -10,22 +10,11 @@ import (
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/juanfont/headscale/hscontrol/types"
 	"github.com/juanfont/headscale/hscontrol/util"
-	"github.com/puzpuzpuz/xsync/v3"
 	"gopkg.in/check.v1"
 	"gorm.io/gorm"
 	"tailscale.com/tailcfg"
 )
 
-var smap = func(m map[types.NodeID]bool) *xsync.MapOf[types.NodeID, bool] {
-	s := xsync.NewMapOf[types.NodeID, bool]()
-
-	for k, v := range m {
-		s.Store(k, v)
-	}
-
-	return s
-}
-
 func (s *Suite) TestGetRoutes(c *check.C) {
 	user, err := db.CreateUser("test")
 	c.Assert(err, check.IsNil)
@@ -342,7 +331,7 @@ func TestFailoverNodeRoutesIfNeccessary(t *testing.T) {
 		name        string
 		nodes       types.Nodes
 		routes      types.Routes
-		isConnected []map[types.NodeID]bool
+		isConnected []types.NodeConnectedMap
 		want        []*types.StateUpdate
 		wantErr     bool
 	}{
@@ -357,7 +346,7 @@ func TestFailoverNodeRoutesIfNeccessary(t *testing.T) {
 				r(1, 1, ipp("10.0.0.0/24"), true, true),
 				r(2, 2, ipp("10.0.0.0/24"), true, false),
 			},
-			isConnected: []map[types.NodeID]bool{
+			isConnected: []types.NodeConnectedMap{
 				// n1 goes down
 				{
 					1: false,
@@ -395,7 +384,7 @@ func TestFailoverNodeRoutesIfNeccessary(t *testing.T) {
 				r(1, 1, ipp("10.0.0.0/24"), true, true),
 				r(2, 2, ipp("10.0.0.0/24"), true, false),
 			},
-			isConnected: []map[types.NodeID]bool{
+			isConnected: []types.NodeConnectedMap{
 				// n1 up recon = noop
 				{
 					1: true,
@@ -439,7 +428,7 @@ func TestFailoverNodeRoutesIfNeccessary(t *testing.T) {
 				r(2, 2, ipp("10.0.0.0/24"), true, false),
 				r(3, 3, ipp("10.0.0.0/24"), true, false),
 			},
-			isConnected: []map[types.NodeID]bool{
+			isConnected: []types.NodeConnectedMap{
 				// n1 goes down
 				{
 					1: false,
@@ -497,7 +486,7 @@ func TestFailoverNodeRoutesIfNeccessary(t *testing.T) {
 				r(2, 2, ipp("10.0.0.0/24"), false, false),
 				r(3, 3, ipp("10.0.0.0/24"), true, false),
 			},
-			isConnected: []map[types.NodeID]bool{
+			isConnected: []types.NodeConnectedMap{
 				// n1 goes down
 				{
 					1: false,
@@ -527,7 +516,7 @@ func TestFailoverNodeRoutesIfNeccessary(t *testing.T) {
 				r(2, 2, ipp("10.0.0.0/24"), true, false),
 				r(3, 3, ipp("10.1.0.0/24"), true, false),
 			},
-			isConnected: []map[types.NodeID]bool{
+			isConnected: []types.NodeConnectedMap{
 				// n1 goes down
 				{
 					1: false,
@@ -550,7 +539,7 @@ func TestFailoverNodeRoutesIfNeccessary(t *testing.T) {
 				r(2, 2, ipp("10.0.0.0/24"), true, false),
 				r(3, 3, ipp("10.1.0.0/24"), false, false),
 			},
-			isConnected: []map[types.NodeID]bool{
+			isConnected: []types.NodeConnectedMap{
 				// n1 goes down
 				{
 					1: false,
@@ -573,7 +562,7 @@ func TestFailoverNodeRoutesIfNeccessary(t *testing.T) {
 				r(2, 2, ipp("10.0.0.0/24"), true, false),
 				r(3, 3, ipp("10.1.0.0/24"), true, false),
 			},
-			isConnected: []map[types.NodeID]bool{
+			isConnected: []types.NodeConnectedMap{
 				// n1 goes down
 				{
 					1: false,
@@ -596,7 +585,7 @@ func TestFailoverNodeRoutesIfNeccessary(t *testing.T) {
 				r(2, 2, ipp("10.0.0.0/24"), true, true),
 				r(3, 3, ipp("10.1.0.0/24"), true, false),
 			},
-			isConnected: []map[types.NodeID]bool{
+			isConnected: []types.NodeConnectedMap{
 				// n1 goes down
 				{
 					1: true,
@@ -629,7 +618,7 @@ func TestFailoverNodeRoutesIfNeccessary(t *testing.T) {
 				want := tt.want[step]
 
 				got, err := Write(db.DB, func(tx *gorm.DB) (*types.StateUpdate, error) {
-					return FailoverNodeRoutesIfNeccessary(tx, smap(isConnected), node)
+					return FailoverNodeRoutesIfNeccessary(tx, isConnected, node)
 				})
 
 				if (err != nil) != tt.wantErr {
@@ -651,7 +640,7 @@ func TestFailoverRouteTx(t *testing.T) {
 		name         string
 		failingRoute types.Route
 		routes       types.Routes
-		isConnected  map[types.NodeID]bool
+		isConnected  types.NodeConnectedMap
 		want         []types.NodeID
 		wantErr      bool
 	}{
@@ -754,7 +743,7 @@ func TestFailoverRouteTx(t *testing.T) {
 					Enabled:   true,
 				},
 			},
-			isConnected: map[types.NodeID]bool{
+			isConnected: types.NodeConnectedMap{
 				1: false,
 				2: true,
 			},
@@ -852,7 +841,7 @@ func TestFailoverRouteTx(t *testing.T) {
 					Enabled:   true,
 				},
 			},
-			isConnected: map[types.NodeID]bool{
+			isConnected: types.NodeConnectedMap{
 				1: true,
 				2: true,
 				3: true,
@@ -900,7 +889,7 @@ func TestFailoverRouteTx(t *testing.T) {
 					Enabled:   true,
 				},
 			},
-			isConnected: map[types.NodeID]bool{
+			isConnected: types.NodeConnectedMap{
 				1: true,
 				4: false,
 			},
@@ -956,7 +945,7 @@ func TestFailoverRouteTx(t *testing.T) {
 					Enabled:   true,
 				},
 			},
-			isConnected: map[types.NodeID]bool{
+			isConnected: types.NodeConnectedMap{
 				1: false,
 				2: true,
 				4: false,
@@ -1021,7 +1010,7 @@ func TestFailoverRouteTx(t *testing.T) {
 			}
 
 			got, err := Write(db.DB, func(tx *gorm.DB) ([]types.NodeID, error) {
-				return failoverRouteTx(tx, smap(tt.isConnected), &tt.failingRoute)
+				return failoverRouteTx(tx, tt.isConnected, &tt.failingRoute)
 			})
 
 			if (err != nil) != tt.wantErr {
@@ -1059,7 +1048,7 @@ func TestFailoverRoute(t *testing.T) {
 		name         string
 		failingRoute types.Route
 		routes       types.Routes
-		isConnected  map[types.NodeID]bool
+		isConnected  types.NodeConnectedMap
 		want         *failover
 	}{
 		{
@@ -1096,7 +1085,7 @@ func TestFailoverRoute(t *testing.T) {
 				r(1, 1, ipp("10.0.0.0/24"), true, true),
 				r(2, 2, ipp("10.0.0.0/24"), true, false),
 			},
-			isConnected: map[types.NodeID]bool{
+			isConnected: types.NodeConnectedMap{
 				1: false,
 				2: true,
 			},
@@ -1122,7 +1111,7 @@ func TestFailoverRoute(t *testing.T) {
 				r(2, 2, ipp("10.0.0.0/24"), true, true),
 				r(3, 3, ipp("10.0.0.0/24"), true, false),
 			},
-			isConnected: map[types.NodeID]bool{
+			isConnected: types.NodeConnectedMap{
 				1: true,
 				2: true,
 				3: true,
@@ -1139,7 +1128,7 @@ func TestFailoverRoute(t *testing.T) {
 				r(1, 1, ipp("10.0.0.0/24"), true, true),
 				r(2, 4, ipp("10.0.0.0/24"), true, false),
 			},
-			isConnected: map[types.NodeID]bool{
+			isConnected: types.NodeConnectedMap{
 				1: true,
 				4: false,
 			},
@@ -1153,7 +1142,7 @@ func TestFailoverRoute(t *testing.T) {
 				r(2, 4, ipp("10.0.0.0/24"), true, false),
 				r(3, 2, ipp("10.0.0.0/24"), true, false),
 			},
-			isConnected: map[types.NodeID]bool{
+			isConnected: types.NodeConnectedMap{
 				1: false,
 				2: true,
 				4: false,
@@ -1183,7 +1172,7 @@ func TestFailoverRoute(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			gotf := failoverRoute(smap(tt.isConnected), &tt.failingRoute, tt.routes)
+			gotf := failoverRoute(tt.isConnected, &tt.failingRoute, tt.routes)
 
 			if tt.want == nil && gotf != nil {
 				t.Fatalf("expected nil, got %+v", gotf)

hscontrol/grpcv1.go

@@ -4,6 +4,7 @@ package hscontrol
 import (
 	"context"
 	"errors"
+	"fmt"
 	"sort"
 	"strings"
 	"time"
@@ -144,7 +145,7 @@ func (api headscaleV1APIServer) ExpirePreAuthKey(
 	ctx context.Context,
 	request *v1.ExpirePreAuthKeyRequest,
 ) (*v1.ExpirePreAuthKeyResponse, error) {
-	err := api.h.db.Write(func(tx *gorm.DB) error {
+	err := api.h.db.DB.Transaction(func(tx *gorm.DB) error {
 		preAuthKey, err := db.GetPreAuthKey(tx, request.GetUser(), request.Key)
 		if err != nil {
 			return err
@@ -278,13 +279,13 @@ func (api headscaleV1APIServer) SetTags(
 
 func validateTag(tag string) error {
 	if strings.Index(tag, "tag:") != 0 {
-		return errors.New("tag must start with the string 'tag:'")
+		return fmt.Errorf("tag must start with the string 'tag:'")
 	}
 	if strings.ToLower(tag) != tag {
-		return errors.New("tag should be lowercase")
+		return fmt.Errorf("tag should be lowercase")
 	}
 	if len(strings.Fields(tag)) > 1 {
-		return errors.New("tag should not contains space")
+		return fmt.Errorf("tag should not contains space")
 	}
 	return nil
 }
@@ -300,7 +301,7 @@ func (api headscaleV1APIServer) DeleteNode(
 
 	changedNodes, err := api.h.db.DeleteNode(
 		node,
-		api.h.nodeNotifier.LikelyConnectedMap(),
+		api.h.nodeNotifier.ConnectedMap(),
 	)
 	if err != nil {
 		return nil, err
@@ -342,7 +343,7 @@ func (api headscaleV1APIServer) ExpireNode(
 	}
 
 	ctx = types.NotifyCtx(ctx, "cli-expirenode-self", node.Hostname)
-	api.h.nodeNotifier.NotifyByNodeID(
+	api.h.nodeNotifier.NotifyByMachineKey(
 		ctx,
 		types.StateUpdate{
 			Type:        types.StateSelfUpdate,
@@ -400,7 +401,7 @@ func (api headscaleV1APIServer) ListNodes(
 	ctx context.Context,
 	request *v1.ListNodesRequest,
 ) (*v1.ListNodesResponse, error) {
-	isLikelyConnected := api.h.nodeNotifier.LikelyConnectedMap()
+	isConnected := api.h.nodeNotifier.ConnectedMap()
 	if request.GetUser() != "" {
 		nodes, err := db.Read(api.h.db.DB, func(rx *gorm.DB) (types.Nodes, error) {
 			return db.ListNodesByUser(rx, request.GetUser())
@@ -415,9 +416,7 @@ func (api headscaleV1APIServer) ListNodes(
 
 			// Populate the online field based on
 			// currently connected nodes.
-			if val, ok := isLikelyConnected.Load(node.ID); ok && val {
-				resp.Online = true
-			}
+			resp.Online = isConnected[node.ID]
 
 			response[index] = resp
 		}
@@ -440,9 +439,7 @@ func (api headscaleV1APIServer) ListNodes(
 
 		// Populate the online field based on
 		// currently connected nodes.
-		if val, ok := isLikelyConnected.Load(node.ID); ok && val {
-			resp.Online = true
-		}
+		resp.Online = isConnected[node.ID]
 
 		validTags, invalidTags := api.h.ACLPolicy.TagsOfNode(
 			node,
@@ -531,7 +528,7 @@ func (api headscaleV1APIServer) DisableRoute(
 	request *v1.DisableRouteRequest,
 ) (*v1.DisableRouteResponse, error) {
 	update, err := db.Write(api.h.db.DB, func(tx *gorm.DB) ([]types.NodeID, error) {
-		return db.DisableRoute(tx, request.GetRouteId(), api.h.nodeNotifier.LikelyConnectedMap())
+		return db.DisableRoute(tx, request.GetRouteId(), api.h.nodeNotifier.ConnectedMap())
 	})
 	if err != nil {
 		return nil, err
@@ -571,7 +568,7 @@ func (api headscaleV1APIServer) DeleteRoute(
 	ctx context.Context,
 	request *v1.DeleteRouteRequest,
 ) (*v1.DeleteRouteResponse, error) {
-	isConnected := api.h.nodeNotifier.LikelyConnectedMap()
+	isConnected := api.h.nodeNotifier.ConnectedMap()
 	update, err := db.Write(api.h.db.DB, func(tx *gorm.DB) ([]types.NodeID, error) {
 		return db.DeleteRoute(tx, request.GetRouteId(), isConnected)
 	})

hscontrol/mapper/mapper.go

@@ -17,7 +17,6 @@ import (
 
 	mapset "github.com/deckarep/golang-set/v2"
 	"github.com/juanfont/headscale/hscontrol/db"
-	"github.com/juanfont/headscale/hscontrol/notifier"
 	"github.com/juanfont/headscale/hscontrol/policy"
 	"github.com/juanfont/headscale/hscontrol/types"
 	"github.com/juanfont/headscale/hscontrol/util"
@@ -52,10 +51,10 @@ var debugDumpMapResponsePath = envknob.String("HEADSCALE_DEBUG_DUMP_MAPRESPONSE_
 type Mapper struct {
 	// Configuration
 	// TODO(kradalby): figure out if this is the format we want this in
-	db      *db.HSDatabase
-	cfg     *types.Config
-	derpMap *tailcfg.DERPMap
-	notif   *notifier.Notifier
+	db                *db.HSDatabase
+	cfg               *types.Config
+	derpMap           *tailcfg.DERPMap
+	isLikelyConnected types.NodeConnectedMap
 
 	uid     string
 	created time.Time
@@ -71,15 +70,15 @@ func NewMapper(
 	db *db.HSDatabase,
 	cfg *types.Config,
 	derpMap *tailcfg.DERPMap,
-	notif *notifier.Notifier,
+	isLikelyConnected types.NodeConnectedMap,
 ) *Mapper {
 	uid, _ := util.GenerateRandomStringDNSSafe(mapperIDLength)
 
 	return &Mapper{
-		db:      db,
-		cfg:     cfg,
-		derpMap: derpMap,
-		notif:   notif,
+		db:                db,
+		cfg:               cfg,
+		derpMap:           derpMap,
+		isLikelyConnected: isLikelyConnected,
 
 		uid:     uid,
 		created: time.Now(),
@@ -518,7 +517,7 @@ func (m *Mapper) ListPeers(nodeID types.NodeID) (types.Nodes, error) {
 	}
 
 	for _, peer := range peers {
-		online := m.notif.IsLikelyConnected(peer.ID)
+		online := m.isLikelyConnected[peer.ID]
 		peer.IsOnline = &online
 	}
 

hscontrol/metrics.go

@@ -1,10 +1,6 @@
 package hscontrol
 
 import (
-	"net/http"
-	"strconv"
-
-	"github.com/gorilla/mux"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 )
@@ -12,94 +8,18 @@ import (
 const prometheusNamespace = "headscale"
 
 var (
-	mapResponseSent = promauto.NewCounterVec(prometheus.CounterOpts{
+	// This is a high cardinality metric (user x node), we might want to make this
+	// configurable/opt-in in the future.
+	nodeRegistrations = promauto.NewCounterVec(prometheus.CounterOpts{
 		Namespace: prometheusNamespace,
-		Name:      "mapresponse_sent_total",
-		Help:      "total count of mapresponses sent to clients",
-	}, []string{"status", "type"})
-	mapResponseUpdateReceived = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name:      "node_registrations_total",
+		Help:      "The total amount of registered node attempts",
+	}, []string{"action", "auth", "status", "user"})
+
+	updateRequestsSentToNode = promauto.NewCounterVec(prometheus.CounterOpts{
 		Namespace: prometheusNamespace,
-		Name:      "mapresponse_updates_received_total",
-		Help:      "total count of mapresponse updates received on update channel",
-	}, []string{"type"})
-	mapResponseWriteUpdatesInStream = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: prometheusNamespace,
-		Name:      "mapresponse_write_updates_in_stream_total",
-		Help:      "total count of writes that occured in a stream session, pre-68 nodes",
-	}, []string{"status"})
-	mapResponseEndpointUpdates = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: prometheusNamespace,
-		Name:      "mapresponse_endpoint_updates_total",
-		Help:      "total count of endpoint updates received",
-	}, []string{"status"})
-	mapResponseReadOnly = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: prometheusNamespace,
-		Name:      "mapresponse_readonly_requests_total",
-		Help:      "total count of readonly requests received",
-	}, []string{"status"})
-	mapResponseSessions = promauto.NewGauge(prometheus.GaugeOpts{
-		Namespace: prometheusNamespace,
-		Name:      "mapresponse_current_sessions_total",
-		Help:      "total count open map response sessions",
-	})
-	mapResponseRejected = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: prometheusNamespace,
-		Name:      "mapresponse_rejected_new_sessions_total",
-		Help:      "total count of new mapsessions rejected",
-	}, []string{"reason"})
-	httpDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
-		Namespace: prometheusNamespace,
-		Name:      "http_duration_seconds",
-		Help:      "Duration of HTTP requests.",
-	}, []string{"path"})
-	httpCounter = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: prometheusNamespace,
-		Name:      "http_requests_total",
-		Help:      "Total number of http requests processed",
-	}, []string{"code", "method", "path"},
-	)
+		Name:      "update_request_sent_to_node_total",
+		Help:      "The number of calls/messages issued on a specific nodes update channel",
+	}, []string{"user", "node", "status"})
+	// TODO(kradalby): This is very debugging, we might want to remove it.
 )
-
-// prometheusMiddleware implements mux.MiddlewareFunc.
-func prometheusMiddleware(next http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		route := mux.CurrentRoute(r)
-		path, _ := route.GetPathTemplate()
-
-		// Ignore streaming and noise sessions
-		// it has its own router further down.
-		if path == "/ts2021" || path == "/machine/map" || path == "/derp" || path == "/derp/probe" || path == "/bootstrap-dns" {
-			next.ServeHTTP(w, r)
-			return
-		}
-
-		rw := &respWriterProm{ResponseWriter: w}
-
-		timer := prometheus.NewTimer(httpDuration.WithLabelValues(path))
-		next.ServeHTTP(rw, r)
-		timer.ObserveDuration()
-		httpCounter.WithLabelValues(strconv.Itoa(rw.status), r.Method, path).Inc()
-	})
-}
-
-type respWriterProm struct {
-	http.ResponseWriter
-	status      int
-	written     int64
-	wroteHeader bool
-}
-
-func (r *respWriterProm) WriteHeader(code int) {
-	r.status = code
-	r.wroteHeader = true
-	r.ResponseWriter.WriteHeader(code)
-}
-
-func (r *respWriterProm) Write(b []byte) (int, error) {
-	if !r.wroteHeader {
-		r.WriteHeader(http.StatusOK)
-	}
-	n, err := r.ResponseWriter.Write(b)
-	r.written += int64(n)
-	return n, err
-}

hscontrol/noise.go

@@ -95,7 +95,6 @@ func (h *Headscale) NoiseUpgradeHandler(
 	// The HTTP2 server that exposes this router is created for
 	// a single hijacked connection from /ts2021, using netutil.NewOneConnListener
 	router := mux.NewRouter()
-	router.Use(prometheusMiddleware)
 
 	router.HandleFunc("/machine/register", noiseServer.NoiseRegistrationHandler).
 		Methods(http.MethodPost)
@@ -226,6 +225,7 @@ func (ns *noiseServer) NoisePollNetMapHandler(
 	if err != nil {
 		log.Error().
 			Str("handler", "NoisePollNetMap").
+			Uint64("node.id", node.ID.Uint64()).
 			Msgf("Failed to fetch node from the database with node key: %s", mapRequest.NodeKey.String())
 		http.Error(writer, "Internal error", http.StatusInternalServerError)
 
@@ -267,12 +267,10 @@ func (ns *noiseServer) NoisePollNetMapHandler(
 			defer ns.headscale.mapSessionMu.Unlock()
 
 			sess.infof("node has an open stream(%p), rejecting new stream", sess)
-			mapResponseRejected.WithLabelValues("exists").Inc()
 			return
 		}
 
 		ns.headscale.mapSessions[node.ID] = sess
-		mapResponseSessions.Inc()
 		ns.headscale.mapSessionMu.Unlock()
 		sess.tracef("releasing lock to check stream")
 	}
@@ -285,7 +283,6 @@ func (ns *noiseServer) NoisePollNetMapHandler(
 		defer ns.headscale.mapSessionMu.Unlock()
 
 		delete(ns.headscale.mapSessions, node.ID)
-		mapResponseSessions.Dec()
 
 		sess.tracef("releasing lock to remove stream")
 	}

hscontrol/notifier/metrics.go

@@ -1,27 +0,0 @@
-package notifier
-
-import (
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-)
-
-const prometheusNamespace = "headscale"
-
-var (
-	notifierWaitForLock = promauto.NewHistogramVec(prometheus.HistogramOpts{
-		Namespace: prometheusNamespace,
-		Name:      "notifier_wait_for_lock_seconds",
-		Help:      "histogram of time spent waiting for the notifier lock",
-		Buckets:   []float64{0.001, 0.01, 0.1, 0.3, 0.5, 1, 3, 5, 10},
-	}, []string{"action"})
-	notifierUpdateSent = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: prometheusNamespace,
-		Name:      "notifier_update_sent_total",
-		Help:      "total count of update sent on nodes channel",
-	}, []string{"status", "type"})
-	notifierNodeUpdateChans = promauto.NewGauge(prometheus.GaugeOpts{
-		Namespace: prometheusNamespace,
-		Name:      "notifier_open_channels_total",
-		Help:      "total count open channels in notifier",
-	})
-)

hscontrol/notifier/notifier.go

@@ -6,23 +6,21 @@ import (
 	"slices"
 	"strings"
 	"sync"
-	"time"
 
 	"github.com/juanfont/headscale/hscontrol/types"
-	"github.com/puzpuzpuz/xsync/v3"
 	"github.com/rs/zerolog/log"
 )
 
 type Notifier struct {
 	l         sync.RWMutex
 	nodes     map[types.NodeID]chan<- types.StateUpdate
-	connected *xsync.MapOf[types.NodeID, bool]
+	connected types.NodeConnectedMap
 }
 
 func NewNotifier() *Notifier {
 	return &Notifier{
 		nodes:     make(map[types.NodeID]chan<- types.StateUpdate),
-		connected: xsync.NewMapOf[types.NodeID, bool](),
+		connected: make(types.NodeConnectedMap),
 	}
 }
 
@@ -33,19 +31,16 @@ func (n *Notifier) AddNode(nodeID types.NodeID, c chan<- types.StateUpdate) {
 		Uint64("node.id", nodeID.Uint64()).
 		Msg("releasing lock to add node")
 
-	start := time.Now()
 	n.l.Lock()
 	defer n.l.Unlock()
-	notifierWaitForLock.WithLabelValues("add").Observe(time.Since(start).Seconds())
 
 	n.nodes[nodeID] = c
-	n.connected.Store(nodeID, true)
+	n.connected[nodeID] = true
 
 	log.Trace().
 		Uint64("node.id", nodeID.Uint64()).
 		Int("open_chans", len(n.nodes)).
 		Msg("Added new channel")
-	notifierNodeUpdateChans.Inc()
 }
 
 func (n *Notifier) RemoveNode(nodeID types.NodeID) {
@@ -55,23 +50,20 @@ func (n *Notifier) RemoveNode(nodeID types.NodeID) {
 		Uint64("node.id", nodeID.Uint64()).
 		Msg("releasing lock to remove node")
 
-	start := time.Now()
 	n.l.Lock()
 	defer n.l.Unlock()
-	notifierWaitForLock.WithLabelValues("remove").Observe(time.Since(start).Seconds())
 
 	if len(n.nodes) == 0 {
 		return
 	}
 
 	delete(n.nodes, nodeID)
-	n.connected.Store(nodeID, false)
+	n.connected[nodeID] = false
 
 	log.Trace().
 		Uint64("node.id", nodeID.Uint64()).
 		Int("open_chans", len(n.nodes)).
 		Msg("Removed channel")
-	notifierNodeUpdateChans.Dec()
 }
 
 // IsConnected reports if a node is connected to headscale and has a
@@ -80,22 +72,17 @@ func (n *Notifier) IsConnected(nodeID types.NodeID) bool {
 	n.l.RLock()
 	defer n.l.RUnlock()
 
-	if val, ok := n.connected.Load(nodeID); ok {
-		return val
-	}
-	return false
+	return n.connected[nodeID]
 }
 
 // IsLikelyConnected reports if a node is connected to headscale and has a
 // poll session open, but doesnt lock, so might be wrong.
 func (n *Notifier) IsLikelyConnected(nodeID types.NodeID) bool {
-	if val, ok := n.connected.Load(nodeID); ok {
-		return val
-	}
-	return false
+	return n.connected[nodeID]
 }
 
-func (n *Notifier) LikelyConnectedMap() *xsync.MapOf[types.NodeID, bool] {
+// TODO(kradalby): This returns a pointer and can be dangerous.
+func (n *Notifier) ConnectedMap() types.NodeConnectedMap {
 	return n.connected
 }
 
@@ -108,16 +95,45 @@ func (n *Notifier) NotifyWithIgnore(
 	update types.StateUpdate,
 	ignoreNodeIDs ...types.NodeID,
 ) {
-	for nodeID := range n.nodes {
+	log.Trace().Caller().Str("type", update.Type.String()).Msg("acquiring lock to notify")
+	defer log.Trace().
+		Caller().
+		Str("type", update.Type.String()).
+		Msg("releasing lock, finished notifying")
+
+	n.l.RLock()
+	defer n.l.RUnlock()
+
+	if update.Type == types.StatePeerChangedPatch {
+		log.Trace().Interface("update", update).Interface("online", n.connected).Msg("PATCH UPDATE SENT")
+	}
+
+	for nodeID, c := range n.nodes {
 		if slices.Contains(ignoreNodeIDs, nodeID) {
 			continue
 		}
 
-		n.NotifyByNodeID(ctx, update, nodeID)
+		select {
+		case <-ctx.Done():
+			log.Error().
+				Err(ctx.Err()).
+				Uint64("node.id", nodeID.Uint64()).
+				Any("origin", ctx.Value("origin")).
+				Any("origin-hostname", ctx.Value("hostname")).
+				Msgf("update not sent, context cancelled")
+
+			return
+		case c <- update:
+			log.Trace().
+				Uint64("node.id", nodeID.Uint64()).
+				Any("origin", ctx.Value("origin")).
+				Any("origin-hostname", ctx.Value("hostname")).
+				Msgf("update successfully sent on chan")
+		}
 	}
 }
 
-func (n *Notifier) NotifyByNodeID(
+func (n *Notifier) NotifyByMachineKey(
 	ctx context.Context,
 	update types.StateUpdate,
 	nodeID types.NodeID,
@@ -128,10 +144,8 @@ func (n *Notifier) NotifyByNodeID(
 		Str("type", update.Type.String()).
 		Msg("releasing lock, finished notifying")
 
-	start := time.Now()
 	n.l.RLock()
 	defer n.l.RUnlock()
-	notifierWaitForLock.WithLabelValues("notify").Observe(time.Since(start).Seconds())
 
 	if c, ok := n.nodes[nodeID]; ok {
 		select {
@@ -142,7 +156,6 @@ func (n *Notifier) NotifyByNodeID(
 				Any("origin", ctx.Value("origin")).
 				Any("origin-hostname", ctx.Value("hostname")).
 				Msgf("update not sent, context cancelled")
-			notifierUpdateSent.WithLabelValues("cancelled", update.Type.String()).Inc()
 
 			return
 		case c <- update:
@@ -151,7 +164,6 @@ func (n *Notifier) NotifyByNodeID(
 				Any("origin", ctx.Value("origin")).
 				Any("origin-hostname", ctx.Value("hostname")).
 				Msgf("update successfully sent on chan")
-			notifierUpdateSent.WithLabelValues("ok", update.Type.String()).Inc()
 		}
 	}
 }
@@ -170,10 +182,9 @@ func (n *Notifier) String() string {
 	b.WriteString("\n")
 	b.WriteString("connected:\n")
 
-	n.connected.Range(func(k types.NodeID, v bool) bool {
+	for k, v := range n.connected {
 		fmt.Fprintf(&b, "\t%d: %t\n", k, v)
-		return true
-	})
+	}
 
 	return b.String()
 }

hscontrol/oidc.go

@@ -602,7 +602,7 @@ func (h *Headscale) registerNodeForOIDCCallback(
 		return err
 	}
 
-	if err := h.db.Write(func(tx *gorm.DB) error {
+	if err := h.db.DB.Transaction(func(tx *gorm.DB) error {
 		if _, err := db.RegisterNodeFromAuthCallback(
 			// TODO(kradalby): find a better way to use the cache across modules
 			tx,

hscontrol/poll.go

@@ -64,7 +64,7 @@ func (h *Headscale) newMapSession(
 	w http.ResponseWriter,
 	node *types.Node,
 ) *mapSession {
-	warnf, infof, tracef, errf := logPollFunc(req, node)
+	warnf, tracef, infof, errf := logPollFunc(req, node)
 
 	// Use a buffered channel in case a node is not fully ready
 	// to receive a message to make sure we dont block the entire
@@ -196,10 +196,8 @@ func (m *mapSession) serve() {
 		// return
 		err := m.handleSaveNode()
 		if err != nil {
-			mapResponseWriteUpdatesInStream.WithLabelValues("error").Inc()
 			return
 		}
-		mapResponseWriteUpdatesInStream.WithLabelValues("ok").Inc()
 	}
 
 	// Set up the client stream
@@ -286,7 +284,6 @@ func (m *mapSession) serve() {
 				patches = filteredPatches
 			}
 
-			updateType := "full"
 			// When deciding what update to send, the following is considered,
 			// Full is a superset of all updates, when a full update is requested,
 			// send only that and move on, all other updates will be present in
@@ -306,15 +303,12 @@ func (m *mapSession) serve() {
 			} else if changed != nil {
 				m.tracef(fmt.Sprintf("Sending Changed MapResponse: %v", lastMessage))
 				data, err = m.mapper.PeerChangedResponse(m.req, m.node, changed, patches, m.h.ACLPolicy, lastMessage)
-				updateType = "change"
 			} else if patches != nil {
 				m.tracef(fmt.Sprintf("Sending Changed Patch MapResponse: %v", lastMessage))
 				data, err = m.mapper.PeerChangedPatchResponse(m.req, m.node, patches, m.h.ACLPolicy)
-				updateType = "patch"
 			} else if derp {
 				m.tracef("Sending DERPUpdate MapResponse")
 				data, err = m.mapper.DERPMapResponse(m.req, m.node, m.h.DERPMap)
-				updateType = "derp"
 			}
 
 			if err != nil {
@@ -330,22 +324,19 @@ func (m *mapSession) serve() {
 				startWrite := time.Now()
 				_, err = m.w.Write(data)
 				if err != nil {
-					mapResponseSent.WithLabelValues("error", updateType).Inc()
 					m.errf(err, "Could not write the map response, for mapSession: %p", m)
 					return
 				}
 
 				err = rc.Flush()
 				if err != nil {
-					mapResponseSent.WithLabelValues("error", updateType).Inc()
 					m.errf(err, "flushing the map response to client, for mapSession: %p", m)
 					return
 				}
 
 				log.Trace().Str("node", m.node.Hostname).TimeDiff("timeSpent", time.Now(), startWrite).Str("mkey", m.node.MachineKey.String()).Msg("finished writing mapresp to node")
 
-				mapResponseSent.WithLabelValues("ok", updateType).Inc()
-				m.tracef("update sent")
+				m.infof("update sent")
 			}
 
 			// reset
@@ -373,8 +364,7 @@ func (m *mapSession) serve() {
 
 		// Consume all updates sent to node
 		case update := <-m.ch:
-			m.tracef("received stream update: %s %s", update.Type.String(), update.Message)
-			mapResponseUpdateReceived.WithLabelValues(update.Type.String()).Inc()
+			m.tracef("received stream update: %d %s", update.Type, update.Message)
 
 			switch update.Type {
 			case types.StateFullUpdate:
@@ -414,30 +404,27 @@ func (m *mapSession) serve() {
 			data, err := m.mapper.KeepAliveResponse(m.req, m.node)
 			if err != nil {
 				m.errf(err, "Error generating the keep alive msg")
-				mapResponseSent.WithLabelValues("error", "keepalive").Inc()
+
 				return
 			}
 			_, err = m.w.Write(data)
 			if err != nil {
 				m.errf(err, "Cannot write keep alive message")
-				mapResponseSent.WithLabelValues("error", "keepalive").Inc()
+
 				return
 			}
 			err = rc.Flush()
 			if err != nil {
 				m.errf(err, "flushing keep alive to client, for mapSession: %p", m)
-				mapResponseSent.WithLabelValues("error", "keepalive").Inc()
 				return
 			}
-
-			mapResponseSent.WithLabelValues("ok", "keepalive").Inc()
 		}
 	}
 }
 
 func (m *mapSession) pollFailoverRoutes(where string, node *types.Node) {
 	update, err := db.Write(m.h.db.DB, func(tx *gorm.DB) (*types.StateUpdate, error) {
-		return db.FailoverNodeRoutesIfNeccessary(tx, m.h.nodeNotifier.LikelyConnectedMap(), node)
+		return db.FailoverNodeRoutesIfNeccessary(tx, m.h.nodeNotifier.ConnectedMap(), node)
 	})
 	if err != nil {
 		m.errf(err, fmt.Sprintf("failed to ensure failover routes, %s", where))
@@ -467,7 +454,7 @@ func (h *Headscale) updateNodeOnlineStatus(online bool, node *types.Node) {
 		node.LastSeen = &now
 		change.LastSeen = &now
 
-		err := h.db.Write(func(tx *gorm.DB) error {
+		err := h.db.DB.Transaction(func(tx *gorm.DB) error {
 			return db.SetLastSeen(tx, node.ID, *node.LastSeen)
 		})
 		if err != nil {
@@ -514,7 +501,6 @@ func (m *mapSession) handleEndpointUpdate() {
 	// If there is no changes and nothing to save,
 	// return early.
 	if peerChangeEmpty(change) && !sendUpdate {
-		mapResponseEndpointUpdates.WithLabelValues("noop").Inc()
 		return
 	}
 
@@ -532,7 +518,6 @@ func (m *mapSession) handleEndpointUpdate() {
 		if err != nil {
 			m.errf(err, "Error processing node routes")
 			http.Error(m.w, "", http.StatusInternalServerError)
-			mapResponseEndpointUpdates.WithLabelValues("error").Inc()
 
 			return
 		}
@@ -542,7 +527,6 @@ func (m *mapSession) handleEndpointUpdate() {
 			err := m.h.db.EnableAutoApprovedRoutes(m.h.ACLPolicy, m.node)
 			if err != nil {
 				m.errf(err, "Error running auto approved routes")
-				mapResponseEndpointUpdates.WithLabelValues("error").Inc()
 			}
 		}
 
@@ -550,19 +534,19 @@ func (m *mapSession) handleEndpointUpdate() {
 		// has an updated packetfilter allowing the new route
 		// if it is defined in the ACL.
 		ctx := types.NotifyCtx(context.Background(), "poll-nodeupdate-self-hostinfochange", m.node.Hostname)
-		m.h.nodeNotifier.NotifyByNodeID(
+		m.h.nodeNotifier.NotifyByMachineKey(
 			ctx,
 			types.StateUpdate{
 				Type:        types.StateSelfUpdate,
 				ChangeNodes: []types.NodeID{m.node.ID},
 			},
 			m.node.ID)
+
 	}
 
 	if err := m.h.db.DB.Save(m.node).Error; err != nil {
 		m.errf(err, "Failed to persist/update node in the database")
 		http.Error(m.w, "", http.StatusInternalServerError)
-		mapResponseEndpointUpdates.WithLabelValues("error").Inc()
 
 		return
 	}
@@ -578,7 +562,6 @@ func (m *mapSession) handleEndpointUpdate() {
 		m.node.ID)
 
 	m.w.WriteHeader(http.StatusOK)
-	mapResponseEndpointUpdates.WithLabelValues("ok").Inc()
 
 	return
 }
@@ -656,7 +639,7 @@ func (m *mapSession) handleReadOnlyRequest() {
 	if err != nil {
 		m.errf(err, "Failed to create MapResponse")
 		http.Error(m.w, "", http.StatusInternalServerError)
-		mapResponseReadOnly.WithLabelValues("error").Inc()
+
 		return
 	}
 
@@ -665,12 +648,9 @@ func (m *mapSession) handleReadOnlyRequest() {
 	_, err = m.w.Write(mapResp)
 	if err != nil {
 		m.errf(err, "Failed to write response")
-		mapResponseReadOnly.WithLabelValues("error").Inc()
-		return
 	}
 
 	m.w.WriteHeader(http.StatusOK)
-	mapResponseReadOnly.WithLabelValues("ok").Inc()
 
 	return
 }

hscontrol/types/node.go

@@ -28,8 +28,7 @@ var (
 )
 
 type NodeID uint64
-
-// type NodeConnectedMap *xsync.MapOf[NodeID, bool]
+type NodeConnectedMap map[NodeID]bool
 
 func (id NodeID) StableID() tailcfg.StableNodeID {
 	return tailcfg.StableNodeID(strconv.FormatUint(uint64(id), util.Base10))

integration/acl_test.go

@@ -51,7 +51,7 @@ func aclScenario(
 	clientsPerUser int,
 ) *Scenario {
 	t.Helper()
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 
 	spec := map[string]int{
@@ -264,7 +264,7 @@ func TestACLHostsInNetMapTable(t *testing.T) {
 
 	for name, testCase := range tests {
 		t.Run(name, func(t *testing.T) {
-			scenario, err := NewScenario(dockertestMaxWait())
+			scenario, err := NewScenario()
 			assertNoErr(t, err)
 
 			spec := testCase.users

integration/auth_oidc_test.go

@@ -42,7 +42,7 @@ func TestOIDCAuthenticationPingAll(t *testing.T) {
 	IntegrationSkip(t)
 	t.Parallel()
 
-	baseScenario, err := NewScenario(dockertestMaxWait())
+	baseScenario, err := NewScenario()
 	assertNoErr(t, err)
 
 	scenario := AuthOIDCScenario{
@@ -100,7 +100,7 @@ func TestOIDCExpireNodesBasedOnTokenExpiry(t *testing.T) {
 
 	shortAccessTTL := 5 * time.Minute
 
-	baseScenario, err := NewScenario(dockertestMaxWait())
+	baseScenario, err := NewScenario()
 	assertNoErr(t, err)
 
 	baseScenario.pool.MaxWait = 5 * time.Minute

integration/auth_web_flow_test.go

@@ -26,7 +26,7 @@ func TestAuthWebFlowAuthenticationPingAll(t *testing.T) {
 	IntegrationSkip(t)
 	t.Parallel()
 
-	baseScenario, err := NewScenario(dockertestMaxWait())
+	baseScenario, err := NewScenario()
 	if err != nil {
 		t.Fatalf("failed to create scenario: %s", err)
 	}
@@ -67,7 +67,7 @@ func TestAuthWebFlowLogoutAndRelogin(t *testing.T) {
 	IntegrationSkip(t)
 	t.Parallel()
 
-	baseScenario, err := NewScenario(dockertestMaxWait())
+	baseScenario, err := NewScenario()
 	assertNoErr(t, err)
 
 	scenario := AuthWebFlowScenario{

integration/cli_test.go

@@ -32,7 +32,7 @@ func TestUserCommand(t *testing.T) {
 	IntegrationSkip(t)
 	t.Parallel()
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 
@@ -112,7 +112,7 @@ func TestPreAuthKeyCommand(t *testing.T) {
 	user := "preauthkeyspace"
 	count := 3
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 
@@ -254,7 +254,7 @@ func TestPreAuthKeyCommandWithoutExpiry(t *testing.T) {
 
 	user := "pre-auth-key-without-exp-user"
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 
@@ -317,7 +317,7 @@ func TestPreAuthKeyCommandReusableEphemeral(t *testing.T) {
 
 	user := "pre-auth-key-reus-ephm-user"
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 
@@ -394,7 +394,7 @@ func TestApiKeyCommand(t *testing.T) {
 
 	count := 5
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 
@@ -562,7 +562,7 @@ func TestNodeTagCommand(t *testing.T) {
 	IntegrationSkip(t)
 	t.Parallel()
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 
@@ -695,7 +695,7 @@ func TestNodeAdvertiseTagNoACLCommand(t *testing.T) {
 	IntegrationSkip(t)
 	t.Parallel()
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 
@@ -745,7 +745,7 @@ func TestNodeAdvertiseTagWithACLCommand(t *testing.T) {
 	IntegrationSkip(t)
 	t.Parallel()
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 
@@ -808,7 +808,7 @@ func TestNodeCommand(t *testing.T) {
 	IntegrationSkip(t)
 	t.Parallel()
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 
@@ -1049,7 +1049,7 @@ func TestNodeExpireCommand(t *testing.T) {
 	IntegrationSkip(t)
 	t.Parallel()
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 
@@ -1176,7 +1176,7 @@ func TestNodeRenameCommand(t *testing.T) {
 	IntegrationSkip(t)
 	t.Parallel()
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 
@@ -1343,7 +1343,7 @@ func TestNodeMoveCommand(t *testing.T) {
 	IntegrationSkip(t)
 	t.Parallel()
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 

integration/embedded_derp_test.go

@@ -23,7 +23,7 @@ func TestDERPServerScenario(t *testing.T) {
 	IntegrationSkip(t)
 	// t.Parallel()
 
-	baseScenario, err := NewScenario(dockertestMaxWait())
+	baseScenario, err := NewScenario()
 	assertNoErr(t, err)
 
 	scenario := EmbeddedDERPServerScenario{

integration/general_test.go

@@ -23,7 +23,7 @@ func TestPingAllByIP(t *testing.T) {
 	IntegrationSkip(t)
 	t.Parallel()
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 
@@ -67,7 +67,7 @@ func TestPingAllByIPPublicDERP(t *testing.T) {
 	IntegrationSkip(t)
 	t.Parallel()
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 
@@ -105,7 +105,7 @@ func TestAuthKeyLogoutAndRelogin(t *testing.T) {
 	IntegrationSkip(t)
 	t.Parallel()
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 
@@ -216,7 +216,7 @@ func TestEphemeral(t *testing.T) {
 	IntegrationSkip(t)
 	t.Parallel()
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 
@@ -299,7 +299,7 @@ func TestPingAllByHostname(t *testing.T) {
 	IntegrationSkip(t)
 	t.Parallel()
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 
@@ -348,7 +348,7 @@ func TestTaildrop(t *testing.T) {
 		return err
 	}
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 
@@ -509,7 +509,7 @@ func TestResolveMagicDNS(t *testing.T) {
 	IntegrationSkip(t)
 	t.Parallel()
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 
@@ -577,7 +577,7 @@ func TestExpireNode(t *testing.T) {
 	IntegrationSkip(t)
 	t.Parallel()
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 
@@ -703,7 +703,7 @@ func TestNodeOnlineStatus(t *testing.T) {
 	IntegrationSkip(t)
 	t.Parallel()
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 
@@ -818,7 +818,7 @@ func TestPingAllByIPManyUpDown(t *testing.T) {
 	IntegrationSkip(t)
 	t.Parallel()
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 

integration/hsic/hsic.go

@@ -18,7 +18,6 @@ import (
 	"net/url"
 	"os"
 	"path"
-	"strconv"
 	"strings"
 	"time"
 
@@ -202,14 +201,6 @@ func WithEmbeddedDERPServerOnly() Option {
 	}
 }
 
-// WithTuning allows changing the tuning settings easily.
-func WithTuning(batchTimeout time.Duration, mapSessionChanSize int) Option {
-	return func(hsic *HeadscaleInContainer) {
-		hsic.env["HEADSCALE_TUNING_BATCH_CHANGE_DELAY"] = batchTimeout.String()
-		hsic.env["HEADSCALE_TUNING_NODE_MAPSESSION_BUFFERED_CHAN_SIZE"] = strconv.Itoa(mapSessionChanSize)
-	}
-}
-
 // New returns a new HeadscaleInContainer instance.
 func New(
 	pool *dockertest.Pool,

integration/route_test.go

@@ -28,7 +28,7 @@ func TestEnablingRoutes(t *testing.T) {
 
 	user := "enable-routing"
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErrf(t, "failed to create scenario: %s", err)
 	defer scenario.Shutdown()
 
@@ -250,7 +250,7 @@ func TestHASubnetRouterFailover(t *testing.T) {
 
 	user := "enable-routing"
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErrf(t, "failed to create scenario: %s", err)
 	// defer scenario.Shutdown()
 
@@ -822,7 +822,7 @@ func TestEnableDisableAutoApprovedRoute(t *testing.T) {
 
 	user := "enable-disable-routing"
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErrf(t, "failed to create scenario: %s", err)
 	defer scenario.Shutdown()
 
@@ -966,7 +966,7 @@ func TestSubnetRouteACL(t *testing.T) {
 
 	user := "subnet-route-acl"
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErrf(t, "failed to create scenario: %s", err)
 	defer scenario.Shutdown()
 

integration/scenario.go

@@ -8,7 +8,6 @@ import (
 	"os"
 	"sort"
 	"sync"
-	"time"
 
 	v1 "github.com/juanfont/headscale/gen/go/headscale/v1"
 	"github.com/juanfont/headscale/hscontrol/util"
@@ -142,7 +141,7 @@ type Scenario struct {
 
 // NewScenario creates a test Scenario which can be used to bootstraps a ControlServer with
 // a set of Users and TailscaleClients.
-func NewScenario(maxWait time.Duration) (*Scenario, error) {
+func NewScenario() (*Scenario, error) {
 	hash, err := util.GenerateRandomStringDNSSafe(scenarioHashLength)
 	if err != nil {
 		return nil, err
@@ -153,7 +152,7 @@ func NewScenario(maxWait time.Duration) (*Scenario, error) {
 		return nil, fmt.Errorf("could not connect to docker: %w", err)
 	}
 
-	pool.MaxWait = maxWait
+	pool.MaxWait = dockertestMaxWait()
 
 	networkName := fmt.Sprintf("hs-%s", hash)
 	if overrideNetworkName := os.Getenv("HEADSCALE_TEST_NETWORK_NAME"); overrideNetworkName != "" {
@@ -511,7 +510,7 @@ func (s *Scenario) GetIPs(user string) ([]netip.Addr, error) {
 	return ips, fmt.Errorf("failed to get ips: %w", errNoUserAvailable)
 }
 
-// GetClients returns all TailscaleClients associated with a User in a Scenario.
+// GetIPs returns all TailscaleClients associated with a User in a Scenario.
 func (s *Scenario) GetClients(user string) ([]TailscaleClient, error) {
 	var clients []TailscaleClient
 	if ns, ok := s.users[user]; ok {
@@ -587,7 +586,7 @@ func (s *Scenario) ListTailscaleClientsIPs(users ...string) ([]netip.Addr, error
 	return allIps, nil
 }
 
-// ListTailscaleClientsFQDNs returns a list of FQDN based on Users
+// ListTailscaleClientsIPs returns a list of FQDN based on Users
 // passed as parameters.
 func (s *Scenario) ListTailscaleClientsFQDNs(users ...string) ([]string, error) {
 	allFQDNs := make([]string, 0)

integration/scenario_test.go

@@ -33,7 +33,7 @@ func TestHeadscale(t *testing.T) {
 
 	user := "test-space"
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 
@@ -78,7 +78,7 @@ func TestCreateTailscale(t *testing.T) {
 
 	user := "only-create-containers"
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 
@@ -114,7 +114,7 @@ func TestTailscaleNodesJoiningHeadcale(t *testing.T) {
 
 	count := 1
 
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 	defer scenario.Shutdown()
 

integration/ssh_test.go

@@ -44,7 +44,7 @@ var retry = func(times int, sleepInterval time.Duration,
 
 func sshScenario(t *testing.T, policy *policy.ACLPolicy, clientsPerUser int) *Scenario {
 	t.Helper()
-	scenario, err := NewScenario(dockertestMaxWait())
+	scenario, err := NewScenario()
 	assertNoErr(t, err)
 
 	spec := map[string]int{