refactor(server): improve proxy protocol handling

.vscode/settings.example.json

@@ -1,10 +1,10 @@
 {
   "yaml.schemas": {
-    "https://github.com/yusing/godoxy-webui/raw/refs/heads/main/src/types/godoxy/config.schema.json": [
+    "https://github.com/yusing/godoxy-webui/raw/refs/heads/main/types/godoxy/config.schema.json": [
       "config.example.yml",
       "config.yml"
     ],
-    "https://github.com/yusing/godoxy-webui/raw/refs/heads/main/src/types/godoxy/routes.schema.json": [
+    "https://github.com/yusing/godoxy-webui/raw/refs/heads/main/types/godoxy/routes.schema.json": [
       "providers.example.yml"
     ]
   }

agent/go.mod

@@ -71,6 +71,7 @@ require (
 	github.com/opencontainers/image-spec v1.1.1 // indirect
 	github.com/oschwald/maxminddb-golang v1.13.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
+	github.com/pires/go-proxyproto v0.8.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect

agent/go.sum

@@ -128,6 +128,8 @@ github.com/oschwald/maxminddb-golang v1.13.1 h1:G3wwjdN9JmIK2o/ermkHM+98oX5fS+k5
 github.com/oschwald/maxminddb-golang v1.13.1/go.mod h1:K4pgV9N/GcK694KSTmVSDTODk4IsCNThNdTmnaBZ/F8=
 github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
+github.com/pires/go-proxyproto v0.8.1 h1:9KEixbdJfhrbtjpz/ZwCdWDD2Xem0NZ38qMYaASJgp0=
+github.com/pires/go-proxyproto v0.8.1/go.mod h1:ZKAAyp3cgy5Y5Mo4n9AlScrkCZwUy0g3Jf+slqQVcuU=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

agent/pkg/agent/http_requests.go

@@ -19,7 +19,6 @@ func (cfg *AgentConfig) Do(ctx context.Context, method, endpoint string, body io
 }
 
 func (cfg *AgentConfig) Forward(req *http.Request, endpoint string) (*http.Response, error) {
-	req = req.WithContext(req.Context())
 	req.URL.Host = AgentHost
 	req.URL.Scheme = "https"
 	req.URL.Path = APIEndpointBase + endpoint
@@ -56,17 +55,11 @@ func (cfg *AgentConfig) Websocket(ctx context.Context, endpoint string) (*websoc
 //
 // It will create a new request with the same context, method, and body, but with the agent host and scheme, and the endpoint
 // If the request has a query, it will be added to the proxy request's URL
-func (cfg *AgentConfig) ReverseProxy(w http.ResponseWriter, req *http.Request, endpoint string) error {
+func (cfg *AgentConfig) ReverseProxy(w http.ResponseWriter, req *http.Request, endpoint string) {
 	rp := reverseproxy.NewReverseProxy("agent", nettypes.NewURL(AgentURL), cfg.Transport())
-	uri := APIEndpointBase + endpoint
-	if req.URL.RawQuery != "" {
-		uri += "?" + req.URL.RawQuery
-	}
-	r, err := http.NewRequestWithContext(req.Context(), req.Method, uri, req.Body)
-	if err != nil {
-		return err
-	}
-	r.Header = req.Header
-	rp.ServeHTTP(w, r)
-	return nil
+	req.URL.Host = AgentHost
+	req.URL.Scheme = "https"
+	req.URL.Path = endpoint
+	req.RequestURI = ""
+	rp.ServeHTTP(w, req)
 }

agent/pkg/server/server.go

@@ -39,5 +39,5 @@ func StartAgentServer(parent task.Parent, opt Options) {
 		TLSConfig: tlsConfig,
 	}
 
-	server.Start(parent, agentServer, nil, &log.Logger)
+	server.Start(parent, agentServer, server.WithLogger(&log.Logger))
 }

config.example.yml

@@ -17,6 +17,10 @@
 
 # 3. other providers, see https://docs.godoxy.dev/DNS-01-Providers
 
+# Access Control
+# When enabled, it will be applied globally at connection level,
+# all incoming connections (web, tcp and udp) will be checked against the ACL rules.
+
 # acl:
 #   default: allow # or deny (default: allow)
 #   allow_local: true # or false (default: true)
@@ -37,6 +41,11 @@
 #     keep: last 10 # (default: none)
 
 entrypoint:
+  # Proxy Protocol: https://www.haproxy.com/blog/use-the-proxy-protocol-to-preserve-a-clients-ip-address
+  # When set to true, web entrypoint and all tcp routeswill be wrapped with Proxy Protocol listener in order to preserve the client's IP address.
+  # Note that HTTP/3 with proxy protocol is not supported yet.
+  support_proxy_protocol: false
+
   # Below define an example of middleware config
   # 1. set security headers
   # 2. block non local IP connections
@@ -57,14 +66,6 @@ entrypoint:
         X-Frame-Options: SAMEORIGIN
         Referrer-Policy: same-origin
         Strict-Transport-Security: max-age=63072000; includeSubDomains; preload
-    # - use: CIDRWhitelist
-    #   allow:
-    #     - "127.0.0.1"
-    #     - "10.0.0.0/8"
-    #     - "172.16.0.0/12"
-    #     - "192.168.0.0/16"
-    #   status: 403
-    #   message: "Forbidden"
     # - use: RedirectHTTP
 
   # below enables access log
@@ -115,8 +116,8 @@ providers:
   #     secret: aaaa-bbbb-cccc-dddd
   #     no_tls_verify: true
 
-# Check https://docs.godoxy.dev/Certificates-and-domain-matching
-# for explaination of `match_domains`
+# Match domains
+# See https://docs.godoxy.dev/Certificates-and-domain-matching
 #
 # match_domains:
 #   - my.site

go.mod

@@ -213,6 +213,7 @@ require (
 
 require (
 	github.com/gin-gonic/gin v1.10.1
+	github.com/pires/go-proxyproto v0.8.1
 	github.com/yusing/ds v0.1.0
 )
 

go.sum

@@ -1424,6 +1424,8 @@ github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
 github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pires/go-proxyproto v0.8.1 h1:9KEixbdJfhrbtjpz/ZwCdWDD2Xem0NZ38qMYaASJgp0=
+github.com/pires/go-proxyproto v0.8.1/go.mod h1:ZKAAyp3cgy5Y5Mo4n9AlScrkCZwUy0g3Jf+slqQVcuU=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=

internal/api/v1/metrics/system_info.go

@@ -46,6 +46,7 @@ func SystemInfo(c *gin.Context) {
 		systeminfo.Poller.ServeHTTP(c)
 		return
 	}
+	c.Request.URL.RawQuery = query.Encode()
 
 	agent, ok := agentPkg.GetAgent(agentAddr)
 	if !ok {
@@ -69,10 +70,6 @@ func SystemInfo(c *gin.Context) {
 		c.Status(resp.StatusCode)
 		io.Copy(c.Writer, resp.Body)
 	} else {
-		err := agent.ReverseProxy(c.Writer, c.Request, agentPkg.EndpointSystemInfo+"?"+query.Encode())
-		if err != nil {
-			c.Error(apitypes.InternalServerError(err, "failed to reverse proxy"))
-			return
-		}
+		agent.ReverseProxy(c.Writer, c.Request, agentPkg.EndpointSystemInfo)
 	}
 }

internal/config/config.go

@@ -204,12 +204,13 @@ func (cfg *Config) StartServers(opts ...*StartServersOptions) {
 	opt := opts[0]
 	if opt.Proxy {
 		server.StartServer(cfg.task, server.Options{
-			Name:         "proxy",
-			CertProvider: cfg.AutoCertProvider(),
-			HTTPAddr:     common.ProxyHTTPAddr,
-			HTTPSAddr:    common.ProxyHTTPSAddr,
-			Handler:      cfg.entrypoint,
-			ACL:          cfg.value.ACL,
+			Name:                 "proxy",
+			CertProvider:         cfg.AutoCertProvider(),
+			HTTPAddr:             common.ProxyHTTPAddr,
+			HTTPSAddr:            common.ProxyHTTPSAddr,
+			Handler:              cfg.entrypoint,
+			ACL:                  cfg.value.ACL,
+			SupportProxyProtocol: cfg.value.Entrypoint.SupportProxyProtocol,
 		})
 	}
 	if opt.API {

internal/config/types/config.go

@@ -37,8 +37,9 @@ type (
 		MaxMind      *maxmind.Config            `json:"maxmind" yaml:"maxmind,omitempty"`
 	}
 	Entrypoint struct {
-		Middlewares []map[string]any               `json:"middlewares"`
-		AccessLog   *accesslog.RequestLoggerConfig `json:"access_log" validate:"omitempty"`
+		SupportProxyProtocol bool                           `json:"support_proxy_protocol"`
+		Middlewares          []map[string]any               `json:"middlewares"`
+		AccessLog            *accesslog.RequestLoggerConfig `json:"access_log" validate:"omitempty"`
 	}
 	HomepageConfig struct {
 		UseDefaultCategories bool `json:"use_default_categories"`

internal/net/gphttp/server/server.go

@@ -9,6 +9,8 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/pires/go-proxyproto"
+	h2proxy "github.com/pires/go-proxyproto/helper/http2"
 	"github.com/quic-go/quic-go/http3"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
@@ -28,6 +30,7 @@ type Server struct {
 	https        *http.Server
 	startTime    time.Time
 	acl          *acl.Config
+	proxyProto   bool
 
 	l zerolog.Logger
 }
@@ -39,6 +42,8 @@ type Options struct {
 	CertProvider CertProvider
 	Handler      http.Handler
 	ACL          *acl.Config
+
+	SupportProxyProtocol bool
 }
 
 type httpServer interface {
@@ -86,6 +91,7 @@ func NewServer(opt Options) (s *Server) {
 		https:        httpsSer,
 		l:            logger,
 		acl:          opt.ACL,
+		proxyProto:   opt.SupportProxyProtocol,
 	}
 }
 
@@ -99,30 +105,86 @@ func (s *Server) Start(parent task.Parent) {
 	subtask := parent.Subtask("server."+s.Name, false)
 
 	if s.https != nil && common.HTTP3Enabled {
-		s.https.TLSConfig.NextProtos = []string{http3.NextProtoH3, "h2", "http/1.1"}
-		h3 := &http3.Server{
-			Addr:      s.https.Addr,
-			Handler:   s.https.Handler,
-			TLSConfig: http3.ConfigureTLSConfig(s.https.TLSConfig),
+		if s.proxyProto {
+			// TODO: support proxy protocol for HTTP/3
+			s.l.Warn().Msg("HTTP/3 is enabled, but proxy protocol is yet not supported for HTTP/3")
+		} else {
+			s.https.TLSConfig.NextProtos = []string{http3.NextProtoH3, "h2", "http/1.1"}
+			h3 := &http3.Server{
+				Addr:      s.https.Addr,
+				Handler:   s.https.Handler,
+				TLSConfig: http3.ConfigureTLSConfig(s.https.TLSConfig),
+			}
+			Start(subtask, h3, WithProxyProtocolSupport(s.proxyProto), WithACL(s.acl), WithLogger(&s.l))
+			if s.http != nil {
+				s.http.Handler = advertiseHTTP3(s.http.Handler, h3)
+			}
+			// s.https is not nil (checked above)
+			s.https.Handler = advertiseHTTP3(s.https.Handler, h3)
 		}
-		Start(subtask, h3, s.acl, &s.l)
-		if s.http != nil {
-			s.http.Handler = advertiseHTTP3(s.http.Handler, h3)
-		}
-		// s.https is not nil (checked above)
-		s.https.Handler = advertiseHTTP3(s.https.Handler, h3)
 	}
 
-	Start(subtask, s.http, s.acl, &s.l)
-	Start(subtask, s.https, s.acl, &s.l)
+	Start(subtask, s.http, WithProxyProtocolSupport(s.proxyProto), WithACL(s.acl), WithLogger(&s.l))
+	Start(subtask, s.https, WithProxyProtocolSupport(s.proxyProto), WithACL(s.acl), WithLogger(&s.l))
 }
 
-func Start[Server httpServer](parent task.Parent, srv Server, acl *acl.Config, logger *zerolog.Logger) (port int) {
+type ServerStartOptions struct {
+	tcpWrappers []func(l net.Listener) net.Listener
+	udpWrappers []func(l net.PacketConn) net.PacketConn
+	logger      *zerolog.Logger
+	proxyProto  bool
+}
+
+type ServerStartOption func(opts *ServerStartOptions)
+
+func WithTCPWrappers(wrappers ...func(l net.Listener) net.Listener) ServerStartOption {
+	return func(opts *ServerStartOptions) {
+		opts.tcpWrappers = wrappers
+	}
+}
+
+func WithUDPWrappers(wrappers ...func(l net.PacketConn) net.PacketConn) ServerStartOption {
+	return func(opts *ServerStartOptions) {
+		opts.udpWrappers = wrappers
+	}
+}
+
+func WithLogger(logger *zerolog.Logger) ServerStartOption {
+	return func(opts *ServerStartOptions) {
+		opts.logger = logger
+	}
+}
+
+func WithACL(acl *acl.Config) ServerStartOption {
+	return func(opts *ServerStartOptions) {
+		if acl == nil {
+			return
+		}
+		opts.tcpWrappers = append(opts.tcpWrappers, acl.WrapTCP)
+		opts.udpWrappers = append(opts.udpWrappers, acl.WrapUDP)
+	}
+}
+
+func WithProxyProtocolSupport(value bool) ServerStartOption {
+	return func(opts *ServerStartOptions) {
+		opts.proxyProto = value
+	}
+}
+
+func Start[Server httpServer](parent task.Parent, srv Server, optFns ...ServerStartOption) (port int) {
 	if srv == nil {
 		return
 	}
 
-	setDebugLogger(srv, logger)
+	var opts ServerStartOptions
+	for _, optFn := range optFns {
+		optFn(&opts)
+	}
+	if opts.logger == nil {
+		opts.logger = &log.Logger
+	}
+
+	setDebugLogger(srv, opts.logger)
 
 	proto := proto(srv)
 	task := parent.Subtask(proto, true)
@@ -137,40 +199,47 @@ func Start[Server httpServer](parent task.Parent, srv Server, acl *acl.Config, l
 		}
 		l, err := lc.Listen(task.Context(), "tcp", srv.Addr)
 		if err != nil {
-			HandleError(logger, err, "failed to listen on port")
+			HandleError(opts.logger, err, "failed to listen on port")
 			return
 		}
 		port = l.Addr().(*net.TCPAddr).Port
+		if opts.proxyProto {
+			l = &proxyproto.Listener{Listener: l}
+		}
 		if srv.TLSConfig != nil {
 			l = tls.NewListener(l, srv.TLSConfig)
 		}
-		if acl != nil {
-			l = acl.WrapTCP(l)
+		for _, wrapper := range opts.tcpWrappers {
+			l = wrapper(l)
+		}
+		if opts.proxyProto {
+			serveFunc = getServeFunc(l, h2proxy.NewServer(srv, nil).Serve)
+		} else {
+			serveFunc = getServeFunc(l, srv.Serve)
 		}
-		serveFunc = getServeFunc(l, srv.Serve)
 		task.OnCancel("stop", func() {
-			stop(srv, l, logger)
+			stop(srv, l, opts.logger)
 		})
 	case *http3.Server:
 		l, err := lc.ListenPacket(task.Context(), "udp", srv.Addr)
 		if err != nil {
-			HandleError(logger, err, "failed to listen on port")
+			HandleError(opts.logger, err, "failed to listen on port")
 			return
 		}
 		port = l.LocalAddr().(*net.UDPAddr).Port
-		if acl != nil {
-			l = acl.WrapUDP(l)
+		for _, wrapper := range opts.udpWrappers {
+			l = wrapper(l)
 		}
 		serveFunc = getServeFunc(l, srv.Serve)
 		task.OnCancel("stop", func() {
-			stop(srv, l, logger)
+			stop(srv, l, opts.logger)
 		})
 	}
-	logStarted(srv, logger)
+	logStarted(srv, opts.logger)
 	go func() {
 		err := convertError(serveFunc())
 		if err != nil {
-			HandleError(logger, err, "failed to serve "+proto+" server")
+			HandleError(opts.logger, err, "failed to serve "+proto+" server")
 		}
 		task.Finish(err)
 	}()

internal/route/stream/tcp_tcp.go

@@ -4,14 +4,16 @@ import (
 	"context"
 	"net"
 
+	"github.com/pires/go-proxyproto"
 	"github.com/rs/zerolog"
+	config "github.com/yusing/go-proxy/internal/config/types"
 	nettypes "github.com/yusing/go-proxy/internal/net/types"
 	"github.com/yusing/go-proxy/internal/utils"
 	"go.uber.org/atomic"
 )
 
 type TCPTCPStream struct {
-	listener *net.TCPListener
+	listener net.Listener
 	laddr    *net.TCPAddr
 	dst      *net.TCPAddr
 
@@ -34,12 +36,20 @@ func NewTCPTCPStream(listenAddr, dstAddr string) (nettypes.Stream, error) {
 }
 
 func (s *TCPTCPStream) ListenAndServe(ctx context.Context, preDial, onRead nettypes.HookFunc) {
-	listener, err := net.ListenTCP("tcp", s.laddr)
+	var err error
+	s.listener, err = net.ListenTCP("tcp", s.laddr)
 	if err != nil {
 		logErr(s, err, "failed to listen")
 		return
 	}
-	s.listener = listener
+
+	if proxyProto := config.GetInstance().Value().Entrypoint.SupportProxyProtocol; proxyProto {
+		s.listener = &proxyproto.Listener{Listener: s.listener}
+	}
+	if acl := config.GetInstance().Value().ACL; acl != nil {
+		s.listener = acl.WrapTCP(s.listener)
+	}
+
 	s.preDial = preDial
 	s.onRead = onRead
 	go s.listen(ctx)

internal/route/stream/udp_udp.go

@@ -3,12 +3,14 @@ package stream
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"maps"
 	"net"
 	"sync"
 	"time"
 
 	"github.com/rs/zerolog"
+	config "github.com/yusing/go-proxy/internal/config/types"
 	nettypes "github.com/yusing/go-proxy/internal/net/types"
 	"github.com/yusing/go-proxy/internal/utils/synk"
 	"go.uber.org/atomic"
@@ -16,7 +18,7 @@ import (
 
 type UDPUDPStream struct {
 	name     string
-	listener *net.UDPConn
+	listener net.PacketConn
 
 	laddr *net.UDPAddr
 	dst   *net.UDPAddr
@@ -34,7 +36,7 @@ type UDPUDPStream struct {
 type udpUDPConn struct {
 	srcAddr  *net.UDPAddr
 	dstConn  *net.UDPConn
-	listener *net.UDPConn
+	listener net.PacketConn
 	lastUsed atomic.Time
 	closed   atomic.Bool
 	mu       sync.Mutex
@@ -66,12 +68,15 @@ func NewUDPUDPStream(listenAddr, dstAddr string) (nettypes.Stream, error) {
 }
 
 func (s *UDPUDPStream) ListenAndServe(ctx context.Context, preDial, onRead nettypes.HookFunc) {
-	listener, err := net.ListenUDP("udp", s.laddr)
+	var err error
+	s.listener, err = net.ListenUDP("udp", s.laddr)
 	if err != nil {
 		logErr(s, err, "failed to listen")
 		return
 	}
-	s.listener = listener
+	if acl := config.GetInstance().Value().ACL; acl != nil {
+		s.listener = acl.WrapUDP(s.listener)
+	}
 	s.preDial = preDial
 	s.onRead = onRead
 	go s.listen(ctx)
@@ -120,7 +125,7 @@ func (s *UDPUDPStream) listen(ctx context.Context) {
 		case <-ctx.Done():
 			return
 		default:
-			n, srcAddr, err := s.listener.ReadFromUDP(buf)
+			n, srcAddr, err := s.listener.ReadFrom(buf)
 			if err != nil {
 				if s.closed.Load() {
 					return
@@ -129,6 +134,12 @@ func (s *UDPUDPStream) listen(ctx context.Context) {
 				continue
 			}
 
+			srcAddrUDP, ok := srcAddr.(*net.UDPAddr)
+			if !ok {
+				logErr(s, fmt.Errorf("unexpected source address type: %T", srcAddr), "unexpected source address type")
+				continue
+			}
+
 			logDebugf(s, "read %d bytes from %s", n, srcAddr)
 
 			if s.onRead != nil {
@@ -139,7 +150,7 @@ func (s *UDPUDPStream) listen(ctx context.Context) {
 			}
 
 			// Get or create connection, passing the initial data
-			go s.getOrCreateConnection(ctx, srcAddr, bytes.Clone(buf[:n]))
+			go s.getOrCreateConnection(ctx, srcAddrUDP, bytes.Clone(buf[:n]))
 		}
 	}
 }
@@ -233,7 +244,7 @@ func (conn *udpUDPConn) handleResponses(ctx context.Context) {
 			_ = conn.dstConn.SetReadDeadline(time.Time{})
 
 			// Forward response back to client using the listener
-			_, err = conn.listener.WriteToUDP(buf[:n], conn.srcAddr)
+			_, err = conn.listener.WriteTo(buf[:n], conn.srcAddr)
 			if err != nil {
 				if !conn.closed.Load() {
 					logErrf(conn, err, "failed to write %d bytes to client", n)