fix: optimize memory usage, fix agent and code refactor (#118)

* refactor: simplify io code and make utils module independent

* fix(docker): agent and socket-proxy docker event flushing with modified reverse proxy handler

* refactor: remove unused code

* refactor: remove the use of logging module in most code

* refactor: streamline domain mismatch check in certState function

* tweak: use ecdsa p-256 for autocert

* fix(tests): update health check tests for invalid host and add case for port in host

* feat(acme): custom acme directory

* refactor: code refactor and improved context and error handling

* tweak: optimize memory usage under load

* fix(oidc): restore old user matching behavior

* docs: add ChatGPT assistant to README

---------

Co-authored-by: yusing <yusing@6uo.me>

internal/net/gphttp/body.go

@@ -6,7 +6,7 @@ import (
 	"errors"
 	"net/http"
 
-	"github.com/yusing/go-proxy/internal/logging"
+	"github.com/rs/zerolog/log"
 )
 
 func WriteBody(w http.ResponseWriter, body []byte) {
@@ -14,9 +14,9 @@ func WriteBody(w http.ResponseWriter, body []byte) {
 		switch {
 		case errors.Is(err, http.ErrHandlerTimeout),
 			errors.Is(err, context.DeadlineExceeded):
-			logging.Err(err).Msg("timeout writing body")
+			log.Err(err).Msg("timeout writing body")
 		default:
-			logging.Err(err).Msg("failed to write body")
+			log.Err(err).Msg("failed to write body")
 		}
 	}
 }

internal/net/gphttp/gpwebsocket/utils.go

@@ -9,11 +9,11 @@ import (
 	"time"
 
 	"github.com/gorilla/websocket"
-	"github.com/yusing/go-proxy/internal/logging"
+	"github.com/rs/zerolog/log"
 )
 
 func warnNoMatchDomains() {
-	logging.Warn().Msg("no match domains configured, accepting websocket API request from all origins")
+	log.Warn().Msg("no match domains configured, accepting websocket API request from all origins")
 }
 
 var warnNoMatchDomainOnce sync.Once

internal/net/gphttp/loadbalancer/loadbalancer.go

@@ -7,8 +7,8 @@ import (
 	"time"
 
 	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
 	"github.com/yusing/go-proxy/internal/gperr"
-	"github.com/yusing/go-proxy/internal/logging"
 	"github.com/yusing/go-proxy/internal/net/gphttp/httpheaders"
 	"github.com/yusing/go-proxy/internal/net/gphttp/loadbalancer/types"
 	"github.com/yusing/go-proxy/internal/task"
@@ -47,7 +47,7 @@ func New(cfg *Config) *LoadBalancer {
 	lb := &LoadBalancer{
 		Config: new(Config),
 		pool:   pool.New[Server]("loadbalancer." + cfg.Link),
-		l:      logging.With().Str("name", cfg.Link).Logger(),
+		l:      log.With().Str("name", cfg.Link).Logger(),
 	}
 	lb.UpdateConfigIfNeeded(cfg)
 	return lb

internal/net/gphttp/loadbalancer/types/weight.go

@@ -1,3 +1,3 @@
 package types
 
-type Weight uint16
+type Weight int

internal/net/gphttp/logging.go

@@ -4,14 +4,14 @@ import (
 	"net/http"
 
 	"github.com/rs/zerolog"
-	"github.com/yusing/go-proxy/internal/logging"
+	"github.com/rs/zerolog/log"
 )
 
 func reqLogger(r *http.Request, level zerolog.Level) *zerolog.Event {
-	return logging.WithLevel(level).
-		Str("remote", r.RemoteAddr).
-		Str("host", r.Host).
-		Str("uri", r.Method+" "+r.RequestURI)
+	return log.WithLevel(level). //nolint:zerologlint
+					Str("remote", r.RemoteAddr).
+					Str("host", r.Host).
+					Str("uri", r.Method+" "+r.RequestURI)
 }
 
 func LogError(r *http.Request) *zerolog.Event { return reqLogger(r, zerolog.ErrorLevel) }

internal/net/gphttp/middleware/captcha/middleware.go

@@ -4,8 +4,8 @@ import (
 	"net/http"
 	"text/template"
 
+	"github.com/rs/zerolog/log"
 	"github.com/yusing/go-proxy/internal/auth"
-	"github.com/yusing/go-proxy/internal/logging"
 	"github.com/yusing/go-proxy/internal/net/gphttp"
 
 	_ "embed"
@@ -55,7 +55,7 @@ func PreRequest(p Provider, w http.ResponseWriter, r *http.Request) (proceed boo
 		"FormHTML":   p.FormHTML(),
 	})
 	if err != nil {
-		logging.Error().Err(err).Msg("failed to execute captcha page")
+		log.Error().Err(err).Msg("failed to execute captcha page")
 	}
 	return false
 }

internal/net/gphttp/middleware/cidr_whitelist.go

@@ -7,7 +7,7 @@ import (
 	"github.com/go-playground/validator/v10"
 	gphttp "github.com/yusing/go-proxy/internal/net/gphttp"
 	"github.com/yusing/go-proxy/internal/net/types"
-	"github.com/yusing/go-proxy/internal/utils"
+	"github.com/yusing/go-proxy/internal/serialization"
 	F "github.com/yusing/go-proxy/internal/utils/functional"
 )
 
@@ -34,7 +34,7 @@ var (
 )
 
 func init() {
-	utils.MustRegisterValidation("status_code", func(fl validator.FieldLevel) bool {
+	serialization.MustRegisterValidation("status_code", func(fl validator.FieldLevel) bool {
 		statusCode := fl.Field().Int()
 		return gphttp.IsStatusCodeValid(int(statusCode))
 	})
@@ -60,7 +60,7 @@ func (wl *cidrWhitelist) checkIP(w http.ResponseWriter, r *http.Request) bool {
 			ipStr = r.RemoteAddr
 		}
 		ip := net.ParseIP(ipStr)
-		for _, cidr := range wl.CIDRWhitelistOpts.Allow {
+		for _, cidr := range wl.Allow {
 			if cidr.Contains(ip) {
 				wl.cachedAddr.Store(r.RemoteAddr, true)
 				allow = true
@@ -70,7 +70,7 @@ func (wl *cidrWhitelist) checkIP(w http.ResponseWriter, r *http.Request) bool {
 		}
 		if !allow {
 			wl.cachedAddr.Store(r.RemoteAddr, false)
-			wl.AddTracef("client %s is forbidden", ipStr).With("allowed CIDRs", wl.CIDRWhitelistOpts.Allow)
+			wl.AddTracef("client %s is forbidden", ipStr).With("allowed CIDRs", wl.Allow)
 		}
 	}
 	if !allow {

internal/net/gphttp/middleware/cidr_whitelist_test.go

@@ -8,7 +8,7 @@ import (
 	"testing"
 
 	"github.com/yusing/go-proxy/internal/gperr"
-	"github.com/yusing/go-proxy/internal/utils"
+	"github.com/yusing/go-proxy/internal/serialization"
 	. "github.com/yusing/go-proxy/internal/utils/testing"
 )
 
@@ -41,7 +41,7 @@ func TestCIDRWhitelistValidation(t *testing.T) {
 		_, err := CIDRWhiteList.New(OptionsRaw{
 			"message": testMessage,
 		})
-		ExpectError(t, utils.ErrValidationError, err)
+		ExpectError(t, serialization.ErrValidationError, err)
 	})
 	t.Run("invalid cidr", func(t *testing.T) {
 		_, err := CIDRWhiteList.New(OptionsRaw{
@@ -56,7 +56,7 @@ func TestCIDRWhitelistValidation(t *testing.T) {
 			"status_code": 600,
 			"message":     testMessage,
 		})
-		ExpectError(t, utils.ErrValidationError, err)
+		ExpectError(t, serialization.ErrValidationError, err)
 	})
 }
 

internal/net/gphttp/middleware/cloudflare_real_ip.go

@@ -1,6 +1,7 @@
 package middleware
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io"
@@ -9,8 +10,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/rs/zerolog/log"
 	"github.com/yusing/go-proxy/internal/common"
-	"github.com/yusing/go-proxy/internal/logging"
 	"github.com/yusing/go-proxy/internal/net/types"
 	"github.com/yusing/go-proxy/internal/utils/atomic"
 	"github.com/yusing/go-proxy/internal/utils/strutils"
@@ -89,21 +90,29 @@ func tryFetchCFCIDR() (cfCIDRs []*types.CIDR) {
 		)
 		if err != nil {
 			cfCIDRsLastUpdate.Store(time.Now().Add(-cfCIDRsUpdateRetryInterval - cfCIDRsUpdateInterval))
-			logging.Err(err).Msg("failed to update cloudflare range, retry in " + strutils.FormatDuration(cfCIDRsUpdateRetryInterval))
+			log.Err(err).Msg("failed to update cloudflare range, retry in " + strutils.FormatDuration(cfCIDRsUpdateRetryInterval))
 			return nil
 		}
 		if len(cfCIDRs) == 0 {
-			logging.Warn().Msg("cloudflare CIDR range is empty")
+			log.Warn().Msg("cloudflare CIDR range is empty")
 		}
 	}
 
 	cfCIDRsLastUpdate.Store(time.Now())
-	logging.Info().Msg("cloudflare CIDR range updated")
+	log.Info().Msg("cloudflare CIDR range updated")
 	return
 }
 
 func fetchUpdateCFIPRange(endpoint string, cfCIDRs *[]*types.CIDR) error {
-	resp, err := http.Get(endpoint)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
+	if err != nil {
+		return err
+	}
+
+	resp, err := http.DefaultClient.Do(req) //nolint:gosec
 	if err != nil {
 		return err
 	}

internal/net/gphttp/middleware/custom_error_page.go

@@ -8,7 +8,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/yusing/go-proxy/internal/logging"
+	"github.com/rs/zerolog/log"
 	gphttp "github.com/yusing/go-proxy/internal/net/gphttp"
 	"github.com/yusing/go-proxy/internal/net/gphttp/httpheaders"
 	"github.com/yusing/go-proxy/internal/net/gphttp/middleware/errorpage"
@@ -32,7 +32,7 @@ func (customErrorPage) modifyResponse(resp *http.Response) error {
 	if !gphttp.IsSuccess(resp.StatusCode) && (contentType.IsHTML() || contentType.IsPlainText()) {
 		errorPage, ok := errorpage.GetErrorPageByStatus(resp.StatusCode)
 		if ok {
-			logging.Debug().Msgf("error page for status %d loaded", resp.StatusCode)
+			log.Debug().Msgf("error page for status %d loaded", resp.StatusCode)
 			_, _ = io.Copy(io.Discard, resp.Body) // drain the original body
 			resp.Body.Close()
 			resp.Body = io.NopCloser(bytes.NewReader(errorPage))
@@ -40,7 +40,7 @@ func (customErrorPage) modifyResponse(resp *http.Response) error {
 			resp.Header.Set(httpheaders.HeaderContentLength, strconv.Itoa(len(errorPage)))
 			resp.Header.Set(httpheaders.HeaderContentType, "text/html; charset=utf-8")
 		} else {
-			logging.Error().Msgf("unable to load error page for status %d", resp.StatusCode)
+			log.Error().Msgf("unable to load error page for status %d", resp.StatusCode)
 		}
 		return nil
 	}
@@ -56,7 +56,7 @@ func ServeStaticErrorPageFile(w http.ResponseWriter, r *http.Request) (served bo
 		filename := path[len(StaticFilePathPrefix):]
 		file, ok := errorpage.GetStaticFile(filename)
 		if !ok {
-			logging.Error().Msg("unable to load resource " + filename)
+			log.Error().Msg("unable to load resource " + filename)
 			return false
 		}
 		ext := filepath.Ext(filename)
@@ -68,10 +68,10 @@ func ServeStaticErrorPageFile(w http.ResponseWriter, r *http.Request) (served bo
 		case ".css":
 			w.Header().Set(httpheaders.HeaderContentType, "text/css; charset=utf-8")
 		default:
-			logging.Error().Msgf("unexpected file type %q for %s", ext, filename)
+			log.Error().Msgf("unexpected file type %q for %s", ext, filename)
 		}
 		if _, err := w.Write(file); err != nil {
-			logging.Err(err).Msg("unable to write resource " + filename)
+			log.Err(err).Msg("unable to write resource " + filename)
 			http.Error(w, "Error page failure", http.StatusInternalServerError)
 		}
 		return true

internal/net/gphttp/middleware/errorpage/error_page.go

@@ -6,9 +6,9 @@ import (
 	"path"
 	"sync"
 
+	"github.com/rs/zerolog/log"
 	"github.com/yusing/go-proxy/internal/common"
 	"github.com/yusing/go-proxy/internal/gperr"
-	"github.com/yusing/go-proxy/internal/logging"
 	"github.com/yusing/go-proxy/internal/task"
 	U "github.com/yusing/go-proxy/internal/utils"
 	F "github.com/yusing/go-proxy/internal/utils/functional"
@@ -48,7 +48,7 @@ func GetErrorPageByStatus(statusCode int) (content []byte, ok bool) {
 func loadContent() {
 	files, err := U.ListFiles(errPagesBasePath, 0)
 	if err != nil {
-		logging.Err(err).Msg("failed to list error page resources")
+		log.Err(err).Msg("failed to list error page resources")
 		return
 	}
 	for _, file := range files {
@@ -57,11 +57,11 @@ func loadContent() {
 		}
 		content, err := os.ReadFile(file)
 		if err != nil {
-			logging.Warn().Err(err).Msgf("failed to read error page resource %s", file)
+			log.Warn().Err(err).Msgf("failed to read error page resource %s", file)
 			continue
 		}
 		file = path.Base(file)
-		logging.Info().Msgf("error page resource %s loaded", file)
+		log.Info().Msgf("error page resource %s loaded", file)
 		fileContentMap.Store(file, content)
 	}
 }
@@ -83,9 +83,9 @@ func watchDir() {
 				loadContent()
 			case events.ActionFileDeleted:
 				fileContentMap.Delete(filename)
-				logging.Warn().Msgf("error page resource %s deleted", filename)
+				log.Warn().Msgf("error page resource %s deleted", filename)
 			case events.ActionFileRenamed:
-				logging.Warn().Msgf("error page resource %s deleted", filename)
+				log.Warn().Msgf("error page resource %s deleted", filename)
 				fileContentMap.Delete(filename)
 				loadContent()
 			}

internal/net/gphttp/middleware/middleware.go

@@ -8,11 +8,11 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/rs/zerolog/log"
 	"github.com/yusing/go-proxy/internal/gperr"
-	"github.com/yusing/go-proxy/internal/logging"
 	gphttp "github.com/yusing/go-proxy/internal/net/gphttp"
 	"github.com/yusing/go-proxy/internal/net/gphttp/reverseproxy"
-	"github.com/yusing/go-proxy/internal/utils"
+	"github.com/yusing/go-proxy/internal/serialization"
 )
 
 type (
@@ -87,7 +87,7 @@ func NewMiddleware[ImplType any]() *Middleware {
 func (m *Middleware) enableTrace() {
 	if tracer, ok := m.impl.(MiddlewareWithTracer); ok {
 		tracer.enableTrace()
-		logging.Trace().Msgf("middleware %s enabled trace", m.name)
+		log.Trace().Msgf("middleware %s enabled trace", m.name)
 	}
 }
 
@@ -118,14 +118,14 @@ func (m *Middleware) apply(optsRaw OptionsRaw) gperr.Error {
 		"priority": optsRaw["priority"],
 		"bypass":   optsRaw["bypass"],
 	}
-	if err := utils.MapUnmarshalValidate(commonOpts, &m.commonOptions); err != nil {
+	if err := serialization.MapUnmarshalValidate(commonOpts, &m.commonOptions); err != nil {
 		return err
 	}
 	optsRaw = maps.Clone(optsRaw)
 	for k := range commonOpts {
 		delete(optsRaw, k)
 	}
-	return utils.MapUnmarshalValidate(optsRaw, m.impl)
+	return serialization.MapUnmarshalValidate(optsRaw, m.impl)
 }
 
 func (m *Middleware) finalize() error {

internal/net/gphttp/middleware/middlewares.go

@@ -3,9 +3,9 @@ package middleware
 import (
 	"path"
 
+	"github.com/rs/zerolog/log"
 	"github.com/yusing/go-proxy/internal/common"
 	"github.com/yusing/go-proxy/internal/gperr"
-	"github.com/yusing/go-proxy/internal/logging"
 	"github.com/yusing/go-proxy/internal/utils"
 	"github.com/yusing/go-proxy/internal/utils/strutils"
 )
@@ -59,7 +59,7 @@ func LoadComposeFiles() {
 	errs := gperr.NewBuilder("middleware compile errors")
 	middlewareDefs, err := utils.ListFiles(common.MiddlewareComposeBasePath, 0)
 	if err != nil {
-		logging.Err(err).Msg("failed to list middleware definitions")
+		log.Err(err).Msg("failed to list middleware definitions")
 		return
 	}
 	for _, defFile := range middlewareDefs {
@@ -75,7 +75,7 @@ func LoadComposeFiles() {
 				continue
 			}
 			allMiddlewares[name] = m
-			logging.Info().
+			log.Info().
 				Str("src", path.Base(defFile)).
 				Str("name", name).
 				Msg("middleware loaded")
@@ -94,7 +94,7 @@ func LoadComposeFiles() {
 				continue
 			}
 			allMiddlewares[name] = m
-			logging.Info().
+			log.Info().
 				Str("src", path.Base(defFile)).
 				Str("name", name).
 				Msg("middleware loaded")

internal/net/gphttp/reverseproxy/reverse_proxy.go

@@ -25,7 +25,7 @@ import (
 	"sync"
 
 	"github.com/rs/zerolog"
-	"github.com/yusing/go-proxy/internal/logging"
+	"github.com/rs/zerolog/log"
 	"github.com/yusing/go-proxy/internal/logging/accesslog"
 	"github.com/yusing/go-proxy/internal/net/gphttp/httpheaders"
 	"github.com/yusing/go-proxy/internal/net/types"
@@ -138,7 +138,7 @@ func NewReverseProxy(name string, target *types.URL, transport http.RoundTripper
 		panic("nil transport")
 	}
 	rp := &ReverseProxy{
-		Logger:     logging.With().Str("name", name).Logger(),
+		Logger:     log.With().Str("name", name).Logger(),
 		Transport:  transport,
 		TargetName: name,
 		TargetURL:  target,
@@ -173,17 +173,17 @@ func (p *ReverseProxy) errorHandler(rw http.ResponseWriter, r *http.Request, err
 	case errors.Is(err, context.Canceled),
 		errors.Is(err, io.EOF),
 		errors.Is(err, context.DeadlineExceeded):
-		logging.Debug().Err(err).Str("url", reqURL).Msg("http proxy error")
+		log.Debug().Err(err).Str("url", reqURL).Msg("http proxy error")
 	default:
 		var recordErr tls.RecordHeaderError
 		if errors.As(err, &recordErr) {
-			logging.Error().
+			log.Error().
 				Str("url", reqURL).
 				Msgf(`scheme was likely misconfigured as https,
 						try setting "proxy.%s.scheme" back to "http"`, p.TargetName)
-			logging.Err(err).Msg("underlying error")
+			log.Err(err).Msg("underlying error")
 		} else {
-			logging.Err(err).Str("url", reqURL).Msg("http proxy error")
+			log.Err(err).Str("url", reqURL).Msg("http proxy error")
 		}
 	}
 
@@ -220,7 +220,6 @@ func (p *ReverseProxy) handler(rw http.ResponseWriter, req *http.Request) {
 	transport := p.Transport
 
 	ctx := req.Context()
-	/* trunk-ignore(golangci-lint/revive) */
 	if ctx.Done() != nil {
 		// CloseNotifier predates context.Context, and has been
 		// entirely superseded by it. If the request contains
@@ -352,7 +351,7 @@ func (p *ReverseProxy) handler(rw http.ResponseWriter, req *http.Request) {
 			return nil
 		},
 	}
-	outreq = outreq.WithContext(httptrace.WithClientTrace(outreq.Context(), trace))
+	outreq = outreq.WithContext(httptrace.WithClientTrace(outreq.Context(), trace)) //nolint:contextcheck
 
 	res, err := transport.RoundTrip(outreq)
 
@@ -507,18 +506,18 @@ func (p *ReverseProxy) handleUpgradeResponse(rw http.ResponseWriter, req *http.R
 	res.Header = rw.Header()
 	res.Body = nil // so res.Write only writes the headers; we have res.Body in backConn above
 	if err := res.Write(brw); err != nil {
-		/* trunk-ignore(golangci-lint/errorlint) */
+		//nolint:errorlint
 		p.errorHandler(rw, req, fmt.Errorf("response write: %s", err), true)
 		return
 	}
 	if err := brw.Flush(); err != nil {
-		/* trunk-ignore(golangci-lint/errorlint) */
+		//nolint:errorlint
 		p.errorHandler(rw, req, fmt.Errorf("response flush: %s", err), true)
 		return
 	}
 
 	bdp := U.NewBidirectionalPipe(req.Context(), conn, backConn)
-	/* trunk-ignore(golangci-lint/errcheck) */
+	//nolint:errcheck
 	bdp.Start()
 }
 

internal/net/gphttp/server/server.go

@@ -9,14 +9,14 @@ import (
 
 	"github.com/quic-go/quic-go/http3"
 	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
 	"github.com/yusing/go-proxy/internal/acl"
 	"github.com/yusing/go-proxy/internal/common"
-	"github.com/yusing/go-proxy/internal/logging"
 	"github.com/yusing/go-proxy/internal/task"
 )
 
 type CertProvider interface {
-	GetCert(*tls.ClientHelloInfo) (*tls.Certificate, error)
+	GetCert(_ *tls.ClientHelloInfo) (*tls.Certificate, error)
 }
 
 type Server struct {
@@ -53,7 +53,7 @@ func StartServer(parent task.Parent, opt Options) (s *Server) {
 func NewServer(opt Options) (s *Server) {
 	var httpSer, httpsSer *http.Server
 
-	logger := logging.With().Str("server", opt.Name).Logger()
+	logger := log.With().Str("server", opt.Name).Logger()
 
 	certAvailable := false
 	if opt.CertProvider != nil {