fix: optimize memory usage, fix agent and code refactor (#118)

* refactor: simplify io code and make utils module independent

* fix(docker): agent and socket-proxy docker event flushing with modified reverse proxy handler

* refactor: remove unused code

* refactor: remove the use of logging module in most code

* refactor: streamline domain mismatch check in certState function

* tweak: use ecdsa p-256 for autocert

* fix(tests): update health check tests for invalid host and add case for port in host

* feat(acme): custom acme directory

* refactor: code refactor and improved context and error handling

* tweak: optimize memory usage under load

* fix(oidc): restore old user matching behavior

* docs: add ChatGPT assistant to README

---------

Co-authored-by: yusing <yusing@6uo.me>

internal/net/gphttp/middleware/captcha/middleware.go

@@ -4,8 +4,8 @@ import (
 	"net/http"
 	"text/template"
 
+	"github.com/rs/zerolog/log"
 	"github.com/yusing/go-proxy/internal/auth"
-	"github.com/yusing/go-proxy/internal/logging"
 	"github.com/yusing/go-proxy/internal/net/gphttp"
 
 	_ "embed"
@@ -55,7 +55,7 @@ func PreRequest(p Provider, w http.ResponseWriter, r *http.Request) (proceed boo
 		"FormHTML":   p.FormHTML(),
 	})
 	if err != nil {
-		logging.Error().Err(err).Msg("failed to execute captcha page")
+		log.Error().Err(err).Msg("failed to execute captcha page")
 	}
 	return false
 }

internal/net/gphttp/middleware/cidr_whitelist.go

@@ -7,7 +7,7 @@ import (
 	"github.com/go-playground/validator/v10"
 	gphttp "github.com/yusing/go-proxy/internal/net/gphttp"
 	"github.com/yusing/go-proxy/internal/net/types"
-	"github.com/yusing/go-proxy/internal/utils"
+	"github.com/yusing/go-proxy/internal/serialization"
 	F "github.com/yusing/go-proxy/internal/utils/functional"
 )
 
@@ -34,7 +34,7 @@ var (
 )
 
 func init() {
-	utils.MustRegisterValidation("status_code", func(fl validator.FieldLevel) bool {
+	serialization.MustRegisterValidation("status_code", func(fl validator.FieldLevel) bool {
 		statusCode := fl.Field().Int()
 		return gphttp.IsStatusCodeValid(int(statusCode))
 	})
@@ -60,7 +60,7 @@ func (wl *cidrWhitelist) checkIP(w http.ResponseWriter, r *http.Request) bool {
 			ipStr = r.RemoteAddr
 		}
 		ip := net.ParseIP(ipStr)
-		for _, cidr := range wl.CIDRWhitelistOpts.Allow {
+		for _, cidr := range wl.Allow {
 			if cidr.Contains(ip) {
 				wl.cachedAddr.Store(r.RemoteAddr, true)
 				allow = true
@@ -70,7 +70,7 @@ func (wl *cidrWhitelist) checkIP(w http.ResponseWriter, r *http.Request) bool {
 		}
 		if !allow {
 			wl.cachedAddr.Store(r.RemoteAddr, false)
-			wl.AddTracef("client %s is forbidden", ipStr).With("allowed CIDRs", wl.CIDRWhitelistOpts.Allow)
+			wl.AddTracef("client %s is forbidden", ipStr).With("allowed CIDRs", wl.Allow)
 		}
 	}
 	if !allow {

internal/net/gphttp/middleware/cidr_whitelist_test.go

@@ -8,7 +8,7 @@ import (
 	"testing"
 
 	"github.com/yusing/go-proxy/internal/gperr"
-	"github.com/yusing/go-proxy/internal/utils"
+	"github.com/yusing/go-proxy/internal/serialization"
 	. "github.com/yusing/go-proxy/internal/utils/testing"
 )
 
@@ -41,7 +41,7 @@ func TestCIDRWhitelistValidation(t *testing.T) {
 		_, err := CIDRWhiteList.New(OptionsRaw{
 			"message": testMessage,
 		})
-		ExpectError(t, utils.ErrValidationError, err)
+		ExpectError(t, serialization.ErrValidationError, err)
 	})
 	t.Run("invalid cidr", func(t *testing.T) {
 		_, err := CIDRWhiteList.New(OptionsRaw{
@@ -56,7 +56,7 @@ func TestCIDRWhitelistValidation(t *testing.T) {
 			"status_code": 600,
 			"message":     testMessage,
 		})
-		ExpectError(t, utils.ErrValidationError, err)
+		ExpectError(t, serialization.ErrValidationError, err)
 	})
 }
 

internal/net/gphttp/middleware/cloudflare_real_ip.go

@@ -1,6 +1,7 @@
 package middleware
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io"
@@ -9,8 +10,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/rs/zerolog/log"
 	"github.com/yusing/go-proxy/internal/common"
-	"github.com/yusing/go-proxy/internal/logging"
 	"github.com/yusing/go-proxy/internal/net/types"
 	"github.com/yusing/go-proxy/internal/utils/atomic"
 	"github.com/yusing/go-proxy/internal/utils/strutils"
@@ -89,21 +90,29 @@ func tryFetchCFCIDR() (cfCIDRs []*types.CIDR) {
 		)
 		if err != nil {
 			cfCIDRsLastUpdate.Store(time.Now().Add(-cfCIDRsUpdateRetryInterval - cfCIDRsUpdateInterval))
-			logging.Err(err).Msg("failed to update cloudflare range, retry in " + strutils.FormatDuration(cfCIDRsUpdateRetryInterval))
+			log.Err(err).Msg("failed to update cloudflare range, retry in " + strutils.FormatDuration(cfCIDRsUpdateRetryInterval))
 			return nil
 		}
 		if len(cfCIDRs) == 0 {
-			logging.Warn().Msg("cloudflare CIDR range is empty")
+			log.Warn().Msg("cloudflare CIDR range is empty")
 		}
 	}
 
 	cfCIDRsLastUpdate.Store(time.Now())
-	logging.Info().Msg("cloudflare CIDR range updated")
+	log.Info().Msg("cloudflare CIDR range updated")
 	return
 }
 
 func fetchUpdateCFIPRange(endpoint string, cfCIDRs *[]*types.CIDR) error {
-	resp, err := http.Get(endpoint)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
+	if err != nil {
+		return err
+	}
+
+	resp, err := http.DefaultClient.Do(req) //nolint:gosec
 	if err != nil {
 		return err
 	}

internal/net/gphttp/middleware/custom_error_page.go

@@ -8,7 +8,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/yusing/go-proxy/internal/logging"
+	"github.com/rs/zerolog/log"
 	gphttp "github.com/yusing/go-proxy/internal/net/gphttp"
 	"github.com/yusing/go-proxy/internal/net/gphttp/httpheaders"
 	"github.com/yusing/go-proxy/internal/net/gphttp/middleware/errorpage"
@@ -32,7 +32,7 @@ func (customErrorPage) modifyResponse(resp *http.Response) error {
 	if !gphttp.IsSuccess(resp.StatusCode) && (contentType.IsHTML() || contentType.IsPlainText()) {
 		errorPage, ok := errorpage.GetErrorPageByStatus(resp.StatusCode)
 		if ok {
-			logging.Debug().Msgf("error page for status %d loaded", resp.StatusCode)
+			log.Debug().Msgf("error page for status %d loaded", resp.StatusCode)
 			_, _ = io.Copy(io.Discard, resp.Body) // drain the original body
 			resp.Body.Close()
 			resp.Body = io.NopCloser(bytes.NewReader(errorPage))
@@ -40,7 +40,7 @@ func (customErrorPage) modifyResponse(resp *http.Response) error {
 			resp.Header.Set(httpheaders.HeaderContentLength, strconv.Itoa(len(errorPage)))
 			resp.Header.Set(httpheaders.HeaderContentType, "text/html; charset=utf-8")
 		} else {
-			logging.Error().Msgf("unable to load error page for status %d", resp.StatusCode)
+			log.Error().Msgf("unable to load error page for status %d", resp.StatusCode)
 		}
 		return nil
 	}
@@ -56,7 +56,7 @@ func ServeStaticErrorPageFile(w http.ResponseWriter, r *http.Request) (served bo
 		filename := path[len(StaticFilePathPrefix):]
 		file, ok := errorpage.GetStaticFile(filename)
 		if !ok {
-			logging.Error().Msg("unable to load resource " + filename)
+			log.Error().Msg("unable to load resource " + filename)
 			return false
 		}
 		ext := filepath.Ext(filename)
@@ -68,10 +68,10 @@ func ServeStaticErrorPageFile(w http.ResponseWriter, r *http.Request) (served bo
 		case ".css":
 			w.Header().Set(httpheaders.HeaderContentType, "text/css; charset=utf-8")
 		default:
-			logging.Error().Msgf("unexpected file type %q for %s", ext, filename)
+			log.Error().Msgf("unexpected file type %q for %s", ext, filename)
 		}
 		if _, err := w.Write(file); err != nil {
-			logging.Err(err).Msg("unable to write resource " + filename)
+			log.Err(err).Msg("unable to write resource " + filename)
 			http.Error(w, "Error page failure", http.StatusInternalServerError)
 		}
 		return true

internal/net/gphttp/middleware/errorpage/error_page.go

@@ -6,9 +6,9 @@ import (
 	"path"
 	"sync"
 
+	"github.com/rs/zerolog/log"
 	"github.com/yusing/go-proxy/internal/common"
 	"github.com/yusing/go-proxy/internal/gperr"
-	"github.com/yusing/go-proxy/internal/logging"
 	"github.com/yusing/go-proxy/internal/task"
 	U "github.com/yusing/go-proxy/internal/utils"
 	F "github.com/yusing/go-proxy/internal/utils/functional"
@@ -48,7 +48,7 @@ func GetErrorPageByStatus(statusCode int) (content []byte, ok bool) {
 func loadContent() {
 	files, err := U.ListFiles(errPagesBasePath, 0)
 	if err != nil {
-		logging.Err(err).Msg("failed to list error page resources")
+		log.Err(err).Msg("failed to list error page resources")
 		return
 	}
 	for _, file := range files {
@@ -57,11 +57,11 @@ func loadContent() {
 		}
 		content, err := os.ReadFile(file)
 		if err != nil {
-			logging.Warn().Err(err).Msgf("failed to read error page resource %s", file)
+			log.Warn().Err(err).Msgf("failed to read error page resource %s", file)
 			continue
 		}
 		file = path.Base(file)
-		logging.Info().Msgf("error page resource %s loaded", file)
+		log.Info().Msgf("error page resource %s loaded", file)
 		fileContentMap.Store(file, content)
 	}
 }
@@ -83,9 +83,9 @@ func watchDir() {
 				loadContent()
 			case events.ActionFileDeleted:
 				fileContentMap.Delete(filename)
-				logging.Warn().Msgf("error page resource %s deleted", filename)
+				log.Warn().Msgf("error page resource %s deleted", filename)
 			case events.ActionFileRenamed:
-				logging.Warn().Msgf("error page resource %s deleted", filename)
+				log.Warn().Msgf("error page resource %s deleted", filename)
 				fileContentMap.Delete(filename)
 				loadContent()
 			}

internal/net/gphttp/middleware/middleware.go

@@ -8,11 +8,11 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/rs/zerolog/log"
 	"github.com/yusing/go-proxy/internal/gperr"
-	"github.com/yusing/go-proxy/internal/logging"
 	gphttp "github.com/yusing/go-proxy/internal/net/gphttp"
 	"github.com/yusing/go-proxy/internal/net/gphttp/reverseproxy"
-	"github.com/yusing/go-proxy/internal/utils"
+	"github.com/yusing/go-proxy/internal/serialization"
 )
 
 type (
@@ -87,7 +87,7 @@ func NewMiddleware[ImplType any]() *Middleware {
 func (m *Middleware) enableTrace() {
 	if tracer, ok := m.impl.(MiddlewareWithTracer); ok {
 		tracer.enableTrace()
-		logging.Trace().Msgf("middleware %s enabled trace", m.name)
+		log.Trace().Msgf("middleware %s enabled trace", m.name)
 	}
 }
 
@@ -118,14 +118,14 @@ func (m *Middleware) apply(optsRaw OptionsRaw) gperr.Error {
 		"priority": optsRaw["priority"],
 		"bypass":   optsRaw["bypass"],
 	}
-	if err := utils.MapUnmarshalValidate(commonOpts, &m.commonOptions); err != nil {
+	if err := serialization.MapUnmarshalValidate(commonOpts, &m.commonOptions); err != nil {
 		return err
 	}
 	optsRaw = maps.Clone(optsRaw)
 	for k := range commonOpts {
 		delete(optsRaw, k)
 	}
-	return utils.MapUnmarshalValidate(optsRaw, m.impl)
+	return serialization.MapUnmarshalValidate(optsRaw, m.impl)
 }
 
 func (m *Middleware) finalize() error {

internal/net/gphttp/middleware/middlewares.go

@@ -3,9 +3,9 @@ package middleware
 import (
 	"path"
 
+	"github.com/rs/zerolog/log"
 	"github.com/yusing/go-proxy/internal/common"
 	"github.com/yusing/go-proxy/internal/gperr"
-	"github.com/yusing/go-proxy/internal/logging"
 	"github.com/yusing/go-proxy/internal/utils"
 	"github.com/yusing/go-proxy/internal/utils/strutils"
 )
@@ -59,7 +59,7 @@ func LoadComposeFiles() {
 	errs := gperr.NewBuilder("middleware compile errors")
 	middlewareDefs, err := utils.ListFiles(common.MiddlewareComposeBasePath, 0)
 	if err != nil {
-		logging.Err(err).Msg("failed to list middleware definitions")
+		log.Err(err).Msg("failed to list middleware definitions")
 		return
 	}
 	for _, defFile := range middlewareDefs {
@@ -75,7 +75,7 @@ func LoadComposeFiles() {
 				continue
 			}
 			allMiddlewares[name] = m
-			logging.Info().
+			log.Info().
 				Str("src", path.Base(defFile)).
 				Str("name", name).
 				Msg("middleware loaded")
@@ -94,7 +94,7 @@ func LoadComposeFiles() {
 				continue
 			}
 			allMiddlewares[name] = m
-			logging.Info().
+			log.Info().
 				Str("src", path.Base(defFile)).
 				Str("name", name).
 				Msg("middleware loaded")