feat: proxmox idlewatcher (#88)

* feat: idle sleep for proxmox LXCs

* refactor: replace deprecated docker api types

* chore(api): remove debug task list endpoint

* refactor: move servemux to gphttp/servemux; favicon.go to v1/favicon

* refactor: introduce Pool interface, move agent_pool to agent module

* refactor: simplify api code

* feat: introduce debug api

* refactor: remove net.URL and net.CIDR types, improved unmarshal handling

* chore: update Makefile for debug build tag, update README

* chore: add gperr.Unwrap method

* feat: relative time and duration formatting

* chore: add ROOT_DIR environment variable, refactor

* migration: move homepage override and icon cache to $BASE_DIR/data, add migration code

* fix: nil dereference on marshalling service health

* fix: wait for route deletion

* chore: enhance tasks debuggability

* feat: stdout access logger and MultiWriter

* fix(agent): remove agent properly on verify error

* fix(metrics): disk exclusion logic and added corresponding tests

* chore: update schema and prettify, fix package.json and Makefile

* fix: I/O buffer not being shrunk before putting back to pool

* feat: enhanced error handling module

* chore: deps upgrade

* feat: better value formatting and handling

---------

Co-authored-by: yusing <yusing@6uo.me>

internal/net/gphttp/middleware/cidr_whitelist.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/go-playground/validator/v10"
 	gphttp "github.com/yusing/go-proxy/internal/net/gphttp"
-	"github.com/yusing/go-proxy/internal/net/types"
 	"github.com/yusing/go-proxy/internal/utils"
 	F "github.com/yusing/go-proxy/internal/utils/functional"
 )
@@ -18,8 +17,8 @@ type (
 		cachedAddr F.Map[string, bool] // cache for trusted IPs
 	}
 	CIDRWhitelistOpts struct {
-		Allow      []*types.CIDR `validate:"min=1"`
-		StatusCode int           `json:"status_code" aliases:"status" validate:"omitempty,status_code"`
+		Allow      []*net.IPNet `validate:"min=1"`
+		StatusCode int          `json:"status_code" aliases:"status" validate:"omitempty,status_code"`
 		Message    string
 	}
 )
@@ -27,7 +26,7 @@ type (
 var (
 	CIDRWhiteList         = NewMiddleware[cidrWhitelist]()
 	cidrWhitelistDefaults = CIDRWhitelistOpts{
-		Allow:      []*types.CIDR{},
+		Allow:      []*net.IPNet{},
 		StatusCode: http.StatusForbidden,
 		Message:    "IP not allowed",
 	}

internal/net/gphttp/middleware/cloudflare_real_ip.go

@@ -11,7 +11,6 @@ import (
 
 	"github.com/yusing/go-proxy/internal/common"
 	"github.com/yusing/go-proxy/internal/logging"
-	"github.com/yusing/go-proxy/internal/net/types"
 	"github.com/yusing/go-proxy/internal/utils/atomic"
 	"github.com/yusing/go-proxy/internal/utils/strutils"
 )
@@ -33,7 +32,7 @@ var (
 	cfCIDRsMu         sync.Mutex
 
 	// RFC 1918.
-	localCIDRs = []*types.CIDR{
+	localCIDRs = []*net.IPNet{
 		{IP: net.IPv4(127, 0, 0, 1), Mask: net.IPv4Mask(255, 255, 255, 255)}, // 127.0.0.1/32
 		{IP: net.IPv4(10, 0, 0, 0), Mask: net.IPv4Mask(255, 0, 0, 0)},        // 10.0.0.0/8
 		{IP: net.IPv4(172, 16, 0, 0), Mask: net.IPv4Mask(255, 240, 0, 0)},    // 172.16.0.0/12
@@ -68,7 +67,7 @@ func (cri *cloudflareRealIP) getTracer() *Tracer {
 	return cri.realIP.getTracer()
 }
 
-func tryFetchCFCIDR() (cfCIDRs []*types.CIDR) {
+func tryFetchCFCIDR() (cfCIDRs []*net.IPNet) {
 	if time.Since(cfCIDRsLastUpdate.Load()) < cfCIDRsUpdateInterval {
 		return
 	}
@@ -83,7 +82,7 @@ func tryFetchCFCIDR() (cfCIDRs []*types.CIDR) {
 	if common.IsTest {
 		cfCIDRs = localCIDRs
 	} else {
-		cfCIDRs = make([]*types.CIDR, 0, 30)
+		cfCIDRs = make([]*net.IPNet, 0, 30)
 		err := errors.Join(
 			fetchUpdateCFIPRange(cfIPv4CIDRsEndpoint, &cfCIDRs),
 			fetchUpdateCFIPRange(cfIPv6CIDRsEndpoint, &cfCIDRs),
@@ -103,7 +102,7 @@ func tryFetchCFCIDR() (cfCIDRs []*types.CIDR) {
 	return
 }
 
-func fetchUpdateCFIPRange(endpoint string, cfCIDRs *[]*types.CIDR) error {
+func fetchUpdateCFIPRange(endpoint string, cfCIDRs *[]*net.IPNet) error {
 	resp, err := http.Get(endpoint)
 	if err != nil {
 		return err
@@ -124,7 +123,7 @@ func fetchUpdateCFIPRange(endpoint string, cfCIDRs *[]*types.CIDR) error {
 			return fmt.Errorf("cloudflare responeded an invalid CIDR: %s", line)
 		}
 
-		*cfCIDRs = append(*cfCIDRs, (*types.CIDR)(cidr))
+		*cfCIDRs = append(*cfCIDRs, (*net.IPNet)(cidr))
 	}
 	*cfCIDRs = append(*cfCIDRs, localCIDRs...)
 	return nil

internal/net/gphttp/middleware/errorpage/error_page.go

@@ -16,8 +16,6 @@ import (
 	"github.com/yusing/go-proxy/internal/watcher/events"
 )
 
-const errPagesBasePath = common.ErrorPagesBasePath
-
 var (
 	setupOnce      sync.Once
 	dirWatcher     W.Watcher
@@ -26,7 +24,7 @@ var (
 
 func setup() {
 	t := task.RootTask("error_page", false)
-	dirWatcher = W.NewDirectoryWatcher(t, errPagesBasePath)
+	dirWatcher = W.NewDirectoryWatcher(t, common.ErrorPagesDir)
 	loadContent()
 	go watchDir()
 }
@@ -46,7 +44,7 @@ func GetErrorPageByStatus(statusCode int) (content []byte, ok bool) {
 }
 
 func loadContent() {
-	files, err := U.ListFiles(errPagesBasePath, 0)
+	files, err := U.ListFiles(common.ErrorPagesDir, 0)
 	if err != nil {
 		logging.Err(err).Msg("failed to list error page resources")
 		return

internal/net/gphttp/middleware/middlewares.go

@@ -55,7 +55,7 @@ func All() map[string]*Middleware {
 
 func LoadComposeFiles() {
 	errs := gperr.NewBuilder("middleware compile errors")
-	middlewareDefs, err := utils.ListFiles(common.MiddlewareComposeBasePath, 0)
+	middlewareDefs, err := utils.ListFiles(common.MiddlewareComposeDir, 0)
 	if err != nil {
 		logging.Err(err).Msg("failed to list middleware definitions")
 		return

internal/net/gphttp/middleware/modify_request_test.go

@@ -4,10 +4,10 @@ import (
 	"bytes"
 	"net"
 	"net/http"
+	"net/url"
 	"slices"
 	"testing"
 
-	"github.com/yusing/go-proxy/internal/net/types"
 	. "github.com/yusing/go-proxy/internal/utils/testing"
 )
 
@@ -51,8 +51,8 @@ func TestModifyRequest(t *testing.T) {
 	})
 
 	t.Run("request_headers", func(t *testing.T) {
-		reqURL := types.MustParseURL("https://my.app/?arg_1=b")
-		upstreamURL := types.MustParseURL("http://test.example.com")
+		reqURL := Must(url.Parse("https://my.app/?arg_1=b"))
+		upstreamURL := Must(url.Parse("http://test.example.com"))
 		result, err := newMiddlewareTest(ModifyRequest, &testArgs{
 			middlewareOpt: opts,
 			reqURL:        reqURL,
@@ -128,8 +128,8 @@ func TestModifyRequest(t *testing.T) {
 
 		for _, tt := range tests {
 			t.Run(tt.name, func(t *testing.T) {
-				reqURL := types.MustParseURL("https://my.app" + tt.path)
-				upstreamURL := types.MustParseURL(tt.upstreamURL)
+				reqURL := Must(url.Parse("https://my.app" + tt.path))
+				upstreamURL := Must(url.Parse(tt.upstreamURL))
 
 				opts["add_prefix"] = tt.addPrefix
 				result, err := newMiddlewareTest(ModifyRequest, &testArgs{

internal/net/gphttp/middleware/modify_response_test.go

@@ -4,10 +4,10 @@ import (
 	"bytes"
 	"net"
 	"net/http"
+	"net/url"
 	"slices"
 	"testing"
 
-	"github.com/yusing/go-proxy/internal/net/types"
 	. "github.com/yusing/go-proxy/internal/utils/testing"
 )
 
@@ -54,8 +54,8 @@ func TestModifyResponse(t *testing.T) {
 	})
 
 	t.Run("response_headers", func(t *testing.T) {
-		reqURL := types.MustParseURL("https://my.app/?arg_1=b")
-		upstreamURL := types.MustParseURL("http://test.example.com")
+		reqURL := Must(url.Parse("https://my.app/?arg_1=b"))
+		upstreamURL := Must(url.Parse("http://test.example.com"))
 		result, err := newMiddlewareTest(ModifyResponse, &testArgs{
 			middlewareOpt: opts,
 			reqURL:        reqURL,

internal/net/gphttp/middleware/real_ip.go

@@ -5,7 +5,6 @@ import (
 	"net/http"
 
 	"github.com/yusing/go-proxy/internal/net/gphttp/httpheaders"
-	"github.com/yusing/go-proxy/internal/net/types"
 )
 
 // https://nginx.org/en/docs/http/ngx_http_realip_module.html
@@ -19,7 +18,7 @@ type (
 		// Header is the name of the header to use for the real client IP
 		Header string `validate:"required"`
 		// From is a list of Address / CIDRs to trust
-		From []*types.CIDR `validate:"required,min=1"`
+		From []*net.IPNet `validate:"required,min=1"`
 		/*
 			If recursive search is disabled,
 			the original client address that matches one of the trusted addresses is replaced by
@@ -36,7 +35,7 @@ var (
 	RealIP            = NewMiddleware[realIP]()
 	realIPOptsDefault = RealIPOpts{
 		Header: "X-Real-IP",
-		From:   []*types.CIDR{},
+		From:   []*net.IPNet{},
 	}
 )
 

internal/net/gphttp/middleware/real_ip_test.go

@@ -7,7 +7,6 @@ import (
 	"testing"
 
 	"github.com/yusing/go-proxy/internal/net/gphttp/httpheaders"
-	"github.com/yusing/go-proxy/internal/net/types"
 	. "github.com/yusing/go-proxy/internal/utils/testing"
 )
 
@@ -23,7 +22,7 @@ func TestSetRealIPOpts(t *testing.T) {
 	}
 	optExpected := &RealIPOpts{
 		Header: httpheaders.HeaderXRealIP,
-		From: []*types.CIDR{
+		From: []*net.IPNet{
 			{
 				IP:   net.ParseIP("127.0.0.0"),
 				Mask: net.IPv4Mask(255, 0, 0, 0),

internal/net/gphttp/middleware/redirect_http_test.go

@@ -2,15 +2,15 @@ package middleware
 
 import (
 	"net/http"
+	"net/url"
 	"testing"
 
-	"github.com/yusing/go-proxy/internal/net/types"
 	. "github.com/yusing/go-proxy/internal/utils/testing"
 )
 
 func TestRedirectToHTTPs(t *testing.T) {
 	result, err := newMiddlewareTest(RedirectHTTP, &testArgs{
-		reqURL: types.MustParseURL("http://example.com"),
+		reqURL: Must(url.Parse("http://example.com")),
 	})
 	ExpectNoError(t, err)
 	ExpectEqual(t, result.ResponseStatus, http.StatusPermanentRedirect)
@@ -19,7 +19,7 @@ func TestRedirectToHTTPs(t *testing.T) {
 
 func TestNoRedirect(t *testing.T) {
 	result, err := newMiddlewareTest(RedirectHTTP, &testArgs{
-		reqURL: types.MustParseURL("https://example.com"),
+		reqURL: Must(url.Parse("https://example.com")),
 	})
 	ExpectNoError(t, err)
 	ExpectEqual(t, result.ResponseStatus, http.StatusOK)

internal/net/gphttp/middleware/test_utils.go

@@ -7,11 +7,11 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 
 	"github.com/yusing/go-proxy/internal/common"
 	"github.com/yusing/go-proxy/internal/gperr"
 	"github.com/yusing/go-proxy/internal/net/gphttp/reverseproxy"
-	"github.com/yusing/go-proxy/internal/net/types"
 	. "github.com/yusing/go-proxy/internal/utils/testing"
 )
 
@@ -80,11 +80,11 @@ type TestResult struct {
 
 type testArgs struct {
 	middlewareOpt OptionsRaw
-	upstreamURL   *types.URL
+	upstreamURL   *url.URL
 
 	realRoundTrip bool
 
-	reqURL    *types.URL
+	reqURL    *url.URL
 	reqMethod string
 	headers   http.Header
 	body      []byte
@@ -96,13 +96,13 @@ type testArgs struct {
 
 func (args *testArgs) setDefaults() {
 	if args.reqURL == nil {
-		args.reqURL = Must(types.ParseURL("https://example.com"))
+		args.reqURL = Must(url.Parse("https://example.com"))
 	}
 	if args.reqMethod == "" {
 		args.reqMethod = http.MethodGet
 	}
 	if args.upstreamURL == nil {
-		args.upstreamURL = Must(types.ParseURL("https://10.0.0.1:8443")) // dummy url, no actual effect
+		args.upstreamURL = Must(url.Parse("https://10.0.0.1:8443")) // dummy url, no actual effect
 	}
 	if args.respHeaders == nil {
 		args.respHeaders = http.Header{}