feat: proxmox idlewatcher (#88)

* feat: idle sleep for proxmox LXCs

* refactor: replace deprecated docker api types

* chore(api): remove debug task list endpoint

* refactor: move servemux to gphttp/servemux; favicon.go to v1/favicon

* refactor: introduce Pool interface, move agent_pool to agent module

* refactor: simplify api code

* feat: introduce debug api

* refactor: remove net.URL and net.CIDR types, improved unmarshal handling

* chore: update Makefile for debug build tag, update README

* chore: add gperr.Unwrap method

* feat: relative time and duration formatting

* chore: add ROOT_DIR environment variable, refactor

* migration: move homepage override and icon cache to $BASE_DIR/data, add migration code

* fix: nil dereference on marshalling service health

* fix: wait for route deletion

* chore: enhance tasks debuggability

* feat: stdout access logger and MultiWriter

* fix(agent): remove agent properly on verify error

* fix(metrics): disk exclusion logic and added corresponding tests

* chore: update schema and prettify, fix package.json and Makefile

* fix: I/O buffer not being shrunk before putting back to pool

* feat: enhanced error handling module

* chore: deps upgrade

* feat: better value formatting and handling

---------

Co-authored-by: yusing <yusing@6uo.me>

internal/api/handler.go

@@ -1,7 +1,6 @@
 package api
 
 import (
-	"fmt"
 	"net/http"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -9,63 +8,17 @@ import (
 	"github.com/yusing/go-proxy/internal/api/v1/auth"
 	"github.com/yusing/go-proxy/internal/api/v1/certapi"
 	"github.com/yusing/go-proxy/internal/api/v1/dockerapi"
+	"github.com/yusing/go-proxy/internal/api/v1/favicon"
 	"github.com/yusing/go-proxy/internal/common"
 	config "github.com/yusing/go-proxy/internal/config/types"
 	"github.com/yusing/go-proxy/internal/logging"
 	"github.com/yusing/go-proxy/internal/logging/memlogger"
 	"github.com/yusing/go-proxy/internal/metrics/uptime"
-	"github.com/yusing/go-proxy/internal/net/gphttp/httpheaders"
-	"github.com/yusing/go-proxy/internal/utils/strutils"
+	"github.com/yusing/go-proxy/internal/net/gphttp/servemux"
 )
 
-type (
-	ServeMux struct {
-		*http.ServeMux
-		cfg config.ConfigInstance
-	}
-	WithCfgHandler = func(config.ConfigInstance, http.ResponseWriter, *http.Request)
-)
-
-func (mux ServeMux) HandleFunc(methods, endpoint string, h any, requireAuth ...bool) {
-	var handler http.HandlerFunc
-	switch h := h.(type) {
-	case func(http.ResponseWriter, *http.Request):
-		handler = h
-	case http.Handler:
-		handler = h.ServeHTTP
-	case WithCfgHandler:
-		handler = func(w http.ResponseWriter, r *http.Request) {
-			h(mux.cfg, w, r)
-		}
-	default:
-		panic(fmt.Errorf("unsupported handler type: %T", h))
-	}
-
-	matchDomains := mux.cfg.Value().MatchDomains
-	if len(matchDomains) > 0 {
-		origHandler := handler
-		handler = func(w http.ResponseWriter, r *http.Request) {
-			if httpheaders.IsWebsocket(r.Header) {
-				httpheaders.SetWebsocketAllowedDomains(r.Header, matchDomains)
-			}
-			origHandler(w, r)
-		}
-	}
-
-	if len(requireAuth) > 0 && requireAuth[0] {
-		handler = auth.RequireAuth(handler)
-	}
-	if methods == "" {
-		mux.ServeMux.HandleFunc(endpoint, handler)
-	} else {
-		for _, m := range strutils.CommaSeperatedList(methods) {
-			mux.ServeMux.HandleFunc(m+" "+endpoint, handler)
-		}
-	}
-}
-
 func NewHandler(cfg config.ConfigInstance) http.Handler {
-	mux := ServeMux{http.NewServeMux(), cfg}
+	mux := servemux.NewServeMux(cfg)
 	mux.HandleFunc("GET", "/v1", v1.Index)
 	mux.HandleFunc("GET", "/v1/version", v1.GetVersion)
 
@@ -79,7 +32,7 @@ func NewHandler(cfg config.ConfigInstance) http.Handler {
 	mux.HandleFunc("POST", "/v1/file/validate/{type}", v1.ValidateFile, true)
 	mux.HandleFunc("GET", "/v1/health", v1.Health, true)
 	mux.HandleFunc("GET", "/v1/logs", memlogger.Handler(), true)
-	mux.HandleFunc("GET", "/v1/favicon", v1.GetFavIcon, true)
+	mux.HandleFunc("GET", "/v1/favicon", favicon.GetFavIcon, true)
 	mux.HandleFunc("POST", "/v1/homepage/set", v1.SetHomePageOverrides, true)
 	mux.HandleFunc("GET", "/v1/agents", v1.ListAgents, true)
 	mux.HandleFunc("GET", "/v1/agents/new", v1.NewAgent, true)

internal/api/v1/agents.go

@@ -4,21 +4,11 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/coder/websocket"
-	"github.com/coder/websocket/wsjson"
+	"github.com/yusing/go-proxy/agent/pkg/agent"
 	config "github.com/yusing/go-proxy/internal/config/types"
-	"github.com/yusing/go-proxy/internal/net/gphttp"
 	"github.com/yusing/go-proxy/internal/net/gphttp/gpwebsocket"
-	"github.com/yusing/go-proxy/internal/net/gphttp/httpheaders"
 )
 
 func ListAgents(cfg config.ConfigInstance, w http.ResponseWriter, r *http.Request) {
-	if httpheaders.IsWebsocket(r.Header) {
-		gpwebsocket.Periodic(w, r, 10*time.Second, func(conn *websocket.Conn) error {
-			wsjson.Write(r.Context(), conn, cfg.ListAgents())
-			return nil
-		})
-	} else {
-		gphttp.RespondJSON(w, r, cfg.ListAgents())
-	}
+	gpwebsocket.DynamicJSONHandler(w, r, agent.Agents.Slice, 10*time.Second)
 }

internal/api/v1/config_file.go

@@ -27,7 +27,7 @@ func fileType(file string) FileType {
 	switch {
 	case strings.HasPrefix(path.Base(file), "config."):
 		return FileTypeConfig
-	case strings.HasPrefix(file, common.MiddlewareComposeBasePath):
+	case strings.HasPrefix(file, common.MiddlewareComposeDir):
 		return FileTypeMiddleware
 	}
 	return FileTypeProvider
@@ -43,9 +43,9 @@ func (t FileType) IsValid() bool {
 
 func (t FileType) GetPath(filename string) string {
 	if t == FileTypeMiddleware {
-		return path.Join(common.MiddlewareComposeBasePath, filename)
+		return path.Join(common.MiddlewareComposeDir, filename)
 	}
-	return path.Join(common.ConfigBasePath, filename)
+	return path.Join(common.ConfigDir, filename)
 }
 
 func getArgs(r *http.Request) (fileType FileType, filename string, err error) {

internal/api/v1/debug/handler.go

@@ -0,0 +1,75 @@
+//go:build debug
+
+package debugapi
+
+import (
+	"iter"
+	"net/http"
+	"sort"
+	"time"
+
+	"github.com/yusing/go-proxy/agent/pkg/agent"
+	config "github.com/yusing/go-proxy/internal/config/types"
+	"github.com/yusing/go-proxy/internal/docker"
+	"github.com/yusing/go-proxy/internal/idlewatcher"
+	"github.com/yusing/go-proxy/internal/net/gphttp/gpwebsocket"
+	"github.com/yusing/go-proxy/internal/net/gphttp/servemux"
+	"github.com/yusing/go-proxy/internal/net/gphttp/server"
+	"github.com/yusing/go-proxy/internal/proxmox"
+	"github.com/yusing/go-proxy/internal/task"
+)
+
+func StartServer(cfg config.ConfigInstance) {
+	srv := server.NewServer(server.Options{
+		Name:     "debug",
+		HTTPAddr: "127.0.0.1:7777",
+		Handler:  newHandler(cfg),
+	})
+	srv.Start(task.RootTask("debug_server", false))
+}
+
+type debuggable interface {
+	MarshalMap() map[string]any
+	Key() string
+}
+
+func toSortedSlice[T debuggable](data iter.Seq2[string, T]) []map[string]any {
+	s := make([]map[string]any, 0)
+	for _, v := range data {
+		m := v.MarshalMap()
+		m["key"] = v.Key()
+		s = append(s, m)
+	}
+	sort.Slice(s, func(i, j int) bool {
+		return s[i]["key"].(string) < s[j]["key"].(string)
+	})
+	return s
+}
+
+func jsonHandler[T debuggable](getData iter.Seq2[string, T]) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		gpwebsocket.DynamicJSONHandler(w, r, func() []map[string]any {
+			return toSortedSlice(getData)
+		}, 1*time.Second)
+	}
+}
+
+func iterMap[K comparable, V debuggable](m func() map[K]V) iter.Seq2[K, V] {
+	return func(yield func(K, V) bool) {
+		for k, v := range m() {
+			if !yield(k, v) {
+				break
+			}
+		}
+	}
+}
+
+func newHandler(cfg config.ConfigInstance) http.Handler {
+	mux := servemux.NewServeMux(cfg)
+	mux.HandleFunc("GET", "/tasks", jsonHandler(task.AllTasks()))
+	mux.HandleFunc("GET", "/idlewatcher", jsonHandler(idlewatcher.Watchers()))
+	mux.HandleFunc("GET", "/agents", jsonHandler(agent.Agents.Iter))
+	mux.HandleFunc("GET", "/proxmox", jsonHandler(proxmox.Clients.Iter))
+	mux.HandleFunc("GET", "/docker", jsonHandler(iterMap(docker.Clients)))
+	return mux
+}

internal/api/v1/debug/handler_production.go

@@ -0,0 +1,11 @@
+//go:build !debug
+
+package debugapi
+
+import (
+	config "github.com/yusing/go-proxy/internal/config/types"
+)
+
+func StartServer(cfg config.ConfigInstance) {
+	// do nothing
+}

internal/api/v1/dockerapi/containers.go

@@ -18,7 +18,7 @@ type Container struct {
 }
 
 func Containers(w http.ResponseWriter, r *http.Request) {
-	serveHTTP[Container, []Container](w, r, GetContainers)
+	serveHTTP[Container](w, r, GetContainers)
 }
 
 func GetContainers(ctx context.Context, dockerClients DockerClients) ([]Container, gperr.Error) {

internal/api/v1/dockerapi/info.go

@@ -25,7 +25,7 @@ func (d *dockerInfo) MarshalJSON() ([]byte, error) {
 		},
 		"images": d.Images,
 		"n_cpu":  d.NCPU,
-		"memory": strutils.FormatByteSizeWithUnit(d.MemTotal),
+		"memory": strutils.FormatByteSize(d.MemTotal),
 	})
 }
 

internal/api/v1/dockerapi/logs.go

@@ -22,7 +22,7 @@ func Logs(w http.ResponseWriter, r *http.Request) {
 	until := query.Get("to")
 	levels := query.Get("levels") // TODO: implement levels
 
-	dockerClient, found, err := getDockerClient(w, server)
+	dockerClient, found, err := getDockerClient(server)
 	if err != nil {
 		gphttp.BadRequest(w, err.Error())
 		return

internal/api/v1/dockerapi/utils.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/coder/websocket"
 	"github.com/coder/websocket/wsjson"
+	"github.com/yusing/go-proxy/agent/pkg/agent"
 	config "github.com/yusing/go-proxy/internal/config/types"
 	"github.com/yusing/go-proxy/internal/docker"
 	"github.com/yusing/go-proxy/internal/gperr"
@@ -44,7 +45,7 @@ func getDockerClients() (DockerClients, gperr.Error) {
 		dockerClients[name] = dockerClient
 	}
 
-	for _, agent := range cfg.ListAgents() {
+	for _, agent := range agent.Agents.Iter {
 		dockerClient, err := docker.NewClient(agent.FakeDockerHost())
 		if err != nil {
 			connErrs.Add(err)
@@ -56,7 +57,7 @@ func getDockerClients() (DockerClients, gperr.Error) {
 	return dockerClients, connErrs.Error()
 }
 
-func getDockerClient(w http.ResponseWriter, server string) (*docker.SharedClient, bool, error) {
+func getDockerClient(server string) (*docker.SharedClient, bool, error) {
 	cfg := config.GetInstance()
 	var host string
 	for name, h := range cfg.Value().Providers.Docker {
@@ -65,7 +66,7 @@ func getDockerClient(w http.ResponseWriter, server string) (*docker.SharedClient
 			break
 		}
 	}
-	for _, agent := range cfg.ListAgents() {
+	for _, agent := range agent.Agents.Iter {
 		if agent.Name() == server {
 			host = agent.FakeDockerHost()
 			break
@@ -119,6 +120,6 @@ func serveHTTP[V any, T ResultType[V]](w http.ResponseWriter, r *http.Request, g
 		})
 	} else {
 		result, err := getResult(r.Context(), dockerClients)
-		handleResult[V, T](w, err, result)
+		handleResult[V](w, err, result)
 	}
 }

internal/api/v1/favicon/favicon.go

@@ -1,4 +1,4 @@
-package v1
+package favicon
 
 import (
 	"errors"

internal/api/v1/health.go

@@ -4,20 +4,10 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/coder/websocket"
-	"github.com/coder/websocket/wsjson"
-	"github.com/yusing/go-proxy/internal/net/gphttp"
 	"github.com/yusing/go-proxy/internal/net/gphttp/gpwebsocket"
-	"github.com/yusing/go-proxy/internal/net/gphttp/httpheaders"
 	"github.com/yusing/go-proxy/internal/route/routes/routequery"
 )
 
 func Health(w http.ResponseWriter, r *http.Request) {
-	if httpheaders.IsWebsocket(r.Header) {
-		gpwebsocket.Periodic(w, r, 1*time.Second, func(conn *websocket.Conn) error {
-			return wsjson.Write(r.Context(), conn, routequery.HealthMap())
-		})
-	} else {
-		gphttp.RespondJSON(w, r, routequery.HealthMap())
-	}
+	gpwebsocket.DynamicJSONHandler(w, r, routequery.HealthMap, 1*time.Second)
 }

internal/api/v1/list.go

@@ -13,7 +13,6 @@ import (
 	"github.com/yusing/go-proxy/internal/net/gphttp/middleware"
 	"github.com/yusing/go-proxy/internal/route/routes/routequery"
 	route "github.com/yusing/go-proxy/internal/route/types"
-	"github.com/yusing/go-proxy/internal/task"
 	"github.com/yusing/go-proxy/internal/utils"
 )
 
@@ -28,7 +27,6 @@ const (
 	ListRouteProviders     = "route_providers"
 	ListHomepageCategories = "homepage_categories"
 	ListIcons              = "icons"
-	ListTasks              = "tasks"
 )
 
 func List(cfg config.ConfigInstance, w http.ResponseWriter, r *http.Request) {
@@ -76,8 +74,6 @@ func List(cfg config.ConfigInstance, w http.ResponseWriter, r *http.Request) {
 			icons = []string{}
 		}
 		gphttp.RespondJSON(w, r, icons)
-	case ListTasks:
-		gphttp.RespondJSON(w, r, task.DebugTaskList())
 	default:
 		gphttp.BadRequest(w, fmt.Sprintf("invalid what: %s", what))
 	}
@@ -98,7 +94,7 @@ func listRoute(which string) any {
 }
 
 func listFiles(w http.ResponseWriter, r *http.Request) {
-	files, err := utils.ListFiles(common.ConfigBasePath, 0, true)
+	files, err := utils.ListFiles(common.ConfigDir, 0, true)
 	if err != nil {
 		gphttp.ServerError(w, r, err)
 		return
@@ -111,17 +107,17 @@ func listFiles(w http.ResponseWriter, r *http.Request) {
 
 	for _, file := range files {
 		t := fileType(file)
-		file = strings.TrimPrefix(file, common.ConfigBasePath+"/")
+		file = strings.TrimPrefix(file, common.ConfigDir+"/")
 		resp[t] = append(resp[t], file)
 	}
 
-	mids, err := utils.ListFiles(common.MiddlewareComposeBasePath, 0, true)
+	mids, err := utils.ListFiles(common.MiddlewareComposeDir, 0, true)
 	if err != nil {
 		gphttp.ServerError(w, r, err)
 		return
 	}
 	for _, mid := range mids {
-		mid = strings.TrimPrefix(mid, common.MiddlewareComposeBasePath+"/")
+		mid = strings.TrimPrefix(mid, common.MiddlewareComposeDir+"/")
 		resp[FileTypeMiddleware] = append(resp[FileTypeMiddleware], mid)
 	}
 	gphttp.RespondJSON(w, r, resp)

internal/api/v1/new_agent.go

@@ -40,7 +40,7 @@ func NewAgent(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	hostport := fmt.Sprintf("%s:%d", host, port)
-	if _, ok := config.GetInstance().GetAgent(hostport); ok {
+	if _, ok := agent.Agents.Get(hostport); ok {
 		gphttp.ClientError(w, gphttp.ErrAlreadyExists("agent", hostport), http.StatusConflict)
 		return
 	}

internal/api/v1/query/query.go

@@ -58,7 +58,3 @@ func ListRoutes() (map[string]map[string]any, gperr.Error) {
 func ListMiddlewareTraces() (middleware.Traces, gperr.Error) {
 	return List[middleware.Traces](v1.ListMiddlewareTraces)
 }
-
-func DebugListTasks() (map[string]any, gperr.Error) {
-	return List[map[string]any](v1.ListTasks)
-}

internal/api/v1/stats.go

@@ -4,30 +4,18 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/coder/websocket"
-	"github.com/coder/websocket/wsjson"
 	config "github.com/yusing/go-proxy/internal/config/types"
-	"github.com/yusing/go-proxy/internal/net/gphttp"
 	"github.com/yusing/go-proxy/internal/net/gphttp/gpwebsocket"
-	"github.com/yusing/go-proxy/internal/net/gphttp/httpheaders"
 	"github.com/yusing/go-proxy/internal/utils/strutils"
 )
 
 func Stats(cfg config.ConfigInstance, w http.ResponseWriter, r *http.Request) {
-	if httpheaders.IsWebsocket(r.Header) {
-		gpwebsocket.Periodic(w, r, 1*time.Second, func(conn *websocket.Conn) error {
-			return wsjson.Write(r.Context(), conn, getStats(cfg))
-		})
-	} else {
-		gphttp.RespondJSON(w, r, getStats(cfg))
-	}
+	gpwebsocket.DynamicJSONHandler(w, r, func() map[string]any {
+		return map[string]any{
+			"proxies": cfg.Statistics(),
+			"uptime":  strutils.FormatDuration(time.Since(startTime)),
+		}
+	}, 1*time.Second)
 }
 
 var startTime = time.Now()
-
-func getStats(cfg config.ConfigInstance) map[string]any {
-	return map[string]any{
-		"proxies": cfg.Statistics(),
-		"uptime":  strutils.FormatDuration(time.Since(startTime)),
-	}
-}

internal/api/v1/system_info.go

@@ -3,8 +3,8 @@ package v1
 import (
 	"net/http"
 
+	"github.com/yusing/go-proxy/agent/pkg/agent"
 	agentPkg "github.com/yusing/go-proxy/agent/pkg/agent"
-	config "github.com/yusing/go-proxy/internal/config/types"
 	"github.com/yusing/go-proxy/internal/gperr"
 	"github.com/yusing/go-proxy/internal/metrics/systeminfo"
 	"github.com/yusing/go-proxy/internal/net/gphttp"
@@ -12,7 +12,7 @@ import (
 	"github.com/yusing/go-proxy/internal/net/gphttp/reverseproxy"
 )
 
-func SystemInfo(cfg config.ConfigInstance, w http.ResponseWriter, r *http.Request) {
+func SystemInfo(w http.ResponseWriter, r *http.Request) {
 	query := r.URL.Query()
 	agentAddr := query.Get("agent_addr")
 	query.Del("agent_addr")
@@ -21,7 +21,7 @@ func SystemInfo(cfg config.ConfigInstance, w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	agent, ok := cfg.GetAgent(agentAddr)
+	agent, ok := agent.Agents.Get(agentAddr)
 	if !ok {
 		gphttp.NotFound(w, "agent_addr")
 		return

internal/api/v1/version.go

@@ -8,5 +8,5 @@ import (
 )
 
 func GetVersion(w http.ResponseWriter, r *http.Request) {
-	gphttp.WriteBody(w, []byte(pkg.GetVersion()))
+	gphttp.WriteBody(w, []byte(pkg.GetVersion().String()))
 }