diff --git a/.gitignore b/.gitignore index b7d53b0c..8298d645 100755 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ tsconfig.tsbuildinfo !agent.compose.yml !agent/pkg/** +dev-data/ \ No newline at end of file diff --git a/Makefile b/Makefile index 1a99ee1b..785a5f75 100755 --- a/Makefile +++ b/Makefile @@ -148,6 +148,7 @@ push-github: git push origin $(shell git rev-parse --abbrev-ref HEAD) gen-swagger: + # go install github.com/swaggo/swag/cmd/swag@latest swag init --parseDependency --parseInternal --parseFuncBody -g handler.go -d internal/api -o internal/api/v1/docs python3 scripts/fix-swagger-json.py # we don't need this diff --git a/internal/api/v1/docs/swagger.json b/internal/api/v1/docs/swagger.json index ce63ac61..9cea7db6 100644 --- a/internal/api/v1/docs/swagger.json +++ b/internal/api/v1/docs/swagger.json @@ -2458,8 +2458,8 @@ "x-nullable": false, "x-omitempty": false }, - "docker_host": { - "type": "string", + "docker_cfg": { + "$ref": "#/definitions/DockerProviderConfig", "x-nullable": false, "x-omitempty": false }, @@ -2715,7 +2715,7 @@ "required": [ "container_id", "container_name", - "docker_host" + "docker_cfg" ], "properties": { "container_id": { @@ -2728,7 +2728,24 @@ "x-nullable": false, "x-omitempty": false }, - "docker_host": { + "docker_cfg": { + "$ref": "#/definitions/DockerProviderConfig", + "x-nullable": false, + "x-omitempty": false + } + }, + "x-nullable": false, + "x-omitempty": false + }, + "DockerProviderConfig": { + "type": "object", + "properties": { + "tls": { + "$ref": "#/definitions/DockerTLSConfig", + "x-nullable": false, + "x-omitempty": false + }, + "url": { "type": "string", "x-nullable": false, "x-omitempty": false @@ -2737,6 +2754,27 @@ "x-nullable": false, "x-omitempty": false }, + "DockerTLSConfig": { + "type": "object", + "required": [ + "ca_file" + ], + "properties": { + "ca_file": { + "type": "string", + "x-nullable": false, + "x-omitempty": false + }, + "cert_file": { + "type": "string" + }, + "key_file": { + "type": "string" + } + }, + "x-nullable": false, + "x-omitempty": false + }, "ErrorResponse": { "type": "object", "properties": { @@ -3430,6 +3468,11 @@ "x-nullable": false, "x-omitempty": false }, + "no_loading_page": { + "type": "boolean", + "x-nullable": false, + "x-omitempty": false + }, "proxmox": { "$ref": "#/definitions/ProxmoxConfig", "x-nullable": false, @@ -4234,6 +4277,12 @@ ], "x-nullable": true }, + "index": { + "description": "Index file to serve for single-page app mode", + "type": "string", + "x-nullable": false, + "x-omitempty": false + }, "load_balance": { "allOf": [ { @@ -4315,6 +4364,12 @@ "x-nullable": false, "x-omitempty": false }, + "spa": { + "description": "Single-page app mode: serves index for non-existent paths", + "type": "boolean", + "x-nullable": false, + "x-omitempty": false + }, "ssl_certificate": { "description": "Path to client certificate", "type": "string", @@ -5354,6 +5409,12 @@ ], "x-nullable": true }, + "index": { + "description": "Index file to serve for single-page app mode", + "type": "string", + "x-nullable": false, + "x-omitempty": false + }, "load_balance": { "allOf": [ { @@ -5435,6 +5496,12 @@ "x-nullable": false, "x-omitempty": false }, + "spa": { + "description": "Single-page app mode: serves index for non-existent paths", + "type": "boolean", + "x-nullable": false, + "x-omitempty": false + }, "ssl_certificate": { "description": "Path to client certificate", "type": "string", diff --git a/internal/api/v1/docs/swagger.yaml b/internal/api/v1/docs/swagger.yaml index 4f1bdcf7..e6744d88 100644 --- a/internal/api/v1/docs/swagger.yaml +++ b/internal/api/v1/docs/swagger.yaml @@ -57,8 +57,8 @@ definitions: type: string container_name: type: string - docker_host: - type: string + docker_cfg: + $ref: '#/definitions/DockerProviderConfig' errors: type: string idlewatcher_config: @@ -192,12 +192,30 @@ definitions: type: string container_name: type: string - docker_host: - type: string + docker_cfg: + $ref: '#/definitions/DockerProviderConfig' required: - container_id - container_name - - docker_host + - docker_cfg + type: object + DockerProviderConfig: + properties: + tls: + $ref: '#/definitions/DockerTLSConfig' + url: + type: string + type: object + DockerTLSConfig: + properties: + ca_file: + type: string + cert_file: + type: string + key_file: + type: string + required: + - ca_file type: object ErrorResponse: properties: @@ -517,6 +535,8 @@ definitions: description: "0: no idle watcher.\nPositive: idle watcher with idle timeout.\nNegative: idle watcher as a dependency.\tIdleTimeout time.Duration `json:\"idle_timeout\" json_ext:\"duration\"`" + no_loading_page: + type: boolean proxmox: $ref: '#/definitions/ProxmoxConfig' start_endpoint: @@ -897,6 +917,9 @@ definitions: allOf: - $ref: '#/definitions/IdlewatcherConfig' x-nullable: true + index: + description: Index file to serve for single-page app mode + type: string load_balance: allOf: - $ref: '#/definitions/LoadBalancerConfig' @@ -944,6 +967,9 @@ definitions: - udp - fileserver type: string + spa: + description: 'Single-page app mode: serves index for non-existent paths' + type: boolean ssl_certificate: description: Path to client certificate type: string @@ -1504,6 +1530,9 @@ definitions: allOf: - $ref: '#/definitions/IdlewatcherConfig' x-nullable: true + index: + description: Index file to serve for single-page app mode + type: string load_balance: allOf: - $ref: '#/definitions/LoadBalancerConfig' @@ -1551,6 +1580,9 @@ definitions: - udp - fileserver type: string + spa: + description: 'Single-page app mode: serves index for non-existent paths' + type: boolean ssl_certificate: description: Path to client certificate type: string diff --git a/internal/api/v1/route/routes.go b/internal/api/v1/route/routes.go index b65a0297..8dcf2820 100644 --- a/internal/api/v1/route/routes.go +++ b/internal/api/v1/route/routes.go @@ -34,12 +34,12 @@ func Routes(c *gin.Context) { provider := c.Query("provider") if provider == "" { - c.JSON(http.StatusOK, slices.Collect(routes.Iter)) + c.JSON(http.StatusOK, slices.Collect(routes.IterAll)) return } - rts := make([]types.Route, 0, routes.NumRoutes()) - for r := range routes.Iter { + rts := make([]types.Route, 0, routes.NumAllRoutes()) + for r := range routes.IterAll { if r.ProviderName() == provider { rts = append(rts, r) } @@ -51,14 +51,14 @@ func RoutesWS(c *gin.Context) { provider := c.Query("provider") if provider == "" { websocket.PeriodicWrite(c, 3*time.Second, func() (any, error) { - return slices.Collect(routes.Iter), nil + return slices.Collect(routes.IterAll), nil }) return } websocket.PeriodicWrite(c, 3*time.Second, func() (any, error) { - rts := make([]types.Route, 0, routes.NumRoutes()) - for r := range routes.Iter { + rts := make([]types.Route, 0, routes.NumAllRoutes()) + for r := range routes.IterAll { if r.ProviderName() == provider { rts = append(rts, r) } diff --git a/internal/docker/client.go b/internal/docker/client.go index cac1ca16..a1ac7612 100644 --- a/internal/docker/client.go +++ b/internal/docker/client.go @@ -2,7 +2,6 @@ package docker import ( "context" - "errors" "fmt" "maps" "net" @@ -17,7 +16,6 @@ import ( "github.com/moby/moby/client" "github.com/rs/zerolog/log" "github.com/yusing/godoxy/agent/pkg/agent" - "github.com/yusing/godoxy/internal/common" "github.com/yusing/godoxy/internal/types" httputils "github.com/yusing/goutils/http" "github.com/yusing/goutils/task" @@ -118,7 +116,7 @@ func Clients() map[string]*SharedClient { // Returns existing client if available. // // Parameters: -// - host: the host to connect to (either a URL or common.DockerHostFromEnv). +// - host: the host to connect to (either a URL or client.DefaultDockerHost). // // Returns: // - Client: the Docker client connection. @@ -161,27 +159,18 @@ func NewClient(cfg types.DockerProviderConfig, unique ...bool) (*SharedClient, e addr = "tcp://" + cfg.Addr dial = cfg.DialContext } else { - switch host { - case "": - return nil, errors.New("empty docker host") - case common.DockerHostFromEnv: + helper, err := connhelper.GetConnectionHelper(host) + if err != nil { + log.Panic().Err(err).Msg("failed to get connection helper") + } + if helper != nil { opt = []client.Opt{ - client.WithHostFromEnv(), + client.WithHost(helper.Host), + client.WithDialContext(helper.Dialer), } - default: - helper, err := connhelper.GetConnectionHelper(host) - if err != nil { - log.Panic().Err(err).Msg("failed to get connection helper") - } - if helper != nil { - opt = []client.Opt{ - client.WithHost(helper.Host), - client.WithDialContext(helper.Dialer), - } - } else { - opt = []client.Opt{ - client.WithHost(host), - } + } else { + opt = []client.Opt{ + client.WithHost(host), } } } diff --git a/internal/metrics/uptime/uptime.go b/internal/metrics/uptime/uptime.go index 99b7f04a..11dc0718 100644 --- a/internal/metrics/uptime/uptime.go +++ b/internal/metrics/uptime/uptime.go @@ -8,7 +8,6 @@ import ( "github.com/bytedance/sonic" "github.com/lithammer/fuzzysearch/fuzzy" - statequery "github.com/yusing/godoxy/internal/config/query" "github.com/yusing/godoxy/internal/metrics/period" metricsutils "github.com/yusing/godoxy/internal/metrics/utils" "github.com/yusing/godoxy/internal/route/routes" @@ -133,7 +132,7 @@ func (rs RouteStatuses) aggregate(limit int, offset int) Aggregated { r, ok := routes.Get(alias) if !ok { // also search for excluded routes - r = statequery.SearchRoute(alias) + r, ok = routes.Excluded.Get(alias) } if r != nil { displayName = r.DisplayName() diff --git a/internal/route/provider/provider.go b/internal/route/provider/provider.go index 6d4dd277..aee257a7 100644 --- a/internal/route/provider/provider.go +++ b/internal/route/provider/provider.go @@ -8,17 +8,14 @@ import ( "sync" "time" - "github.com/moby/moby/client" "github.com/rs/zerolog" "github.com/yusing/godoxy/agent/pkg/agent" - "github.com/yusing/godoxy/internal/common" "github.com/yusing/godoxy/internal/docker" "github.com/yusing/godoxy/internal/route" provider "github.com/yusing/godoxy/internal/route/provider/types" "github.com/yusing/godoxy/internal/types" W "github.com/yusing/godoxy/internal/watcher" "github.com/yusing/godoxy/internal/watcher/events" - "github.com/yusing/goutils/env" gperr "github.com/yusing/goutils/errs" "github.com/yusing/goutils/task" ) @@ -70,10 +67,6 @@ func NewFileProvider(filename string) (p *Provider, err error) { } func NewDockerProvider(name string, dockerCfg types.DockerProviderConfig) *Provider { - if dockerCfg.URL == common.DockerHostFromEnv { - dockerCfg.URL = env.GetEnvString("DOCKER_HOST", client.DefaultDockerHost) - } - p := newProvider(provider.ProviderTypeDocker) p.ProviderImpl = DockerProviderImpl(name, dockerCfg) p.watcher = p.NewWatcher() diff --git a/internal/route/route.go b/internal/route/route.go index 7d447ea3..420b7c9f 100644 --- a/internal/route/route.go +++ b/internal/route/route.go @@ -24,12 +24,14 @@ import ( "github.com/yusing/godoxy/internal/proxmox" "github.com/yusing/godoxy/internal/serialization" "github.com/yusing/godoxy/internal/types" + "github.com/yusing/godoxy/internal/watcher/health/monitor" gperr "github.com/yusing/goutils/errs" strutils "github.com/yusing/goutils/strings" "github.com/yusing/goutils/task" "github.com/yusing/godoxy/internal/common" "github.com/yusing/godoxy/internal/logging/accesslog" + "github.com/yusing/godoxy/internal/route/routes" "github.com/yusing/godoxy/internal/route/rules" rulepresets "github.com/yusing/godoxy/internal/route/rules/presets" route "github.com/yusing/godoxy/internal/route/types" @@ -397,8 +399,17 @@ func (r *Route) start(parent task.Parent) gperr.Error { if err := r.impl.Start(parent); err != nil { return err } - } else { // required by idlewatcher - r.task = parent.Subtask("excluded."+r.Name(), false) + } else { + r.task = parent.Subtask("excluded."+r.Name(), true) + routes.Excluded.Add(r.impl) + r.task.OnCancel("remove_route_from_excluded", func() { + routes.Excluded.Del(r.impl) + }) + if r.UseHealthCheck() { + r.HealthMon = monitor.NewMonitor(r) + err := r.HealthMon.Start(r.task) + return err + } } return nil } diff --git a/internal/route/routes/query.go b/internal/route/routes/query.go index 527774cb..51d9c98a 100644 --- a/internal/route/routes/query.go +++ b/internal/route/routes/query.go @@ -17,17 +17,23 @@ type HealthInfoWithoutDetail struct { Latency time.Duration `json:"latency" swaggertype:"number"` // latency in microseconds } // @name HealthInfoWithoutDetail +// GetHealthInfo returns a map of route name to health info. +// +// The health info is for all routes, including excluded routes. func GetHealthInfo() map[string]HealthInfo { - healthMap := make(map[string]HealthInfo, NumRoutes()) - for r := range Iter { + healthMap := make(map[string]HealthInfo, NumAllRoutes()) + for r := range IterAll { healthMap[r.Name()] = getHealthInfo(r) } return healthMap } +// GetHealthInfoWithoutDetail returns a map of route name to health info without detail. +// +// The health info is for all routes, including excluded routes. func GetHealthInfoWithoutDetail() map[string]HealthInfoWithoutDetail { - healthMap := make(map[string]HealthInfoWithoutDetail, NumRoutes()) - for r := range Iter { + healthMap := make(map[string]HealthInfoWithoutDetail, NumAllRoutes()) + for r := range IterAll { healthMap[r.Name()] = getHealthInfoWithoutDetail(r) } return healthMap @@ -67,9 +73,12 @@ func getHealthInfoWithoutDetail(r types.Route) HealthInfoWithoutDetail { } } +// ByProvider returns a map of provider name to routes. +// +// The routes are all routes, including excluded routes. func ByProvider() map[string][]types.Route { rts := make(map[string][]types.Route) - for r := range Iter { + for r := range IterAll { rts[r.ProviderName()] = append(rts[r.ProviderName()], r) } return rts diff --git a/internal/route/routes/routes.go b/internal/route/routes/routes.go index 4c325124..61034610 100644 --- a/internal/route/routes/routes.go +++ b/internal/route/routes/routes.go @@ -8,9 +8,11 @@ import ( var ( HTTP = pool.New[types.HTTPRoute]("http_routes") Stream = pool.New[types.StreamRoute]("stream_routes") + + Excluded = pool.New[types.Route]("excluded_routes") ) -func Iter(yield func(r types.Route) bool) { +func IterActive(yield func(r types.Route) bool) { for _, r := range HTTP.Iter { if !yield(r) { break @@ -23,26 +25,36 @@ func Iter(yield func(r types.Route) bool) { } } -func IterKV(yield func(alias string, r types.Route) bool) { - for k, r := range HTTP.Iter { - if !yield(k, r) { +func IterAll(yield func(r types.Route) bool) { + for _, r := range HTTP.Iter { + if !yield(r) { break } } - for k, r := range Stream.Iter { - if !yield(k, r) { + for _, r := range Stream.Iter { + if !yield(r) { + break + } + } + for _, r := range Excluded.Iter { + if !yield(r) { break } } } -func NumRoutes() int { +func NumActiveRoutes() int { return HTTP.Size() + Stream.Size() } +func NumAllRoutes() int { + return HTTP.Size() + Stream.Size() + Excluded.Size() +} + func Clear() { HTTP.Clear() Stream.Clear() + Excluded.Clear() } func GetHTTPRouteOrExact(alias, host string) (types.HTTPRoute, bool) { @@ -54,6 +66,9 @@ func GetHTTPRouteOrExact(alias, host string) (types.HTTPRoute, bool) { return HTTP.Get(host) } +// Get returns the route with the given alias. +// +// It does not return excluded routes. func Get(alias string) (types.Route, bool) { if r, ok := HTTP.Get(alias); ok { return r, true diff --git a/internal/route/rules/do.go b/internal/route/rules/do.go index 3189f586..4a4a17d4 100644 --- a/internal/route/rules/do.go +++ b/internal/route/rules/do.go @@ -15,6 +15,7 @@ import ( nettypes "github.com/yusing/godoxy/internal/net/types" "github.com/yusing/godoxy/internal/notif" "github.com/yusing/godoxy/internal/route/routes" + "github.com/yusing/godoxy/internal/types" gperr "github.com/yusing/goutils/errs" httputils "github.com/yusing/goutils/http" "github.com/yusing/goutils/http/reverseproxy" @@ -38,6 +39,7 @@ const ( CommandServe = "serve" CommandProxy = "proxy" CommandRedirect = "redirect" + CommandRoute = "route" CommandError = "error" CommandRequireBasicAuth = "require_basic_auth" CommandSet = "set" @@ -171,6 +173,42 @@ var commands = map[string]struct { }) }, }, + CommandRoute: { + help: Help{ + command: CommandRoute, + description: makeLines( + "Route the request to another route, e.g.:", + helpExample(CommandRoute, "route1"), + ), + args: map[string]string{ + "route": "the route to route to", + }, + }, + validate: func(args []string) (any, gperr.Error) { + if len(args) != 1 { + return nil, ErrExpectOneArg + } + return args[0], nil + }, + build: func(args any) CommandHandler { + route := args.(string) + return TerminatingCommand(func(w http.ResponseWriter, req *http.Request) error { + r, ok := routes.HTTP.Get(route) + if !ok { + excluded, has := routes.Excluded.Get(route) + if has { + r, ok = excluded.(types.HTTPRoute) + } + } + if ok { + r.ServeHTTP(w, req) + } else { + http.Error(w, fmt.Sprintf("Route %q not found", route), http.StatusNotFound) + } + return nil + }) + }, + }, CommandError: { help: Help{ command: CommandError, diff --git a/internal/types/docker_provider_config.go b/internal/types/docker_provider_config.go index 9796f7ab..5b78ba31 100644 --- a/internal/types/docker_provider_config.go +++ b/internal/types/docker_provider_config.go @@ -10,6 +10,7 @@ import ( "github.com/yusing/godoxy/internal/common" "github.com/yusing/godoxy/internal/serialization" + "github.com/yusing/goutils/env" gperr "github.com/yusing/goutils/errs" ) @@ -36,6 +37,11 @@ func (cfg *DockerProviderConfig) MarshalJSON() ([]byte, error) { } func (cfg *DockerProviderConfig) Parse(value string) error { + if value == common.DockerHostFromEnv { + cfg.URL = env.GetEnvString("DOCKER_HOST", "unix:///var/run/docker.sock") + return nil + } + u, err := url.Parse(value) if err != nil { return err