mirror of
https://github.com/yusing/godoxy.git
synced 2026-04-25 09:48:32 +02:00
fix(health): correct context handling, move NewMonitor, and improve docker health check errors
- Correct BaseContext nil check in Context() method - Move NewMonitor from monitor.go to new.go - Export ErrDockerHealthCheckFailedTooManyTimes and add ErrDockerHealthCheckNotAvailable - Return ErrDockerHealthCheckNotAvailable when container has no health check configured - Only log first docker health check failure and skip logging for ErrDockerHealthCheckNotAvailable - Use mon.Context() instead of mon.task.Context() to avoid nil panic
This commit is contained in:
@@ -23,7 +23,8 @@ type DockerHealthcheckState struct {
|
|||||||
|
|
||||||
const dockerFailuresThreshold = 3
|
const dockerFailuresThreshold = 3
|
||||||
|
|
||||||
var errDockerHealthCheckFailedTooManyTimes = errors.New("docker health check failed too many times")
|
var ErrDockerHealthCheckFailedTooManyTimes = errors.New("docker health check failed too many times")
|
||||||
|
var ErrDockerHealthCheckNotAvailable = errors.New("docker health check not available")
|
||||||
|
|
||||||
func NewDockerHealthcheckState(client *docker.SharedClient, containerId string) *DockerHealthcheckState {
|
func NewDockerHealthcheckState(client *docker.SharedClient, containerId string) *DockerHealthcheckState {
|
||||||
client.InterceptHTTPClient(interceptDockerInspectResponse)
|
client.InterceptHTTPClient(interceptDockerInspectResponse)
|
||||||
@@ -36,7 +37,7 @@ func NewDockerHealthcheckState(client *docker.SharedClient, containerId string)
|
|||||||
|
|
||||||
func Docker(ctx context.Context, state *DockerHealthcheckState, containerId string, timeout time.Duration) (types.HealthCheckResult, error) {
|
func Docker(ctx context.Context, state *DockerHealthcheckState, containerId string, timeout time.Duration) (types.HealthCheckResult, error) {
|
||||||
if state.numDockerFailures > dockerFailuresThreshold {
|
if state.numDockerFailures > dockerFailuresThreshold {
|
||||||
return types.HealthCheckResult{}, errDockerHealthCheckFailedTooManyTimes
|
return types.HealthCheckResult{}, ErrDockerHealthCheckFailedTooManyTimes
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||||
@@ -76,9 +77,9 @@ func Docker(ctx context.Context, state *DockerHealthcheckState, containerId stri
|
|||||||
|
|
||||||
health := containerState.Health
|
health := containerState.Health
|
||||||
if health == nil {
|
if health == nil {
|
||||||
// no health check from docker, directly use fallback
|
// no health check from docker, return error to trigger fallback
|
||||||
state.numDockerFailures = dockerFailuresThreshold + 1
|
state.numDockerFailures = dockerFailuresThreshold + 1
|
||||||
return types.HealthCheckResult{}, errDockerHealthCheckFailedTooManyTimes
|
return types.HealthCheckResult{}, ErrDockerHealthCheckNotAvailable
|
||||||
}
|
}
|
||||||
|
|
||||||
state.numDockerFailures = 0
|
state.numDockerFailures = 0
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ import (
|
|||||||
"github.com/rs/zerolog"
|
"github.com/rs/zerolog"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/rs/zerolog/log"
|
||||||
config "github.com/yusing/godoxy/internal/config/types"
|
config "github.com/yusing/godoxy/internal/config/types"
|
||||||
"github.com/yusing/godoxy/internal/docker"
|
|
||||||
"github.com/yusing/godoxy/internal/notif"
|
"github.com/yusing/godoxy/internal/notif"
|
||||||
"github.com/yusing/godoxy/internal/types"
|
"github.com/yusing/godoxy/internal/types"
|
||||||
gperr "github.com/yusing/goutils/errs"
|
gperr "github.com/yusing/goutils/errs"
|
||||||
@@ -43,38 +42,6 @@ type (
|
|||||||
|
|
||||||
var ErrNegativeInterval = gperr.New("negative interval")
|
var ErrNegativeInterval = gperr.New("negative interval")
|
||||||
|
|
||||||
func NewMonitor(r types.Route) types.HealthMonCheck {
|
|
||||||
target := &r.TargetURL().URL
|
|
||||||
|
|
||||||
var mon types.HealthMonCheck
|
|
||||||
if r.IsAgent() {
|
|
||||||
mon = NewAgentProxiedMonitor(r.HealthCheckConfig(), r.GetAgent(), target)
|
|
||||||
} else {
|
|
||||||
switch r := r.(type) {
|
|
||||||
case types.ReverseProxyRoute:
|
|
||||||
mon = NewHTTPHealthMonitor(r.HealthCheckConfig(), target)
|
|
||||||
case types.FileServerRoute:
|
|
||||||
mon = NewFileServerHealthMonitor(r.HealthCheckConfig(), r.RootPath())
|
|
||||||
case types.StreamRoute:
|
|
||||||
mon = NewStreamHealthMonitor(r.HealthCheckConfig(), target)
|
|
||||||
default:
|
|
||||||
log.Panic().Msgf("unexpected route type: %T", r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if r.IsDocker() {
|
|
||||||
cont := r.ContainerInfo()
|
|
||||||
client, err := docker.NewClient(cont.DockerCfg, true)
|
|
||||||
if err != nil {
|
|
||||||
return mon
|
|
||||||
}
|
|
||||||
r.Task().OnCancel("close_docker_client", client.Close)
|
|
||||||
|
|
||||||
fallback := mon
|
|
||||||
return NewDockerHealthMonitor(r.HealthCheckConfig(), client, cont.ContainerID, fallback)
|
|
||||||
}
|
|
||||||
return mon
|
|
||||||
}
|
|
||||||
|
|
||||||
func (mon *monitor) init(u *url.URL, cfg types.HealthCheckConfig, healthCheckFunc HealthCheckFunc) *monitor {
|
func (mon *monitor) init(u *url.URL, cfg types.HealthCheckConfig, healthCheckFunc HealthCheckFunc) *monitor {
|
||||||
if state := config.WorkingState.Load(); state != nil {
|
if state := config.WorkingState.Load(); state != nil {
|
||||||
cfg.ApplyDefaults(state.Value().Defaults.HealthCheck)
|
cfg.ApplyDefaults(state.Value().Defaults.HealthCheck)
|
||||||
@@ -96,16 +63,14 @@ func (mon *monitor) init(u *url.URL, cfg types.HealthCheckConfig, healthCheckFun
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (mon *monitor) ContextWithTimeout(cause string) (ctx context.Context, cancel context.CancelFunc) {
|
func (mon *monitor) Context() context.Context {
|
||||||
switch {
|
if mon.config.BaseContext != nil {
|
||||||
case mon.config.BaseContext != nil:
|
return mon.config.BaseContext()
|
||||||
ctx = mon.config.BaseContext()
|
|
||||||
case mon.task != nil:
|
|
||||||
ctx = mon.task.Context()
|
|
||||||
default:
|
|
||||||
ctx = context.Background()
|
|
||||||
}
|
}
|
||||||
return context.WithTimeoutCause(ctx, mon.config.Timeout, gperr.New(cause))
|
if mon.task != nil {
|
||||||
|
return mon.task.Context()
|
||||||
|
}
|
||||||
|
return context.Background()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (mon *monitor) CheckHealth() (types.HealthCheckResult, error) {
|
func (mon *monitor) CheckHealth() (types.HealthCheckResult, error) {
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package monitor
|
package monitor
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
@@ -16,6 +17,41 @@ import (
|
|||||||
type Result = types.HealthCheckResult
|
type Result = types.HealthCheckResult
|
||||||
type Monitor = types.HealthMonCheck
|
type Monitor = types.HealthMonCheck
|
||||||
|
|
||||||
|
// NewMonitor creates a health monitor based on the route type and configuration.
|
||||||
|
//
|
||||||
|
// See internal/health/monitor/README.md for detailed health check flow and conditions.
|
||||||
|
func NewMonitor(r types.Route) Monitor {
|
||||||
|
target := &r.TargetURL().URL
|
||||||
|
|
||||||
|
var mon Monitor
|
||||||
|
if r.IsAgent() {
|
||||||
|
mon = NewAgentProxiedMonitor(r.HealthCheckConfig(), r.GetAgent(), target)
|
||||||
|
} else {
|
||||||
|
switch r := r.(type) {
|
||||||
|
case types.ReverseProxyRoute:
|
||||||
|
mon = NewHTTPHealthMonitor(r.HealthCheckConfig(), target)
|
||||||
|
case types.FileServerRoute:
|
||||||
|
mon = NewFileServerHealthMonitor(r.HealthCheckConfig(), r.RootPath())
|
||||||
|
case types.StreamRoute:
|
||||||
|
mon = NewStreamHealthMonitor(r.HealthCheckConfig(), target)
|
||||||
|
default:
|
||||||
|
log.Panic().Msgf("unexpected route type: %T", r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if r.IsDocker() {
|
||||||
|
cont := r.ContainerInfo()
|
||||||
|
client, err := docker.NewClient(cont.DockerCfg, true)
|
||||||
|
if err != nil {
|
||||||
|
return mon
|
||||||
|
}
|
||||||
|
r.Task().OnCancel("close_docker_client", client.Close)
|
||||||
|
|
||||||
|
fallback := mon
|
||||||
|
return NewDockerHealthMonitor(r.HealthCheckConfig(), client, cont.ContainerID, fallback)
|
||||||
|
}
|
||||||
|
return mon
|
||||||
|
}
|
||||||
|
|
||||||
func NewHTTPHealthMonitor(config types.HealthCheckConfig, u *url.URL) Monitor {
|
func NewHTTPHealthMonitor(config types.HealthCheckConfig, u *url.URL) Monitor {
|
||||||
var method string
|
var method string
|
||||||
if config.UseGet {
|
if config.UseGet {
|
||||||
@@ -27,7 +63,7 @@ func NewHTTPHealthMonitor(config types.HealthCheckConfig, u *url.URL) Monitor {
|
|||||||
var mon monitor
|
var mon monitor
|
||||||
mon.init(u, config, func(u *url.URL) (result Result, err error) {
|
mon.init(u, config, func(u *url.URL) (result Result, err error) {
|
||||||
if u.Scheme == "h2c" {
|
if u.Scheme == "h2c" {
|
||||||
return healthcheck.H2C(mon.task.Context(), u, method, config.Path, config.Timeout)
|
return healthcheck.H2C(mon.Context(), u, method, config.Path, config.Timeout)
|
||||||
}
|
}
|
||||||
return healthcheck.HTTP(u, method, config.Path, config.Timeout)
|
return healthcheck.HTTP(u, method, config.Path, config.Timeout)
|
||||||
})
|
})
|
||||||
@@ -45,7 +81,7 @@ func NewFileServerHealthMonitor(config types.HealthCheckConfig, path string) Mon
|
|||||||
func NewStreamHealthMonitor(config types.HealthCheckConfig, targetUrl *url.URL) Monitor {
|
func NewStreamHealthMonitor(config types.HealthCheckConfig, targetUrl *url.URL) Monitor {
|
||||||
var mon monitor
|
var mon monitor
|
||||||
mon.init(targetUrl, config, func(u *url.URL) (result Result, err error) {
|
mon.init(targetUrl, config, func(u *url.URL) (result Result, err error) {
|
||||||
return healthcheck.Stream(mon.task.Context(), u, config.Timeout)
|
return healthcheck.Stream(mon.Context(), u, config.Timeout)
|
||||||
})
|
})
|
||||||
return &mon
|
return &mon
|
||||||
}
|
}
|
||||||
@@ -58,12 +94,18 @@ func NewDockerHealthMonitor(config types.HealthCheckConfig, client *docker.Share
|
|||||||
Path: "/containers/" + containerId + "/json",
|
Path: "/containers/" + containerId + "/json",
|
||||||
}
|
}
|
||||||
logger := log.With().Str("host", client.DaemonHost()).Str("container_id", containerId).Logger()
|
logger := log.With().Str("host", client.DaemonHost()).Str("container_id", containerId).Logger()
|
||||||
|
isFirstFailure := true
|
||||||
|
|
||||||
var mon monitor
|
var mon monitor
|
||||||
mon.init(displayURL, config, func(u *url.URL) (result Result, err error) {
|
mon.init(displayURL, config, func(u *url.URL) (result Result, err error) {
|
||||||
result, err = healthcheck.Docker(mon.task.Context(), state, containerId, config.Timeout)
|
result, err = healthcheck.Docker(mon.Context(), state, containerId, config.Timeout)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Err(err).Msg("docker health check failed, using fallback")
|
if isFirstFailure {
|
||||||
|
isFirstFailure = false
|
||||||
|
if !errors.Is(err, healthcheck.ErrDockerHealthCheckNotAvailable) {
|
||||||
|
logger.Err(err).Msg("docker health check failed, using fallback")
|
||||||
|
}
|
||||||
|
}
|
||||||
return fallback.CheckHealth()
|
return fallback.CheckHealth()
|
||||||
}
|
}
|
||||||
return result, nil
|
return result, nil
|
||||||
|
|||||||
Reference in New Issue
Block a user