mirror of
https://github.com/yusing/godoxy.git
synced 2026-01-11 14:20:32 +01:00
fix(health): correct context handling, move NewMonitor, and improve docker health check errors
- Correct BaseContext nil check in Context() method - Move NewMonitor from monitor.go to new.go - Export ErrDockerHealthCheckFailedTooManyTimes and add ErrDockerHealthCheckNotAvailable - Return ErrDockerHealthCheckNotAvailable when container has no health check configured - Only log first docker health check failure and skip logging for ErrDockerHealthCheckNotAvailable - Use mon.Context() instead of mon.task.Context() to avoid nil panic
This commit is contained in:
@@ -23,7 +23,8 @@ type DockerHealthcheckState struct {
|
||||
|
||||
const dockerFailuresThreshold = 3
|
||||
|
||||
var errDockerHealthCheckFailedTooManyTimes = errors.New("docker health check failed too many times")
|
||||
var ErrDockerHealthCheckFailedTooManyTimes = errors.New("docker health check failed too many times")
|
||||
var ErrDockerHealthCheckNotAvailable = errors.New("docker health check not available")
|
||||
|
||||
func NewDockerHealthcheckState(client *docker.SharedClient, containerId string) *DockerHealthcheckState {
|
||||
client.InterceptHTTPClient(interceptDockerInspectResponse)
|
||||
@@ -36,7 +37,7 @@ func NewDockerHealthcheckState(client *docker.SharedClient, containerId string)
|
||||
|
||||
func Docker(ctx context.Context, state *DockerHealthcheckState, containerId string, timeout time.Duration) (types.HealthCheckResult, error) {
|
||||
if state.numDockerFailures > dockerFailuresThreshold {
|
||||
return types.HealthCheckResult{}, errDockerHealthCheckFailedTooManyTimes
|
||||
return types.HealthCheckResult{}, ErrDockerHealthCheckFailedTooManyTimes
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
@@ -76,9 +77,9 @@ func Docker(ctx context.Context, state *DockerHealthcheckState, containerId stri
|
||||
|
||||
health := containerState.Health
|
||||
if health == nil {
|
||||
// no health check from docker, directly use fallback
|
||||
// no health check from docker, return error to trigger fallback
|
||||
state.numDockerFailures = dockerFailuresThreshold + 1
|
||||
return types.HealthCheckResult{}, errDockerHealthCheckFailedTooManyTimes
|
||||
return types.HealthCheckResult{}, ErrDockerHealthCheckNotAvailable
|
||||
}
|
||||
|
||||
state.numDockerFailures = 0
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/rs/zerolog/log"
|
||||
config "github.com/yusing/godoxy/internal/config/types"
|
||||
"github.com/yusing/godoxy/internal/docker"
|
||||
"github.com/yusing/godoxy/internal/notif"
|
||||
"github.com/yusing/godoxy/internal/types"
|
||||
gperr "github.com/yusing/goutils/errs"
|
||||
@@ -43,38 +42,6 @@ type (
|
||||
|
||||
var ErrNegativeInterval = gperr.New("negative interval")
|
||||
|
||||
func NewMonitor(r types.Route) types.HealthMonCheck {
|
||||
target := &r.TargetURL().URL
|
||||
|
||||
var mon types.HealthMonCheck
|
||||
if r.IsAgent() {
|
||||
mon = NewAgentProxiedMonitor(r.HealthCheckConfig(), r.GetAgent(), target)
|
||||
} else {
|
||||
switch r := r.(type) {
|
||||
case types.ReverseProxyRoute:
|
||||
mon = NewHTTPHealthMonitor(r.HealthCheckConfig(), target)
|
||||
case types.FileServerRoute:
|
||||
mon = NewFileServerHealthMonitor(r.HealthCheckConfig(), r.RootPath())
|
||||
case types.StreamRoute:
|
||||
mon = NewStreamHealthMonitor(r.HealthCheckConfig(), target)
|
||||
default:
|
||||
log.Panic().Msgf("unexpected route type: %T", r)
|
||||
}
|
||||
}
|
||||
if r.IsDocker() {
|
||||
cont := r.ContainerInfo()
|
||||
client, err := docker.NewClient(cont.DockerCfg, true)
|
||||
if err != nil {
|
||||
return mon
|
||||
}
|
||||
r.Task().OnCancel("close_docker_client", client.Close)
|
||||
|
||||
fallback := mon
|
||||
return NewDockerHealthMonitor(r.HealthCheckConfig(), client, cont.ContainerID, fallback)
|
||||
}
|
||||
return mon
|
||||
}
|
||||
|
||||
func (mon *monitor) init(u *url.URL, cfg types.HealthCheckConfig, healthCheckFunc HealthCheckFunc) *monitor {
|
||||
if state := config.WorkingState.Load(); state != nil {
|
||||
cfg.ApplyDefaults(state.Value().Defaults.HealthCheck)
|
||||
@@ -96,16 +63,14 @@ func (mon *monitor) init(u *url.URL, cfg types.HealthCheckConfig, healthCheckFun
|
||||
return nil
|
||||
}
|
||||
|
||||
func (mon *monitor) ContextWithTimeout(cause string) (ctx context.Context, cancel context.CancelFunc) {
|
||||
switch {
|
||||
case mon.config.BaseContext != nil:
|
||||
ctx = mon.config.BaseContext()
|
||||
case mon.task != nil:
|
||||
ctx = mon.task.Context()
|
||||
default:
|
||||
ctx = context.Background()
|
||||
func (mon *monitor) Context() context.Context {
|
||||
if mon.config.BaseContext != nil {
|
||||
return mon.config.BaseContext()
|
||||
}
|
||||
return context.WithTimeoutCause(ctx, mon.config.Timeout, gperr.New(cause))
|
||||
if mon.task != nil {
|
||||
return mon.task.Context()
|
||||
}
|
||||
return context.Background()
|
||||
}
|
||||
|
||||
func (mon *monitor) CheckHealth() (types.HealthCheckResult, error) {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -16,6 +17,41 @@ import (
|
||||
type Result = types.HealthCheckResult
|
||||
type Monitor = types.HealthMonCheck
|
||||
|
||||
// NewMonitor creates a health monitor based on the route type and configuration.
|
||||
//
|
||||
// See internal/health/monitor/README.md for detailed health check flow and conditions.
|
||||
func NewMonitor(r types.Route) Monitor {
|
||||
target := &r.TargetURL().URL
|
||||
|
||||
var mon Monitor
|
||||
if r.IsAgent() {
|
||||
mon = NewAgentProxiedMonitor(r.HealthCheckConfig(), r.GetAgent(), target)
|
||||
} else {
|
||||
switch r := r.(type) {
|
||||
case types.ReverseProxyRoute:
|
||||
mon = NewHTTPHealthMonitor(r.HealthCheckConfig(), target)
|
||||
case types.FileServerRoute:
|
||||
mon = NewFileServerHealthMonitor(r.HealthCheckConfig(), r.RootPath())
|
||||
case types.StreamRoute:
|
||||
mon = NewStreamHealthMonitor(r.HealthCheckConfig(), target)
|
||||
default:
|
||||
log.Panic().Msgf("unexpected route type: %T", r)
|
||||
}
|
||||
}
|
||||
if r.IsDocker() {
|
||||
cont := r.ContainerInfo()
|
||||
client, err := docker.NewClient(cont.DockerCfg, true)
|
||||
if err != nil {
|
||||
return mon
|
||||
}
|
||||
r.Task().OnCancel("close_docker_client", client.Close)
|
||||
|
||||
fallback := mon
|
||||
return NewDockerHealthMonitor(r.HealthCheckConfig(), client, cont.ContainerID, fallback)
|
||||
}
|
||||
return mon
|
||||
}
|
||||
|
||||
func NewHTTPHealthMonitor(config types.HealthCheckConfig, u *url.URL) Monitor {
|
||||
var method string
|
||||
if config.UseGet {
|
||||
@@ -27,7 +63,7 @@ func NewHTTPHealthMonitor(config types.HealthCheckConfig, u *url.URL) Monitor {
|
||||
var mon monitor
|
||||
mon.init(u, config, func(u *url.URL) (result Result, err error) {
|
||||
if u.Scheme == "h2c" {
|
||||
return healthcheck.H2C(mon.task.Context(), u, method, config.Path, config.Timeout)
|
||||
return healthcheck.H2C(mon.Context(), u, method, config.Path, config.Timeout)
|
||||
}
|
||||
return healthcheck.HTTP(u, method, config.Path, config.Timeout)
|
||||
})
|
||||
@@ -45,7 +81,7 @@ func NewFileServerHealthMonitor(config types.HealthCheckConfig, path string) Mon
|
||||
func NewStreamHealthMonitor(config types.HealthCheckConfig, targetUrl *url.URL) Monitor {
|
||||
var mon monitor
|
||||
mon.init(targetUrl, config, func(u *url.URL) (result Result, err error) {
|
||||
return healthcheck.Stream(mon.task.Context(), u, config.Timeout)
|
||||
return healthcheck.Stream(mon.Context(), u, config.Timeout)
|
||||
})
|
||||
return &mon
|
||||
}
|
||||
@@ -58,12 +94,18 @@ func NewDockerHealthMonitor(config types.HealthCheckConfig, client *docker.Share
|
||||
Path: "/containers/" + containerId + "/json",
|
||||
}
|
||||
logger := log.With().Str("host", client.DaemonHost()).Str("container_id", containerId).Logger()
|
||||
isFirstFailure := true
|
||||
|
||||
var mon monitor
|
||||
mon.init(displayURL, config, func(u *url.URL) (result Result, err error) {
|
||||
result, err = healthcheck.Docker(mon.task.Context(), state, containerId, config.Timeout)
|
||||
result, err = healthcheck.Docker(mon.Context(), state, containerId, config.Timeout)
|
||||
if err != nil {
|
||||
logger.Err(err).Msg("docker health check failed, using fallback")
|
||||
if isFirstFailure {
|
||||
isFirstFailure = false
|
||||
if !errors.Is(err, healthcheck.ErrDockerHealthCheckNotAvailable) {
|
||||
logger.Err(err).Msg("docker health check failed, using fallback")
|
||||
}
|
||||
}
|
||||
return fallback.CheckHealth()
|
||||
}
|
||||
return result, nil
|
||||
|
||||
Reference in New Issue
Block a user