fix(health): correct context handling, move NewMonitor, and improve docker health check errors

- Correct BaseContext nil check in Context() method
- Move NewMonitor from monitor.go to new.go
- Export ErrDockerHealthCheckFailedTooManyTimes and add ErrDockerHealthCheckNotAvailable
- Return ErrDockerHealthCheckNotAvailable when container has no health check configured
- Only log first docker health check failure and skip logging for ErrDockerHealthCheckNotAvailable
- Use mon.Context() instead of mon.task.Context() to avoid nil panic
This commit is contained in:
yusing
2026-01-08 18:14:14 +08:00
parent 581503e160
commit 7385761bdf
3 changed files with 58 additions and 50 deletions

View File

@@ -23,7 +23,8 @@ type DockerHealthcheckState struct {
const dockerFailuresThreshold = 3
var errDockerHealthCheckFailedTooManyTimes = errors.New("docker health check failed too many times")
var ErrDockerHealthCheckFailedTooManyTimes = errors.New("docker health check failed too many times")
var ErrDockerHealthCheckNotAvailable = errors.New("docker health check not available")
func NewDockerHealthcheckState(client *docker.SharedClient, containerId string) *DockerHealthcheckState {
client.InterceptHTTPClient(interceptDockerInspectResponse)
@@ -36,7 +37,7 @@ func NewDockerHealthcheckState(client *docker.SharedClient, containerId string)
func Docker(ctx context.Context, state *DockerHealthcheckState, containerId string, timeout time.Duration) (types.HealthCheckResult, error) {
if state.numDockerFailures > dockerFailuresThreshold {
return types.HealthCheckResult{}, errDockerHealthCheckFailedTooManyTimes
return types.HealthCheckResult{}, ErrDockerHealthCheckFailedTooManyTimes
}
ctx, cancel := context.WithTimeout(ctx, timeout)
@@ -76,9 +77,9 @@ func Docker(ctx context.Context, state *DockerHealthcheckState, containerId stri
health := containerState.Health
if health == nil {
// no health check from docker, directly use fallback
// no health check from docker, return error to trigger fallback
state.numDockerFailures = dockerFailuresThreshold + 1
return types.HealthCheckResult{}, errDockerHealthCheckFailedTooManyTimes
return types.HealthCheckResult{}, ErrDockerHealthCheckNotAvailable
}
state.numDockerFailures = 0

View File

@@ -11,7 +11,6 @@ import (
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
config "github.com/yusing/godoxy/internal/config/types"
"github.com/yusing/godoxy/internal/docker"
"github.com/yusing/godoxy/internal/notif"
"github.com/yusing/godoxy/internal/types"
gperr "github.com/yusing/goutils/errs"
@@ -43,38 +42,6 @@ type (
var ErrNegativeInterval = gperr.New("negative interval")
func NewMonitor(r types.Route) types.HealthMonCheck {
target := &r.TargetURL().URL
var mon types.HealthMonCheck
if r.IsAgent() {
mon = NewAgentProxiedMonitor(r.HealthCheckConfig(), r.GetAgent(), target)
} else {
switch r := r.(type) {
case types.ReverseProxyRoute:
mon = NewHTTPHealthMonitor(r.HealthCheckConfig(), target)
case types.FileServerRoute:
mon = NewFileServerHealthMonitor(r.HealthCheckConfig(), r.RootPath())
case types.StreamRoute:
mon = NewStreamHealthMonitor(r.HealthCheckConfig(), target)
default:
log.Panic().Msgf("unexpected route type: %T", r)
}
}
if r.IsDocker() {
cont := r.ContainerInfo()
client, err := docker.NewClient(cont.DockerCfg, true)
if err != nil {
return mon
}
r.Task().OnCancel("close_docker_client", client.Close)
fallback := mon
return NewDockerHealthMonitor(r.HealthCheckConfig(), client, cont.ContainerID, fallback)
}
return mon
}
func (mon *monitor) init(u *url.URL, cfg types.HealthCheckConfig, healthCheckFunc HealthCheckFunc) *monitor {
if state := config.WorkingState.Load(); state != nil {
cfg.ApplyDefaults(state.Value().Defaults.HealthCheck)
@@ -96,16 +63,14 @@ func (mon *monitor) init(u *url.URL, cfg types.HealthCheckConfig, healthCheckFun
return nil
}
func (mon *monitor) ContextWithTimeout(cause string) (ctx context.Context, cancel context.CancelFunc) {
switch {
case mon.config.BaseContext != nil:
ctx = mon.config.BaseContext()
case mon.task != nil:
ctx = mon.task.Context()
default:
ctx = context.Background()
func (mon *monitor) Context() context.Context {
if mon.config.BaseContext != nil {
return mon.config.BaseContext()
}
return context.WithTimeoutCause(ctx, mon.config.Timeout, gperr.New(cause))
if mon.task != nil {
return mon.task.Context()
}
return context.Background()
}
func (mon *monitor) CheckHealth() (types.HealthCheckResult, error) {

View File

@@ -1,6 +1,7 @@
package monitor
import (
"errors"
"fmt"
"net/http"
"net/url"
@@ -16,6 +17,41 @@ import (
type Result = types.HealthCheckResult
type Monitor = types.HealthMonCheck
// NewMonitor creates a health monitor based on the route type and configuration.
//
// See internal/health/monitor/README.md for detailed health check flow and conditions.
func NewMonitor(r types.Route) Monitor {
target := &r.TargetURL().URL
var mon Monitor
if r.IsAgent() {
mon = NewAgentProxiedMonitor(r.HealthCheckConfig(), r.GetAgent(), target)
} else {
switch r := r.(type) {
case types.ReverseProxyRoute:
mon = NewHTTPHealthMonitor(r.HealthCheckConfig(), target)
case types.FileServerRoute:
mon = NewFileServerHealthMonitor(r.HealthCheckConfig(), r.RootPath())
case types.StreamRoute:
mon = NewStreamHealthMonitor(r.HealthCheckConfig(), target)
default:
log.Panic().Msgf("unexpected route type: %T", r)
}
}
if r.IsDocker() {
cont := r.ContainerInfo()
client, err := docker.NewClient(cont.DockerCfg, true)
if err != nil {
return mon
}
r.Task().OnCancel("close_docker_client", client.Close)
fallback := mon
return NewDockerHealthMonitor(r.HealthCheckConfig(), client, cont.ContainerID, fallback)
}
return mon
}
func NewHTTPHealthMonitor(config types.HealthCheckConfig, u *url.URL) Monitor {
var method string
if config.UseGet {
@@ -27,7 +63,7 @@ func NewHTTPHealthMonitor(config types.HealthCheckConfig, u *url.URL) Monitor {
var mon monitor
mon.init(u, config, func(u *url.URL) (result Result, err error) {
if u.Scheme == "h2c" {
return healthcheck.H2C(mon.task.Context(), u, method, config.Path, config.Timeout)
return healthcheck.H2C(mon.Context(), u, method, config.Path, config.Timeout)
}
return healthcheck.HTTP(u, method, config.Path, config.Timeout)
})
@@ -45,7 +81,7 @@ func NewFileServerHealthMonitor(config types.HealthCheckConfig, path string) Mon
func NewStreamHealthMonitor(config types.HealthCheckConfig, targetUrl *url.URL) Monitor {
var mon monitor
mon.init(targetUrl, config, func(u *url.URL) (result Result, err error) {
return healthcheck.Stream(mon.task.Context(), u, config.Timeout)
return healthcheck.Stream(mon.Context(), u, config.Timeout)
})
return &mon
}
@@ -58,12 +94,18 @@ func NewDockerHealthMonitor(config types.HealthCheckConfig, client *docker.Share
Path: "/containers/" + containerId + "/json",
}
logger := log.With().Str("host", client.DaemonHost()).Str("container_id", containerId).Logger()
isFirstFailure := true
var mon monitor
mon.init(displayURL, config, func(u *url.URL) (result Result, err error) {
result, err = healthcheck.Docker(mon.task.Context(), state, containerId, config.Timeout)
result, err = healthcheck.Docker(mon.Context(), state, containerId, config.Timeout)
if err != nil {
logger.Err(err).Msg("docker health check failed, using fallback")
if isFirstFailure {
isFirstFailure = false
if !errors.Is(err, healthcheck.ErrDockerHealthCheckNotAvailable) {
logger.Err(err).Msg("docker health check failed, using fallback")
}
}
return fallback.CheckHealth()
}
return result, nil