diff --git a/internal/health/check/docker.go b/internal/health/check/docker.go index 80567240..49462827 100644 --- a/internal/health/check/docker.go +++ b/internal/health/check/docker.go @@ -23,7 +23,8 @@ type DockerHealthcheckState struct { const dockerFailuresThreshold = 3 -var errDockerHealthCheckFailedTooManyTimes = errors.New("docker health check failed too many times") +var ErrDockerHealthCheckFailedTooManyTimes = errors.New("docker health check failed too many times") +var ErrDockerHealthCheckNotAvailable = errors.New("docker health check not available") func NewDockerHealthcheckState(client *docker.SharedClient, containerId string) *DockerHealthcheckState { client.InterceptHTTPClient(interceptDockerInspectResponse) @@ -36,7 +37,7 @@ func NewDockerHealthcheckState(client *docker.SharedClient, containerId string) func Docker(ctx context.Context, state *DockerHealthcheckState, containerId string, timeout time.Duration) (types.HealthCheckResult, error) { if state.numDockerFailures > dockerFailuresThreshold { - return types.HealthCheckResult{}, errDockerHealthCheckFailedTooManyTimes + return types.HealthCheckResult{}, ErrDockerHealthCheckFailedTooManyTimes } ctx, cancel := context.WithTimeout(ctx, timeout) @@ -76,9 +77,9 @@ func Docker(ctx context.Context, state *DockerHealthcheckState, containerId stri health := containerState.Health if health == nil { - // no health check from docker, directly use fallback + // no health check from docker, return error to trigger fallback state.numDockerFailures = dockerFailuresThreshold + 1 - return types.HealthCheckResult{}, errDockerHealthCheckFailedTooManyTimes + return types.HealthCheckResult{}, ErrDockerHealthCheckNotAvailable } state.numDockerFailures = 0 diff --git a/internal/health/monitor/monitor.go b/internal/health/monitor/monitor.go index 5f56ec40..64bd4f62 100644 --- a/internal/health/monitor/monitor.go +++ b/internal/health/monitor/monitor.go @@ -11,7 +11,6 @@ import ( "github.com/rs/zerolog" "github.com/rs/zerolog/log" config "github.com/yusing/godoxy/internal/config/types" - "github.com/yusing/godoxy/internal/docker" "github.com/yusing/godoxy/internal/notif" "github.com/yusing/godoxy/internal/types" gperr "github.com/yusing/goutils/errs" @@ -43,38 +42,6 @@ type ( var ErrNegativeInterval = gperr.New("negative interval") -func NewMonitor(r types.Route) types.HealthMonCheck { - target := &r.TargetURL().URL - - var mon types.HealthMonCheck - if r.IsAgent() { - mon = NewAgentProxiedMonitor(r.HealthCheckConfig(), r.GetAgent(), target) - } else { - switch r := r.(type) { - case types.ReverseProxyRoute: - mon = NewHTTPHealthMonitor(r.HealthCheckConfig(), target) - case types.FileServerRoute: - mon = NewFileServerHealthMonitor(r.HealthCheckConfig(), r.RootPath()) - case types.StreamRoute: - mon = NewStreamHealthMonitor(r.HealthCheckConfig(), target) - default: - log.Panic().Msgf("unexpected route type: %T", r) - } - } - if r.IsDocker() { - cont := r.ContainerInfo() - client, err := docker.NewClient(cont.DockerCfg, true) - if err != nil { - return mon - } - r.Task().OnCancel("close_docker_client", client.Close) - - fallback := mon - return NewDockerHealthMonitor(r.HealthCheckConfig(), client, cont.ContainerID, fallback) - } - return mon -} - func (mon *monitor) init(u *url.URL, cfg types.HealthCheckConfig, healthCheckFunc HealthCheckFunc) *monitor { if state := config.WorkingState.Load(); state != nil { cfg.ApplyDefaults(state.Value().Defaults.HealthCheck) @@ -96,16 +63,14 @@ func (mon *monitor) init(u *url.URL, cfg types.HealthCheckConfig, healthCheckFun return nil } -func (mon *monitor) ContextWithTimeout(cause string) (ctx context.Context, cancel context.CancelFunc) { - switch { - case mon.config.BaseContext != nil: - ctx = mon.config.BaseContext() - case mon.task != nil: - ctx = mon.task.Context() - default: - ctx = context.Background() +func (mon *monitor) Context() context.Context { + if mon.config.BaseContext != nil { + return mon.config.BaseContext() } - return context.WithTimeoutCause(ctx, mon.config.Timeout, gperr.New(cause)) + if mon.task != nil { + return mon.task.Context() + } + return context.Background() } func (mon *monitor) CheckHealth() (types.HealthCheckResult, error) { diff --git a/internal/health/monitor/new.go b/internal/health/monitor/new.go index 3344c40a..d7e99eb5 100644 --- a/internal/health/monitor/new.go +++ b/internal/health/monitor/new.go @@ -1,6 +1,7 @@ package monitor import ( + "errors" "fmt" "net/http" "net/url" @@ -16,6 +17,41 @@ import ( type Result = types.HealthCheckResult type Monitor = types.HealthMonCheck +// NewMonitor creates a health monitor based on the route type and configuration. +// +// See internal/health/monitor/README.md for detailed health check flow and conditions. +func NewMonitor(r types.Route) Monitor { + target := &r.TargetURL().URL + + var mon Monitor + if r.IsAgent() { + mon = NewAgentProxiedMonitor(r.HealthCheckConfig(), r.GetAgent(), target) + } else { + switch r := r.(type) { + case types.ReverseProxyRoute: + mon = NewHTTPHealthMonitor(r.HealthCheckConfig(), target) + case types.FileServerRoute: + mon = NewFileServerHealthMonitor(r.HealthCheckConfig(), r.RootPath()) + case types.StreamRoute: + mon = NewStreamHealthMonitor(r.HealthCheckConfig(), target) + default: + log.Panic().Msgf("unexpected route type: %T", r) + } + } + if r.IsDocker() { + cont := r.ContainerInfo() + client, err := docker.NewClient(cont.DockerCfg, true) + if err != nil { + return mon + } + r.Task().OnCancel("close_docker_client", client.Close) + + fallback := mon + return NewDockerHealthMonitor(r.HealthCheckConfig(), client, cont.ContainerID, fallback) + } + return mon +} + func NewHTTPHealthMonitor(config types.HealthCheckConfig, u *url.URL) Monitor { var method string if config.UseGet { @@ -27,7 +63,7 @@ func NewHTTPHealthMonitor(config types.HealthCheckConfig, u *url.URL) Monitor { var mon monitor mon.init(u, config, func(u *url.URL) (result Result, err error) { if u.Scheme == "h2c" { - return healthcheck.H2C(mon.task.Context(), u, method, config.Path, config.Timeout) + return healthcheck.H2C(mon.Context(), u, method, config.Path, config.Timeout) } return healthcheck.HTTP(u, method, config.Path, config.Timeout) }) @@ -45,7 +81,7 @@ func NewFileServerHealthMonitor(config types.HealthCheckConfig, path string) Mon func NewStreamHealthMonitor(config types.HealthCheckConfig, targetUrl *url.URL) Monitor { var mon monitor mon.init(targetUrl, config, func(u *url.URL) (result Result, err error) { - return healthcheck.Stream(mon.task.Context(), u, config.Timeout) + return healthcheck.Stream(mon.Context(), u, config.Timeout) }) return &mon } @@ -58,12 +94,18 @@ func NewDockerHealthMonitor(config types.HealthCheckConfig, client *docker.Share Path: "/containers/" + containerId + "/json", } logger := log.With().Str("host", client.DaemonHost()).Str("container_id", containerId).Logger() + isFirstFailure := true var mon monitor mon.init(displayURL, config, func(u *url.URL) (result Result, err error) { - result, err = healthcheck.Docker(mon.task.Context(), state, containerId, config.Timeout) + result, err = healthcheck.Docker(mon.Context(), state, containerId, config.Timeout) if err != nil { - logger.Err(err).Msg("docker health check failed, using fallback") + if isFirstFailure { + isFirstFailure = false + if !errors.Is(err, healthcheck.ErrDockerHealthCheckNotAvailable) { + logger.Err(err).Msg("docker health check failed, using fallback") + } + } return fallback.CheckHealth() } return result, nil