refactor health module

internal/watcher/health/monitor/http.go

@@ -0,0 +1,80 @@
+package monitor
+
+import (
+	"crypto/tls"
+	"errors"
+	"net/http"
+
+	"github.com/yusing/go-proxy/internal/net/types"
+	"github.com/yusing/go-proxy/internal/watcher/health"
+	"github.com/yusing/go-proxy/pkg"
+)
+
+type HTTPHealthMonitor struct {
+	*monitor
+	method string
+}
+
+var pinger = &http.Client{
+	Transport: &http.Transport{
+		DisableKeepAlives: true,
+		ForceAttemptHTTP2: false,
+	},
+	CheckRedirect: func(r *http.Request, via []*http.Request) error {
+		return http.ErrUseLastResponse
+	},
+}
+
+func NewHTTPHealthMonitor(url types.URL, config *health.HealthCheckConfig) *HTTPHealthMonitor {
+	mon := new(HTTPHealthMonitor)
+	mon.monitor = newMonitor(url, config, mon.CheckHealth)
+	if config.UseGet {
+		mon.method = http.MethodGet
+	} else {
+		mon.method = http.MethodHead
+	}
+	return mon
+}
+
+func NewHTTPHealthChecker(url types.URL, config *health.HealthCheckConfig) health.HealthChecker {
+	return NewHTTPHealthMonitor(url, config)
+}
+
+func (mon *HTTPHealthMonitor) CheckHealth() (healthy bool, detail string, err error) {
+	ctx, cancel := mon.ContextWithTimeout("ping request timed out")
+	defer cancel()
+
+	req, reqErr := http.NewRequestWithContext(
+		ctx,
+		mon.method,
+		mon.url.Load().JoinPath(mon.config.Path).String(),
+		nil,
+	)
+	if reqErr != nil {
+		err = reqErr
+		return
+	}
+
+	req.Header.Set("Connection", "close")
+	req.Header.Set("User-Agent", "GoDoxy/"+pkg.GetVersion())
+	resp, respErr := pinger.Do(req)
+	if respErr == nil {
+		resp.Body.Close()
+	}
+
+	switch {
+	case respErr != nil:
+		// treat tls error as healthy
+		var tlsErr *tls.CertificateVerificationError
+		if ok := errors.As(respErr, &tlsErr); !ok {
+			detail = respErr.Error()
+			return
+		}
+	case resp.StatusCode == http.StatusServiceUnavailable:
+		detail = resp.Status
+		return
+	}
+
+	healthy = true
+	return
+}

internal/watcher/health/monitor/json.go

@@ -0,0 +1,38 @@
+package monitor
+
+import (
+	"encoding/json"
+	"time"
+
+	"github.com/yusing/go-proxy/internal/net/types"
+	"github.com/yusing/go-proxy/internal/utils/strutils"
+	"github.com/yusing/go-proxy/internal/watcher/health"
+)
+
+type JSONRepresentation struct {
+	Name    string
+	Config  *health.HealthCheckConfig
+	Status  health.Status
+	Started time.Time
+	Uptime  time.Duration
+	URL     types.URL
+	Extra   map[string]any
+}
+
+func (jsonRepr *JSONRepresentation) MarshalJSON() ([]byte, error) {
+	url := jsonRepr.URL.String()
+	if url == "http://:0" {
+		url = ""
+	}
+	return json.Marshal(map[string]any{
+		"name":       jsonRepr.Name,
+		"config":     jsonRepr.Config,
+		"started":    jsonRepr.Started.Unix(),
+		"startedStr": strutils.FormatTime(jsonRepr.Started),
+		"status":     jsonRepr.Status.String(),
+		"uptime":     jsonRepr.Uptime.Seconds(),
+		"uptimeStr":  strutils.FormatDuration(jsonRepr.Uptime),
+		"url":        url,
+		"extra":      jsonRepr.Extra,
+	})
+}

internal/watcher/health/monitor/monitor.go

@@ -0,0 +1,194 @@
+package monitor
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/yusing/go-proxy/internal/common"
+	E "github.com/yusing/go-proxy/internal/error"
+	"github.com/yusing/go-proxy/internal/logging"
+	"github.com/yusing/go-proxy/internal/metrics"
+	"github.com/yusing/go-proxy/internal/net/types"
+	"github.com/yusing/go-proxy/internal/notif"
+	"github.com/yusing/go-proxy/internal/task"
+	U "github.com/yusing/go-proxy/internal/utils"
+	"github.com/yusing/go-proxy/internal/watcher/health"
+)
+
+type (
+	HealthCheckFunc func() (healthy bool, detail string, err error)
+	monitor         struct {
+		service string
+		config  *health.HealthCheckConfig
+		url     U.AtomicValue[types.URL]
+
+		status      U.AtomicValue[health.Status]
+		checkHealth HealthCheckFunc
+		startTime   time.Time
+
+		metric *metrics.Gauge
+
+		task task.Task
+	}
+)
+
+var ErrNegativeInterval = errors.New("negative interval")
+
+func newMonitor(url types.URL, config *health.HealthCheckConfig, healthCheckFunc HealthCheckFunc) *monitor {
+	mon := &monitor{
+		config:      config,
+		checkHealth: healthCheckFunc,
+		startTime:   time.Now(),
+	}
+	mon.url.Store(url)
+	mon.status.Store(health.StatusHealthy)
+	return mon
+}
+
+func (mon *monitor) ContextWithTimeout(cause string) (ctx context.Context, cancel context.CancelFunc) {
+	if mon.task != nil {
+		return context.WithTimeoutCause(mon.task.Context(), mon.config.Timeout, errors.New(cause))
+	}
+	return context.WithTimeoutCause(context.Background(), mon.config.Timeout, errors.New(cause))
+}
+
+// Start implements task.TaskStarter.
+func (mon *monitor) Start(routeSubtask task.Task) E.Error {
+	mon.service = routeSubtask.Parent().Name()
+	mon.task = routeSubtask
+
+	if mon.config.Interval <= 0 {
+		return E.From(ErrNegativeInterval)
+	}
+
+	if common.PrometheusEnabled {
+		mon.metric = metrics.GetServiceMetrics().HealthStatus.With(metrics.HealthMetricLabels(mon.service))
+	}
+
+	go func() {
+		logger := logging.With().Str("name", mon.service).Logger()
+
+		defer func() {
+			if mon.status.Load() != health.StatusError {
+				mon.status.Store(health.StatusUnknown)
+			}
+			mon.task.Finish(nil)
+			if mon.metric != nil {
+				prometheus.Unregister(mon.metric)
+			}
+		}()
+
+		if err := mon.checkUpdateHealth(); err != nil {
+			logger.Err(err).Msg("healthchecker failure")
+			return
+		}
+
+		ticker := time.NewTicker(mon.config.Interval)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-mon.task.Context().Done():
+				return
+			case <-ticker.C:
+				err := mon.checkUpdateHealth()
+				if err != nil {
+					logger.Err(err).Msg("healthchecker failure")
+					return
+				}
+			}
+		}
+	}()
+	return nil
+}
+
+// Finish implements task.TaskFinisher.
+func (mon *monitor) Finish(reason any) {
+	mon.task.Finish(reason)
+}
+
+// UpdateURL implements HealthChecker.
+func (mon *monitor) UpdateURL(url types.URL) {
+	mon.url.Store(url)
+}
+
+// URL implements HealthChecker.
+func (mon *monitor) URL() types.URL {
+	return mon.url.Load()
+}
+
+// Config implements HealthChecker.
+func (mon *monitor) Config() *health.HealthCheckConfig {
+	return mon.config
+}
+
+// Status implements HealthMonitor.
+func (mon *monitor) Status() health.Status {
+	return mon.status.Load()
+}
+
+// Uptime implements HealthMonitor.
+func (mon *monitor) Uptime() time.Duration {
+	return time.Since(mon.startTime)
+}
+
+// Name implements HealthMonitor.
+func (mon *monitor) Name() string {
+	parts := strings.Split(mon.service, "/")
+	return parts[len(parts)-1]
+}
+
+// String implements fmt.Stringer of HealthMonitor.
+func (mon *monitor) String() string {
+	return mon.Name()
+}
+
+// MarshalJSON implements json.Marshaler of HealthMonitor.
+func (mon *monitor) MarshalJSON() ([]byte, error) {
+	return (&JSONRepresentation{
+		Name:    mon.service,
+		Config:  mon.config,
+		Status:  mon.status.Load(),
+		Started: mon.startTime,
+		Uptime:  mon.Uptime(),
+		URL:     mon.url.Load(),
+	}).MarshalJSON()
+}
+
+func (mon *monitor) checkUpdateHealth() error {
+	logger := logging.With().Str("name", mon.Name()).Logger()
+	healthy, detail, err := mon.checkHealth()
+	if err != nil {
+		defer mon.task.Finish(err)
+		mon.status.Store(health.StatusError)
+		if !errors.Is(err, context.Canceled) {
+			return fmt.Errorf("check health: %w", err)
+		}
+		return nil
+	}
+	var status health.Status
+	if healthy {
+		status = health.StatusHealthy
+	} else {
+		status = health.StatusUnhealthy
+	}
+	if healthy != (mon.status.Swap(status) == health.StatusHealthy) {
+		if healthy {
+			logger.Info().Msg("server is up")
+			notif.Notify(mon.service, "server is up")
+		} else {
+			logger.Warn().Msg("server is down")
+			logger.Debug().Msg(detail)
+			notif.Notify(mon.service, "server is down")
+		}
+	}
+	if mon.metric != nil {
+		mon.metric.Set(float64(status))
+	}
+
+	return nil
+}

internal/watcher/health/monitor/raw.go

@@ -0,0 +1,45 @@
+package monitor
+
+import (
+	"net"
+
+	"github.com/yusing/go-proxy/internal/net/types"
+	"github.com/yusing/go-proxy/internal/watcher/health"
+)
+
+type (
+	RawHealthMonitor struct {
+		*monitor
+		dialer *net.Dialer
+	}
+)
+
+func NewRawHealthMonitor(url types.URL, config *health.HealthCheckConfig) *RawHealthMonitor {
+	mon := new(RawHealthMonitor)
+	mon.monitor = newMonitor(url, config, mon.CheckHealth)
+	mon.dialer = &net.Dialer{
+		Timeout:       config.Timeout,
+		FallbackDelay: -1,
+	}
+	return mon
+}
+
+func NewRawHealthChecker(url types.URL, config *health.HealthCheckConfig) health.HealthChecker {
+	return NewRawHealthMonitor(url, config)
+}
+
+func (mon *RawHealthMonitor) CheckHealth() (healthy bool, detail string, err error) {
+	ctx, cancel := mon.ContextWithTimeout("ping request timed out")
+	defer cancel()
+
+	url := mon.url.Load()
+	conn, dialErr := mon.dialer.DialContext(ctx, url.Scheme, url.Host)
+	if dialErr != nil {
+		detail = dialErr.Error()
+		/* trunk-ignore(golangci-lint/nilerr) */
+		return
+	}
+	conn.Close()
+	healthy = true
+	return
+}