refactored some stuff, added healthcheck support, fixed 'include file' reload not showing in log

internal/watcher/directory_watcher.go

@@ -26,6 +26,14 @@ type DirWatcher struct {
 	ctx context.Context
 }
 
+// NewDirectoryWatcher returns a DirWatcher instance.
+//
+// The DirWatcher watches the given directory for file system events.
+// Currently, only events on files directly in the given directory are watched, not
+// recursively.
+//
+// Note that the returned DirWatcher is not ready to use until the goroutine
+// started by NewDirectoryWatcher has finished.
 func NewDirectoryWatcher(ctx context.Context, dirPath string) *DirWatcher {
 	//! subdirectories are not watched
 	w, err := fsnotify.NewWatcher()
@@ -70,16 +78,8 @@ func (h *DirWatcher) Add(relPath string) Watcher {
 			close(s.eventCh)
 			close(s.errCh)
 		}()
-		for {
-			select {
-			case <-h.ctx.Done():
-				return
-			case _, ok := <-h.eventCh:
-				if !ok { // directory watcher closed
-					return
-				}
-			}
-		}
+		<-h.ctx.Done()
+		logrus.Debugf("file watcher %s stopped", relPath)
 	}()
 	h.fwMap.Store(relPath, s)
 	return s
@@ -88,6 +88,7 @@ func (h *DirWatcher) Add(relPath string) Watcher {
 func (h *DirWatcher) start() {
 	defer close(h.eventCh)
 	defer h.w.Close()
+	defer logrus.Debugf("directory watcher %s stopped", h.dir)
 
 	for {
 		select {
@@ -121,7 +122,9 @@ func (h *DirWatcher) start() {
 			// send event to directory watcher
 			select {
 			case h.eventCh <- msg:
+				logrus.Debugf("sent event to directory watcher %s", h.dir)
 			default:
+				logrus.Debugf("failed to send event to directory watcher %s", h.dir)
 			}
 
 			// send event to file watcher too
@@ -129,8 +132,12 @@ func (h *DirWatcher) start() {
 			if ok {
 				select {
 				case w.eventCh <- msg:
+					logrus.Debugf("sent event to file watcher %s", relPath)
 				default:
+					logrus.Debugf("failed to send event to file watcher %s", relPath)
 				}
+			} else {
+				logrus.Debugf("file watcher not found: %s", relPath)
 			}
 		case err := <-h.w.Errors:
 			if errors.Is(err, fsnotify.ErrClosed) {

internal/watcher/health/healthcheck_config.go

@@ -0,0 +1,22 @@
+package health
+
+import (
+	"time"
+
+	"github.com/yusing/go-proxy/internal/common"
+)
+
+type HealthCheckConfig struct {
+	Disabled bool          `json:"disabled" yaml:"disabled"`
+	Path     string        `json:"path" yaml:"path"`
+	UseGet   bool          `json:"use_get" yaml:"use_get"`
+	Interval time.Duration `json:"interval" yaml:"interval"`
+	Timeout  time.Duration `json:"timeout" yaml:"timeout"`
+}
+
+func DefaultHealthCheckConfig() HealthCheckConfig {
+	return HealthCheckConfig{
+		Interval: common.HealthCheckIntervalDefault,
+		Timeout:  common.HealthCheckTimeoutDefault,
+	}
+}

internal/watcher/health/http.go

@@ -0,0 +1,63 @@
+package health
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"net/http"
+
+	"github.com/yusing/go-proxy/internal/net/types"
+)
+
+type HTTPHealthMonitor struct {
+	*monitor
+	method string
+	pinger *http.Client
+}
+
+func NewHTTPHealthMonitor(ctx context.Context, name string, url types.URL, config HealthCheckConfig) HealthMonitor {
+	mon := new(HTTPHealthMonitor)
+	mon.monitor = newMonitor(ctx, name, url, &config, mon.checkHealth)
+	mon.pinger = &http.Client{Timeout: config.Timeout}
+	if config.UseGet {
+		mon.method = http.MethodGet
+	} else {
+		mon.method = http.MethodHead
+	}
+	return mon
+}
+
+func (mon *HTTPHealthMonitor) checkHealth() (healthy bool, detail string, err error) {
+	req, reqErr := http.NewRequestWithContext(
+		mon.ctx,
+		mon.method,
+		mon.URL.String(),
+		nil,
+	)
+	if reqErr != nil {
+		err = reqErr
+		return
+	}
+	req.Header.Set("Connection", "close")
+
+	resp, respErr := mon.pinger.Do(req)
+	if respErr == nil {
+		resp.Body.Close()
+	}
+
+	switch {
+	case respErr != nil:
+		// treat tls error as healthy
+		var tlsErr *tls.CertificateVerificationError
+		if ok := errors.As(respErr, &tlsErr); !ok {
+			detail = respErr.Error()
+			return
+		}
+	case resp.StatusCode == http.StatusServiceUnavailable:
+		detail = resp.Status
+		return
+	}
+
+	healthy = true
+	return
+}

internal/watcher/health/logger.go

@@ -0,0 +1,5 @@
+package health
+
+import "github.com/sirupsen/logrus"
+
+var logger = logrus.WithField("module", "health_mon")

internal/watcher/health/monitor.go

@@ -0,0 +1,139 @@
+package health
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/yusing/go-proxy/internal/net/types"
+	F "github.com/yusing/go-proxy/internal/utils/functional"
+)
+
+type (
+	HealthMonitor interface {
+		Start()
+		Stop()
+		IsHealthy() bool
+		String() string
+	}
+	HealthCheckFunc func() (healthy bool, detail string, err error)
+	monitor         struct {
+		Name     string
+		URL      types.URL
+		Interval time.Duration
+
+		healthy     atomic.Bool
+		checkHealth HealthCheckFunc
+
+		ctx    context.Context
+		cancel context.CancelFunc
+		done   chan struct{}
+
+		mu sync.Mutex
+	}
+)
+
+var monMap = F.NewMapOf[string, HealthMonitor]()
+
+func newMonitor(parentCtx context.Context, name string, url types.URL, config *HealthCheckConfig, healthCheckFunc HealthCheckFunc) *monitor {
+	if parentCtx == nil {
+		parentCtx = context.Background()
+	}
+	ctx, cancel := context.WithCancel(parentCtx)
+	mon := &monitor{
+		Name:        name,
+		URL:         url.JoinPath(config.Path),
+		Interval:    config.Interval,
+		checkHealth: healthCheckFunc,
+		ctx:         ctx,
+		cancel:      cancel,
+		done:        make(chan struct{}),
+	}
+	mon.healthy.Store(true)
+	monMap.Store(name, mon)
+	return mon
+}
+
+func IsHealthy(name string) (healthy bool, ok bool) {
+	mon, ok := monMap.Load(name)
+	if !ok {
+		return
+	}
+	return mon.IsHealthy(), true
+}
+
+func (mon *monitor) Start() {
+	go func() {
+		defer close(mon.done)
+
+		ok := mon.checkUpdateHealth()
+		if !ok {
+			return
+		}
+
+		ticker := time.NewTicker(mon.Interval)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-mon.ctx.Done():
+				return
+			case <-ticker.C:
+				ok = mon.checkUpdateHealth()
+				if !ok {
+					return
+				}
+			}
+		}
+	}()
+	logger.Debugf("health monitor %q started", mon)
+}
+
+func (mon *monitor) Stop() {
+	defer logger.Debugf("health monitor %q stopped", mon)
+
+	monMap.Delete(mon.Name)
+
+	mon.mu.Lock()
+	defer mon.mu.Unlock()
+
+	if mon.cancel == nil {
+		return
+	}
+
+	mon.cancel()
+	<-mon.done
+
+	mon.cancel = nil
+}
+
+func (mon *monitor) IsHealthy() bool {
+	return mon.healthy.Load()
+}
+
+func (mon *monitor) String() string {
+	return mon.Name
+}
+
+func (mon *monitor) checkUpdateHealth() (hasError bool) {
+	healthy, detail, err := mon.checkHealth()
+	if err != nil {
+		mon.healthy.Store(false)
+		if !errors.Is(err, context.Canceled) {
+			logger.Errorf("server %q failed to check health: %s", mon, err)
+		}
+		mon.Stop()
+		return false
+	}
+	if healthy != mon.healthy.Swap(healthy) {
+		if healthy {
+			logger.Infof("server %q is up", mon)
+		} else {
+			logger.Warnf("server %q is down: %s", mon, detail)
+		}
+	}
+
+	return true
+}

internal/watcher/health/raw.go

@@ -0,0 +1,37 @@
+package health
+
+import (
+	"context"
+	"net"
+
+	"github.com/yusing/go-proxy/internal/net/types"
+)
+
+type (
+	RawHealthMonitor struct {
+		*monitor
+		dialer *net.Dialer
+	}
+)
+
+func NewRawHealthMonitor(ctx context.Context, name string, url types.URL, config HealthCheckConfig) HealthMonitor {
+	mon := new(RawHealthMonitor)
+	mon.monitor = newMonitor(ctx, name, url, &config, mon.checkAvail)
+	mon.dialer = &net.Dialer{
+		Timeout:       config.Timeout,
+		FallbackDelay: -1,
+	}
+	return mon
+}
+
+func (mon *RawHealthMonitor) checkAvail() (avail bool, detail string, err error) {
+	conn, dialErr := mon.dialer.DialContext(mon.ctx, mon.URL.Scheme, mon.URL.Host)
+	if dialErr != nil {
+		detail = dialErr.Error()
+		/* trunk-ignore(golangci-lint/nilerr) */
+		return
+	}
+	conn.Close()
+	avail = true
+	return
+}