feat: agent as docker provider, drop / reload routes when docker connection state changed, refactor

internal/watcher/docker_watcher.go

@@ -2,12 +2,15 @@ package watcher
 
 import (
 	"context"
+	"errors"
 	"time"
 
 	docker_events "github.com/docker/docker/api/types/events"
 	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/client"
 	"github.com/yusing/go-proxy/internal/docker"
 	"github.com/yusing/go-proxy/internal/gperr"
+	"github.com/yusing/go-proxy/internal/logging"
 	"github.com/yusing/go-proxy/internal/watcher/events"
 )
 
@@ -41,72 +44,110 @@ var (
 	)}
 
 	dockerWatcherRetryInterval = 3 * time.Second
+
+	reloadTrigger = Event{
+		Type:            events.EventTypeDocker,
+		Action:          events.ActionForceReload,
+		ActorAttributes: map[string]string{},
+		ActorName:       "",
+		ActorID:         "",
+	}
 )
 
 func DockerFilterContainerNameID(nameOrID string) filters.KeyValuePair {
 	return filters.Arg("container", nameOrID)
 }
 
-func NewDockerWatcher(host string) DockerWatcher {
-	return DockerWatcher{host: host}
+func NewDockerWatcher(host string) *DockerWatcher {
+	return &DockerWatcher{host: host}
 }
 
 func (w *DockerWatcher) Events(ctx context.Context) (<-chan Event, <-chan gperr.Error) {
 	return w.EventsWithOptions(ctx, optionsDefault)
 }
 
+func (w DockerWatcher) parseError(err error) gperr.Error {
+	if errors.Is(err, context.DeadlineExceeded) {
+		return gperr.New("docker client connection timeout")
+	}
+	if client.IsErrConnectionFailed(err) {
+		return gperr.New("docker client connection failure")
+	}
+	return gperr.Wrap(err)
+}
+
+func (w *DockerWatcher) checkConnection(ctx context.Context) bool {
+	ctx, cancel := context.WithTimeout(ctx, dockerWatcherRetryInterval)
+	defer cancel()
+	err := w.client.CheckConnection(ctx)
+	if err != nil {
+		logging.Debug().Err(err).Msg("docker watcher: connection failed")
+		return false
+	}
+	return true
+}
+
+func (w *DockerWatcher) handleEvent(event docker_events.Message, ch chan<- Event) {
+	action, ok := events.DockerEventMap[event.Action]
+	if !ok {
+		return
+	}
+	ch <- Event{
+		Type:            events.EventTypeDocker,
+		ActorID:         event.Actor.ID,
+		ActorAttributes: event.Actor.Attributes, // labels
+		ActorName:       event.Actor.Attributes["name"],
+		Action:          action,
+	}
+}
+
 func (w *DockerWatcher) EventsWithOptions(ctx context.Context, options DockerListOptions) (<-chan Event, <-chan gperr.Error) {
 	eventCh := make(chan Event)
 	errCh := make(chan gperr.Error)
 
 	go func() {
+		var err error
+		w.client, err = docker.NewClient(w.host)
+		if err != nil {
+			errCh <- gperr.Wrap(err, "docker watcher: failed to initialize client")
+			return
+		}
+
 		defer func() {
-			defer close(eventCh)
-			defer close(errCh)
+			close(eventCh)
+			close(errCh)
 			w.client.Close()
 		}()
 
-		client, err := docker.ConnectClient(w.host)
-		if err != nil {
-			errCh <- E.From(err)
-			return
-		}
-		w.client = client
-
 		cEventCh, cErrCh := w.client.Events(ctx, options)
-
+		defer logging.Debug().Str("host", w.client.Address()).Msg("docker watcher closed")
 		for {
 			select {
 			case <-ctx.Done():
-				if err := E.From(ctx.Err()); err != nil && !err.Is(context.Canceled) {
-					errCh <- err
-				}
 				return
 			case msg := <-cEventCh:
-				action, ok := events.DockerEventMap[msg.Action]
-				if !ok {
-					continue
-				}
-				event := Event{
-					Type:            events.EventTypeDocker,
-					ActorID:         msg.Actor.ID,
-					ActorAttributes: msg.Actor.Attributes, // labels
-					ActorName:       msg.Actor.Attributes["name"],
-					Action:          action,
-				}
-				eventCh <- event
+				w.handleEvent(msg, eventCh)
 			case err := <-cErrCh:
 				if err == nil {
 					continue
 				}
-				errCh <- E.From(err)
-				select {
-				case <-ctx.Done():
-					return
-				default:
-					time.Sleep(dockerWatcherRetryInterval)
-					cEventCh, cErrCh = w.client.Events(ctx, options)
+				errCh <- w.parseError(err)
+				// release the error because reopening event channel may block
+				err = nil
+				// trigger reload (clear routes)
+				eventCh <- reloadTrigger
+				for !w.checkConnection(ctx) {
+					select {
+					case <-ctx.Done():
+						return
+					case <-time.After(dockerWatcherRetryInterval):
+						continue
+					}
 				}
+				// connection successful, trigger reload (reload routes)
+				eventCh <- reloadTrigger
+				// reopen event channel
+				cEventCh, cErrCh = w.client.Events(ctx, options)
 			}
 		}
 	}()