Fixed a few issues:

- Incorrect name being shown on dashboard "Proxies page" - Apps being shown when homepage.show is false - Load balanced routes are shown on homepage instead of the load balancer - Route with idlewatcher will now be removed on container destroy - Idlewatcher panic - Performance improvement - Idlewatcher infinitely loading - Reload stucked / not working properly - Streams stuck on shutdown / reload - etc... Added: - support idlewatcher for loadbalanced routes - partial implementation for stream type idlewatcher Issues: - graceful shutdown
2026-04-09 18:33:36 +02:00 · 2024-10-18 16:47:01 +08:00
parent c0c61709ca
commit 53557e38b6
69 changed files with 2368 additions and 1654 deletions
--- a/internal/docker/client.go
+++ b/internal/docker/client.go
@@ -9,6 +9,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/yusing/go-proxy/internal/common"
 	E "github.com/yusing/go-proxy/internal/error"
+	"github.com/yusing/go-proxy/internal/task"
 	U "github.com/yusing/go-proxy/internal/utils"
 	F "github.com/yusing/go-proxy/internal/utils/functional"
 )
@@ -36,22 +37,13 @@ var (
 )

 func init() {
-	go func() {
-		task := common.NewTask("close all docker client")
-		defer task.Finished()
-		for {
-			select {
-			case <-task.Context().Done():
-				clientMap.RangeAllParallel(func(_ string, c Client) {
-					if c.Connected() {
-						c.Client.Close()
-					}
-				})
-				clientMap.Clear()
-				return
+	task.GlobalTask("close docker clients").OnComplete("", func() {
+		clientMap.RangeAllParallel(func(_ string, c Client) {
+			if c.Connected() {
+				c.Client.Close()
 			}
-		}
-	}()
+		})
+	})
 }

 func (c *SharedClient) Connected() bool {
@@ -141,19 +133,10 @@ func ConnectClient(host string) (Client, E.NestedError) {
 		<-c.refCount.Zero()
 		clientMap.Delete(c.key)

-		if c.Client != nil {
+		if c.Connected() {
 			c.Client.Close()
-			c.Client = nil
 			c.l.Debugf("client closed")
 		}
 	}()
 	return c, nil
 }
-
-func CloseAllClients() {
-	clientMap.RangeAllParallel(func(_ string, c Client) {
-		c.Client.Close()
-	})
-	clientMap.Clear()
-	logger.Debug("closed all clients")
-}
--- a/internal/docker/client_info.go
+++ b/internal/docker/client_info.go
@@ -2,6 +2,7 @@ package docker

 import (
 	"context"
+	"errors"
 	"time"

 	"github.com/docker/docker/api/types"
@@ -16,10 +17,13 @@ type ClientInfo struct {
 }

 var listOptions = container.ListOptions{
+	// created|restarting|running|removing|paused|exited|dead
 	// Filters: filters.NewArgs(
-	// 	filters.Arg("health", "healthy"),
-	// 	filters.Arg("health", "none"),
-	// 	filters.Arg("health", "starting"),
+	// 	filters.Arg("status", "created"),
+	// 	filters.Arg("status", "restarting"),
+	// 	filters.Arg("status", "running"),
+	// 	filters.Arg("status", "paused"),
+	// 	filters.Arg("status", "exited"),
 	// ),
 	All: true,
 }
@@ -31,7 +35,7 @@ func GetClientInfo(clientHost string, getContainer bool) (*ClientInfo, E.NestedE
 	}
 	defer dockerClient.Close()

-	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	ctx, cancel := context.WithTimeoutCause(context.Background(), 3*time.Second, errors.New("docker client connection timeout"))
 	defer cancel()

 	var containers []types.Container
--- a/internal/docker/container.go
+++ b/internal/docker/container.go
@@ -32,11 +32,11 @@ type (
 		IsExcluded  bool     `json:"is_excluded" yaml:"-"`
 		IsExplicit  bool     `json:"is_explicit" yaml:"-"`
 		IsDatabase  bool     `json:"is_database" yaml:"-"`
-		IdleTimeout string   `json:"idle_timeout" yaml:"-"`
-		WakeTimeout string   `json:"wake_timeout" yaml:"-"`
-		StopMethod  string   `json:"stop_method" yaml:"-"`
-		StopTimeout string   `json:"stop_timeout" yaml:"-"` // stop_method = "stop" only
-		StopSignal  string   `json:"stop_signal" yaml:"-"`  // stop_method = "stop" | "kill" only
+		IdleTimeout string   `json:"idle_timeout,omitempty" yaml:"-"`
+		WakeTimeout string   `json:"wake_timeout,omitempty" yaml:"-"`
+		StopMethod  string   `json:"stop_method,omitempty" yaml:"-"`
+		StopTimeout string   `json:"stop_timeout,omitempty" yaml:"-"` // stop_method = "stop" only
+		StopSignal  string   `json:"stop_signal,omitempty" yaml:"-"`  // stop_method = "stop" | "kill" only
 		Running     bool     `json:"running" yaml:"-"`
 	}
 )
--- a/internal/docker/idlewatcher/config/config.go
+++ b/internal/docker/idlewatcher/config/config.go
@@ -0,0 +1,112 @@
+package idlewatcher
+
+import (
+	"time"
+
+	"github.com/yusing/go-proxy/internal/docker"
+	E "github.com/yusing/go-proxy/internal/error"
+)
+
+type (
+	Config struct {
+		IdleTimeout time.Duration `json:"idle_timeout,omitempty"`
+		WakeTimeout time.Duration `json:"wake_timeout,omitempty"`
+		StopTimeout int           `json:"stop_timeout,omitempty"` // docker api takes integer seconds for timeout argument
+		StopMethod  StopMethod    `json:"stop_method,omitempty"`
+		StopSignal  Signal        `json:"stop_signal,omitempty"`
+
+		DockerHost       string `json:"docker_host,omitempty"`
+		ContainerName    string `json:"container_name,omitempty"`
+		ContainerID      string `json:"container_id,omitempty"`
+		ContainerRunning bool   `json:"container_running,omitempty"`
+	}
+	StopMethod string
+	Signal     string
+)
+
+const (
+	StopMethodPause StopMethod = "pause"
+	StopMethodStop  StopMethod = "stop"
+	StopMethodKill  StopMethod = "kill"
+)
+
+func ValidateConfig(cont *docker.Container) (cfg *Config, res E.NestedError) {
+	if cont == nil {
+		return nil, nil
+	}
+
+	if cont.IdleTimeout == "" {
+		return &Config{
+			DockerHost:       cont.DockerHost,
+			ContainerName:    cont.ContainerName,
+			ContainerID:      cont.ContainerID,
+			ContainerRunning: cont.Running,
+		}, nil
+	}
+
+	b := E.NewBuilder("invalid idlewatcher config")
+	defer b.To(&res)
+
+	idleTimeout, err := validateDurationPostitive(cont.IdleTimeout)
+	b.Add(err.Subjectf("%s", "idle_timeout"))
+
+	wakeTimeout, err := validateDurationPostitive(cont.WakeTimeout)
+	b.Add(err.Subjectf("%s", "wake_timeout"))
+
+	stopTimeout, err := validateDurationPostitive(cont.StopTimeout)
+	b.Add(err.Subjectf("%s", "stop_timeout"))
+
+	stopMethod, err := validateStopMethod(cont.StopMethod)
+	b.Add(err)
+
+	signal, err := validateSignal(cont.StopSignal)
+	b.Add(err)
+
+	if err := b.Build(); err != nil {
+		return
+	}
+
+	return &Config{
+		IdleTimeout: idleTimeout,
+		WakeTimeout: wakeTimeout,
+		StopTimeout: int(stopTimeout.Seconds()),
+		StopMethod:  stopMethod,
+		StopSignal:  signal,
+
+		DockerHost:       cont.DockerHost,
+		ContainerName:    cont.ContainerName,
+		ContainerID:      cont.ContainerID,
+		ContainerRunning: cont.Running,
+	}, nil
+}
+
+func validateDurationPostitive(value string) (time.Duration, E.NestedError) {
+	d, err := time.ParseDuration(value)
+	if err != nil {
+		return 0, E.Invalid("duration", value).With(err)
+	}
+	if d < 0 {
+		return 0, E.Invalid("duration", "negative value")
+	}
+	return d, nil
+}
+
+func validateSignal(s string) (Signal, E.NestedError) {
+	switch s {
+	case "", "SIGINT", "SIGTERM", "SIGHUP", "SIGQUIT",
+		"INT", "TERM", "HUP", "QUIT":
+		return Signal(s), nil
+	}
+
+	return "", E.Invalid("signal", s)
+}
+
+func validateStopMethod(s string) (StopMethod, E.NestedError) {
+	sm := StopMethod(s)
+	switch sm {
+	case StopMethodPause, StopMethodStop, StopMethodKill:
+		return sm, nil
+	default:
+		return "", E.Invalid("stop_method", sm)
+	}
+}
--- a/internal/docker/idlewatcher/loading_page.go
+++ b/internal/docker/idlewatcher/loading_page.go
@@ -20,16 +20,15 @@ var loadingPageTmpl = template.Must(template.New("loading_page").Parse(string(lo

 const headerCheckRedirect = "X-Goproxy-Check-Redirect"

-func (w *Watcher) makeRespBody(format string, args ...any) []byte {
-	msg := fmt.Sprintf(format, args...)
+func (w *Watcher) makeLoadingPageBody() []byte {
+	msg := fmt.Sprintf("%s is starting...", w.ContainerName)

 	data := new(templateData)
 	data.CheckRedirectHeader = headerCheckRedirect
 	data.Title = w.ContainerName
-	data.Message = strings.ReplaceAll(msg, "\n", "<br>")
-	data.Message = strings.ReplaceAll(data.Message, " ", "&ensp;")
+	data.Message = strings.ReplaceAll(msg, " ", "&ensp;")

-	buf := bytes.NewBuffer(make([]byte, 128)) // more than enough
+	buf := bytes.NewBuffer(make([]byte, len(loadingPage)+len(data.Title)+len(data.Message)+len(headerCheckRedirect)))
 	err := loadingPageTmpl.Execute(buf, data)
 	if err != nil { // should never happen in production
 		panic(err)
--- a/internal/docker/idlewatcher/waker.go
+++ b/internal/docker/idlewatcher/waker.go
@@ -1,197 +1,133 @@
 package idlewatcher

 import (
-	"context"
 	"net/http"
-	"strconv"
+	"sync/atomic"
 	"time"

-	"github.com/sirupsen/logrus"
 	E "github.com/yusing/go-proxy/internal/error"
 	gphttp "github.com/yusing/go-proxy/internal/net/http"
-	"github.com/yusing/go-proxy/internal/net/types"
+	net "github.com/yusing/go-proxy/internal/net/types"
+	"github.com/yusing/go-proxy/internal/proxy/entry"
+	"github.com/yusing/go-proxy/internal/task"
+	U "github.com/yusing/go-proxy/internal/utils"
 	"github.com/yusing/go-proxy/internal/watcher/health"
 )

-type Waker struct {
-	*Watcher
+type Waker interface {
+	health.HealthMonitor
+	http.Handler
+	net.Stream
+}
+
+type waker struct {
+	_ U.NoCopy

-	client *http.Client
 	rp     *gphttp.ReverseProxy
+	stream net.Stream
+	hc     health.HealthChecker
+
+	ready atomic.Bool
 }

-func NewWaker(w *Watcher, rp *gphttp.ReverseProxy) *Waker {
-	return &Waker{
-		Watcher: w,
-		client: &http.Client{
-			Timeout:   1 * time.Second,
-			Transport: rp.Transport,
-		},
-		rp: rp,
+const (
+	idleWakerCheckInterval = 100 * time.Millisecond
+	idleWakerCheckTimeout  = time.Second
+)
+
+// TODO: support stream
+
+func newWaker(providerSubTask task.Task, entry entry.Entry, rp *gphttp.ReverseProxy, stream net.Stream) (Waker, E.NestedError) {
+	hcCfg := entry.HealthCheckConfig()
+	hcCfg.Timeout = idleWakerCheckTimeout
+
+	waker := &waker{
+		rp:     rp,
+		stream: stream,
 	}
-}

-func (w *Waker) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
-	shouldNext := w.wake(rw, r)
-	if !shouldNext {
-		return
+	watcher, err := registerWatcher(providerSubTask, entry, waker)
+	if err != nil {
+		return nil, err
 	}
-	w.rp.ServeHTTP(rw, r)
+
+	if rp != nil {
+		waker.hc = health.NewHTTPHealthChecker(entry.TargetURL(), hcCfg, rp.Transport)
+	} else if stream != nil {
+		waker.hc = health.NewRawHealthChecker(entry.TargetURL(), hcCfg)
+	} else {
+		panic("both nil")
+	}
+	return watcher, nil
 }

-/* HealthMonitor interface */
-
-func (w *Waker) Start() {}
-
-func (w *Waker) Stop() {
-	w.Unregister()
+// lifetime should follow route provider
+func NewHTTPWaker(providerSubTask task.Task, entry entry.Entry, rp *gphttp.ReverseProxy) (Waker, E.NestedError) {
+	return newWaker(providerSubTask, entry, rp, nil)
 }

-func (w *Waker) UpdateConfig(config health.HealthCheckConfig) {
-	panic("use idlewatcher.Register instead")
+func NewStreamWaker(providerSubTask task.Task, entry entry.Entry, stream net.Stream) (Waker, E.NestedError) {
+	return newWaker(providerSubTask, entry, nil, stream)
 }

-func (w *Waker) Name() string {
+// Start implements health.HealthMonitor.
+func (w *Watcher) Start(routeSubTask task.Task) E.NestedError {
+	w.task.OnComplete("stop route", func() {
+		routeSubTask.Parent().Finish("watcher stopped")
+	})
+	return nil
+}
+
+// Finish implements health.HealthMonitor.
+func (w *Watcher) Finish(reason string) {}
+
+// Name implements health.HealthMonitor.
+func (w *Watcher) Name() string {
 	return w.String()
 }

-func (w *Waker) String() string {
-	return string(w.Alias)
+// String implements health.HealthMonitor.
+func (w *Watcher) String() string {
+	return w.ContainerName
 }

-func (w *Waker) Status() health.Status {
-	if w.ready.Load() {
-		return health.StatusHealthy
-	}
-	if !w.ContainerRunning {
-		return health.StatusNapping
-	}
-	return health.StatusStarting
-}
-
-func (w *Waker) Uptime() time.Duration {
+// Uptime implements health.HealthMonitor.
+func (w *Watcher) Uptime() time.Duration {
 	return 0
 }

-func (w *Waker) MarshalJSON() ([]byte, error) {
-	var url types.URL
-	if w.URL.String() != "http://:0" {
-		url = w.URL
+// Status implements health.HealthMonitor.
+func (w *Watcher) Status() health.Status {
+	if !w.ContainerRunning {
+		return health.StatusNapping
+	}
+
+	if w.ready.Load() {
+		return health.StatusHealthy
+	}
+
+	healthy, _, err := w.hc.CheckHealth()
+	switch {
+	case err != nil:
+		return health.StatusError
+	case healthy:
+		w.ready.Store(true)
+		return health.StatusHealthy
+	default:
+		return health.StatusStarting
+	}
+}
+
+// MarshalJSON implements health.HealthMonitor.
+func (w *Watcher) MarshalJSON() ([]byte, error) {
+	var url net.URL
+	if w.hc.URL().Port() != "0" {
+		url = w.hc.URL()
 	}
 	return (&health.JSONRepresentation{
 		Name:   w.Name(),
 		Status: w.Status(),
-		Config: &health.HealthCheckConfig{
-			Interval: w.IdleTimeout,
-			Timeout:  w.WakeTimeout,
-		},
-		URL: url,
+		Config: w.hc.Config(),
+		URL:    url,
 	}).MarshalJSON()
 }
-
-/* End of HealthMonitor interface */
-
-func (w *Waker) wake(rw http.ResponseWriter, r *http.Request) (shouldNext bool) {
-	w.resetIdleTimer()
-
-	if r.Body != nil {
-		defer r.Body.Close()
-	}
-
-	// pass through if container is ready
-	if w.ready.Load() {
-		return true
-	}
-
-	ctx, cancel := context.WithTimeout(r.Context(), w.WakeTimeout)
-	defer cancel()
-
-	accept := gphttp.GetAccept(r.Header)
-	acceptHTML := (r.Method == http.MethodGet && accept.AcceptHTML() || r.RequestURI == "/" && accept.IsEmpty())
-
-	isCheckRedirect := r.Header.Get(headerCheckRedirect) != ""
-	if !isCheckRedirect && acceptHTML {
-		// Send a loading response to the client
-		body := w.makeRespBody("%s waking up...", w.ContainerName)
-		rw.Header().Set("Content-Type", "text/html; charset=utf-8")
-		rw.Header().Set("Content-Length", strconv.Itoa(len(body)))
-		rw.Header().Add("Cache-Control", "no-cache")
-		rw.Header().Add("Cache-Control", "no-store")
-		rw.Header().Add("Cache-Control", "must-revalidate")
-		if _, err := rw.Write(body); err != nil {
-			w.l.Errorf("error writing http response: %s", err)
-		}
-		return
-	}
-
-	select {
-	case <-w.task.Context().Done():
-		http.Error(rw, "Service unavailable", http.StatusServiceUnavailable)
-		return
-	case <-ctx.Done():
-		http.Error(rw, "Waking timed out", http.StatusGatewayTimeout)
-		return
-	default:
-	}
-
-	w.l.Debug("wake signal received")
-	err := w.wakeIfStopped()
-	if err != nil {
-		w.l.Error(E.FailWith("wake", err))
-		http.Error(rw, "Error waking container", http.StatusInternalServerError)
-		return
-	}
-
-	// maybe another request came in while we were waiting for the wake
-	if w.ready.Load() {
-		if isCheckRedirect {
-			rw.WriteHeader(http.StatusOK)
-			return
-		}
-		return true
-	}
-
-	for {
-		select {
-		case <-w.task.Context().Done():
-			http.Error(rw, "Service unavailable", http.StatusServiceUnavailable)
-			return
-		case <-ctx.Done():
-			http.Error(rw, "Waking timed out", http.StatusGatewayTimeout)
-			return
-		default:
-		}
-
-		wakeReq, err := http.NewRequestWithContext(
-			ctx,
-			http.MethodHead,
-			w.URL.String(),
-			nil,
-		)
-		if err != nil {
-			w.l.Errorf("new request err to %s: %s", r.URL, err)
-			http.Error(rw, "Internal server error", http.StatusInternalServerError)
-			return
-		}
-
-		wakeResp, err := w.client.Do(wakeReq)
-		if err == nil && wakeResp.StatusCode != http.StatusServiceUnavailable {
-			w.ready.Store(true)
-			w.l.Debug("awaken")
-			if isCheckRedirect {
-				rw.WriteHeader(http.StatusOK)
-				return
-			}
-			logrus.Infof("container %s is ready, passing through to %s", w.Alias, w.rp.TargetURL)
-			return true
-		}
-
-		// retry until the container is ready or timeout
-		time.Sleep(100 * time.Millisecond)
-	}
-}
-
-// static HealthMonitor interface check
-func (w *Waker) _() health.HealthMonitor {
-	return w
-}
--- a/internal/docker/idlewatcher/waker_http.go
+++ b/internal/docker/idlewatcher/waker_http.go
@@ -0,0 +1,105 @@
+package idlewatcher
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	E "github.com/yusing/go-proxy/internal/error"
+	gphttp "github.com/yusing/go-proxy/internal/net/http"
+	"github.com/yusing/go-proxy/internal/watcher/health"
+)
+
+// ServeHTTP implements http.Handler
+func (w *Watcher) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
+	shouldNext := w.wakeFromHTTP(rw, r)
+	if !shouldNext {
+		return
+	}
+	w.rp.ServeHTTP(rw, r)
+}
+
+func (w *Watcher) wakeFromHTTP(rw http.ResponseWriter, r *http.Request) (shouldNext bool) {
+	w.resetIdleTimer()
+
+	if r.Body != nil {
+		defer r.Body.Close()
+	}
+
+	// pass through if container is already ready
+	if w.ready.Load() {
+		return true
+	}
+
+	accept := gphttp.GetAccept(r.Header)
+	acceptHTML := (r.Method == http.MethodGet && accept.AcceptHTML() || r.RequestURI == "/" && accept.IsEmpty())
+
+	isCheckRedirect := r.Header.Get(headerCheckRedirect) != ""
+	if !isCheckRedirect && acceptHTML {
+		// Send a loading response to the client
+		body := w.makeLoadingPageBody()
+		rw.Header().Set("Content-Type", "text/html; charset=utf-8")
+		rw.Header().Set("Content-Length", strconv.Itoa(len(body)))
+		rw.Header().Add("Cache-Control", "no-cache")
+		rw.Header().Add("Cache-Control", "no-store")
+		rw.Header().Add("Cache-Control", "must-revalidate")
+		rw.Header().Add("Connection", "close")
+		if _, err := rw.Write(body); err != nil {
+			w.l.Errorf("error writing http response: %s", err)
+		}
+		return
+	}
+
+	ctx, cancel := context.WithTimeoutCause(r.Context(), w.WakeTimeout, errors.New("wake timeout"))
+	defer cancel()
+
+	checkCancelled := func() bool {
+		select {
+		case <-w.task.Context().Done():
+			w.l.Debugf("wake cancelled: %s", context.Cause(w.task.Context()))
+			http.Error(rw, "Service unavailable", http.StatusServiceUnavailable)
+			return true
+		case <-ctx.Done():
+			w.l.Debugf("wake cancelled: %s", context.Cause(ctx))
+			http.Error(rw, "Waking timed out", http.StatusGatewayTimeout)
+			return true
+		default:
+			return false
+		}
+	}
+
+	if checkCancelled() {
+		return false
+	}
+
+	w.l.Debug("wake signal received")
+	err := w.wakeIfStopped()
+	if err != nil {
+		w.l.Error(E.FailWith("wake", err))
+		http.Error(rw, "Error waking container", http.StatusInternalServerError)
+		return
+	}
+
+	for {
+		if checkCancelled() {
+			return false
+		}
+
+		if w.Status() == health.StatusHealthy {
+			w.resetIdleTimer()
+			if isCheckRedirect {
+				logrus.Debugf("container %s is ready, redirecting...", w.String())
+				rw.WriteHeader(http.StatusOK)
+				return
+			}
+			logrus.Infof("container %s is ready, passing through to %s", w.String(), w.hc.URL())
+			return true
+		}
+
+		// retry until the container is ready or timeout
+		time.Sleep(idleWakerCheckInterval)
+	}
+}
--- a/internal/docker/idlewatcher/waker_stream.go
+++ b/internal/docker/idlewatcher/waker_stream.go
@@ -0,0 +1,87 @@
+package idlewatcher
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	"github.com/yusing/go-proxy/internal/net/types"
+	"github.com/yusing/go-proxy/internal/watcher/health"
+)
+
+// Setup implements types.Stream.
+func (w *Watcher) Setup() error {
+	return w.stream.Setup()
+}
+
+// Accept implements types.Stream.
+func (w *Watcher) Accept() (conn types.StreamConn, err error) {
+	conn, err = w.stream.Accept()
+	// timeout means no connection is accepted
+	var nErr *net.OpError
+	ok := errors.As(err, &nErr)
+	if ok && nErr.Timeout() {
+		return
+	}
+	if err := w.wakeFromStream(); err != nil {
+		return nil, err
+	}
+	return w.stream.Accept()
+}
+
+// CloseListeners implements types.Stream.
+func (w *Watcher) CloseListeners() {
+	w.stream.CloseListeners()
+}
+
+// Handle implements types.Stream.
+func (w *Watcher) Handle(conn types.StreamConn) error {
+	if err := w.wakeFromStream(); err != nil {
+		return err
+	}
+	return w.stream.Handle(conn)
+}
+
+func (w *Watcher) wakeFromStream() error {
+	// pass through if container is already ready
+	if w.ready.Load() {
+		return nil
+	}
+
+	w.l.Debug("wake signal received")
+	wakeErr := w.wakeIfStopped()
+	if wakeErr != nil {
+		wakeErr = fmt.Errorf("wake failed with error: %w", wakeErr)
+		w.l.Error(wakeErr)
+		return wakeErr
+	}
+
+	ctx, cancel := context.WithTimeoutCause(w.task.Context(), w.WakeTimeout, errors.New("wake timeout"))
+	defer cancel()
+
+	for {
+		select {
+		case <-w.task.Context().Done():
+			cause := w.task.FinishCause()
+			w.l.Debugf("wake cancelled: %s", cause)
+			return cause
+		case <-ctx.Done():
+			cause := context.Cause(ctx)
+			w.l.Debugf("wake cancelled: %s", cause)
+			return cause
+		default:
+		}
+
+		if w.Status() == health.StatusHealthy {
+			w.resetIdleTimer()
+			logrus.Infof("container %s is ready, passing through to %s", w.String(), w.hc.URL())
+			return nil
+		}
+
+		// retry until the container is ready or timeout
+		time.Sleep(idleWakerCheckInterval)
+	}
+}
--- a/internal/docker/idlewatcher/watcher.go
+++ b/internal/docker/idlewatcher/watcher.go
@@ -2,191 +2,193 @@ package idlewatcher

 import (
 	"context"
+	"errors"
+	"fmt"
 	"sync"
-	"sync/atomic"
 	"time"

 	"github.com/docker/docker/api/types/container"
 	"github.com/sirupsen/logrus"
-	"github.com/yusing/go-proxy/internal/common"
 	D "github.com/yusing/go-proxy/internal/docker"
+	idlewatcher "github.com/yusing/go-proxy/internal/docker/idlewatcher/config"
 	E "github.com/yusing/go-proxy/internal/error"
-	P "github.com/yusing/go-proxy/internal/proxy"
-	PT "github.com/yusing/go-proxy/internal/proxy/fields"
+	"github.com/yusing/go-proxy/internal/proxy/entry"
+	"github.com/yusing/go-proxy/internal/task"
 	U "github.com/yusing/go-proxy/internal/utils"
 	F "github.com/yusing/go-proxy/internal/utils/functional"
+	"github.com/yusing/go-proxy/internal/watcher"
 	W "github.com/yusing/go-proxy/internal/watcher"
+	"github.com/yusing/go-proxy/internal/watcher/events"
 )

 type (
 	Watcher struct {
-		*P.ReverseProxyEntry
+		_ U.NoCopy

-		client D.Client
+		*idlewatcher.Config
+		*waker

-		ready        atomic.Bool  // whether the site is ready to accept connection
+		client       D.Client
 		stopByMethod StopCallback // send a docker command w.r.t. `stop_method`
-
-		ticker *time.Ticker
-
-		task   common.Task
-		cancel context.CancelFunc
-
-		refCount *U.RefCount
-
-		l logrus.FieldLogger
+		ticker       *time.Ticker
+		task         task.Task
+		l            *logrus.Entry
 	}

 	WakeDone     <-chan error
 	WakeFunc     func() WakeDone
-	StopCallback func() E.NestedError
+	StopCallback func() error
 )

 var (
 	watcherMap   = F.NewMapOf[string, *Watcher]()
 	watcherMapMu sync.Mutex

-	portHistoryMap = F.NewMapOf[PT.Alias, string]()
-
 	logger = logrus.WithField("module", "idle_watcher")
 )

-func Register(entry *P.ReverseProxyEntry) (*Watcher, E.NestedError) {
-	failure := E.Failure("idle_watcher register")
+const dockerReqTimeout = 3 * time.Second

-	if entry.IdleTimeout == 0 {
-		return nil, failure.With(E.Invalid("idle_timeout", 0))
+func registerWatcher(providerSubtask task.Task, entry entry.Entry, waker *waker) (*Watcher, E.NestedError) {
+	failure := E.Failure("idle_watcher register")
+	cfg := entry.IdlewatcherConfig()
+
+	if cfg.IdleTimeout == 0 {
+		panic("should not reach here")
 	}

 	watcherMapMu.Lock()
 	defer watcherMapMu.Unlock()

-	key := entry.ContainerID
-
-	if entry.URL.Port() != "0" {
-		portHistoryMap.Store(entry.Alias, entry.URL.Port())
-	}
+	key := cfg.ContainerID

 	if w, ok := watcherMap.Load(key); ok {
-		w.refCount.Add()
-		w.ReverseProxyEntry = entry
+		w.Config = cfg
+		w.waker = waker
+		w.resetIdleTimer()
 		return w, nil
 	}

-	client, err := D.ConnectClient(entry.DockerHost)
+	client, err := D.ConnectClient(cfg.DockerHost)
 	if err.HasError() {
 		return nil, failure.With(err)
 	}

 	w := &Watcher{
-		ReverseProxyEntry: entry,
-		client:            client,
-		refCount:          U.NewRefCounter(),
-		ticker:            time.NewTicker(entry.IdleTimeout),
-		l:                 logger.WithField("container", entry.ContainerName),
+		Config: cfg,
+		waker:  waker,
+		client: client,
+		task:   providerSubtask,
+		ticker: time.NewTicker(cfg.IdleTimeout),
+		l:      logger.WithField("container", cfg.ContainerName),
 	}
-	w.task, w.cancel = common.NewTaskWithCancel("Idlewatcher for %s", w.Alias)
 	w.stopByMethod = w.getStopCallback()
-
 	watcherMap.Store(key, w)

-	go w.watchUntilCancel()
+	go func() {
+		cause := w.watchUntilDestroy()
+
+		watcherMapMu.Lock()
+		watcherMap.Delete(w.ContainerID)
+		watcherMapMu.Unlock()
+
+		w.ticker.Stop()
+		w.client.Close()
+		w.task.Finish(cause.Error())
+	}()

 	return w, nil
 }

-func (w *Watcher) Unregister() {
-	w.refCount.Sub()
-}
-
-func (w *Watcher) containerStop() error {
-	return w.client.ContainerStop(w.task.Context(), w.ContainerID, container.StopOptions{
+func (w *Watcher) containerStop(ctx context.Context) error {
+	return w.client.ContainerStop(ctx, w.ContainerID, container.StopOptions{
 		Signal:  string(w.StopSignal),
 		Timeout: &w.StopTimeout,
 	})
 }

-func (w *Watcher) containerPause() error {
-	return w.client.ContainerPause(w.task.Context(), w.ContainerID)
+func (w *Watcher) containerPause(ctx context.Context) error {
+	return w.client.ContainerPause(ctx, w.ContainerID)
 }

-func (w *Watcher) containerKill() error {
-	return w.client.ContainerKill(w.task.Context(), w.ContainerID, string(w.StopSignal))
+func (w *Watcher) containerKill(ctx context.Context) error {
+	return w.client.ContainerKill(ctx, w.ContainerID, string(w.StopSignal))
 }

-func (w *Watcher) containerUnpause() error {
-	return w.client.ContainerUnpause(w.task.Context(), w.ContainerID)
+func (w *Watcher) containerUnpause(ctx context.Context) error {
+	return w.client.ContainerUnpause(ctx, w.ContainerID)
 }

-func (w *Watcher) containerStart() error {
-	return w.client.ContainerStart(w.task.Context(), w.ContainerID, container.StartOptions{})
+func (w *Watcher) containerStart(ctx context.Context) error {
+	return w.client.ContainerStart(ctx, w.ContainerID, container.StartOptions{})
 }

-func (w *Watcher) containerStatus() (string, E.NestedError) {
+func (w *Watcher) containerStatus() (string, error) {
 	if !w.client.Connected() {
-		return "", E.Failure("docker client closed")
+		return "", errors.New("docker client not connected")
 	}
-	json, err := w.client.ContainerInspect(w.task.Context(), w.ContainerID)
+	ctx, cancel := context.WithTimeoutCause(w.task.Context(), dockerReqTimeout, errors.New("docker request timeout"))
+	defer cancel()
+	json, err := w.client.ContainerInspect(ctx, w.ContainerID)
 	if err != nil {
-		return "", E.FailWith("inspect container", err)
+		return "", fmt.Errorf("failed to inspect container: %w", err)
 	}
 	return json.State.Status, nil
 }

-func (w *Watcher) wakeIfStopped() E.NestedError {
-	if w.ready.Load() || w.ContainerRunning {
+func (w *Watcher) wakeIfStopped() error {
+	if w.ContainerRunning {
 		return nil
 	}

 	status, err := w.containerStatus()
-
-	if err.HasError() {
+	if err != nil {
 		return err
 	}
-	// "created", "running", "paused", "restarting", "removing", "exited", or "dead"
+
+	ctx, cancel := context.WithTimeout(w.task.Context(), dockerReqTimeout)
+	defer cancel()
+
+	// !Hard coded here since theres no constants from Docker API
 	switch status {
 	case "exited", "dead":
-		return E.From(w.containerStart())
+		return w.containerStart(ctx)
 	case "paused":
-		return E.From(w.containerUnpause())
+		return w.containerUnpause(ctx)
 	case "running":
 		return nil
 	default:
-		return E.Unexpected("container state", status)
+		panic("should not reach here")
 	}
 }

 func (w *Watcher) getStopCallback() StopCallback {
-	var cb func() error
+	var cb func(context.Context) error
 	switch w.StopMethod {
-	case PT.StopMethodPause:
+	case idlewatcher.StopMethodPause:
 		cb = w.containerPause
-	case PT.StopMethodStop:
+	case idlewatcher.StopMethodStop:
 		cb = w.containerStop
-	case PT.StopMethodKill:
+	case idlewatcher.StopMethodKill:
 		cb = w.containerKill
 	default:
 		panic("should not reach here")
 	}
-	return func() E.NestedError {
-		status, err := w.containerStatus()
-		if err.HasError() {
-			return err
-		}
-		if status != "running" {
-			return nil
-		}
-		return E.From(cb())
+	return func() error {
+		ctx, cancel := context.WithTimeout(w.task.Context(), dockerReqTimeout)
+		defer cancel()
+		return cb(ctx)
 	}
 }

 func (w *Watcher) resetIdleTimer() {
+	w.l.Trace("reset idle timer")
 	w.ticker.Reset(w.IdleTimeout)
 }

-func (w *Watcher) watchUntilCancel() {
-	dockerWatcher := W.NewDockerWatcherWithClient(w.client)
-	dockerEventCh, dockerEventErrCh := dockerWatcher.EventsWithOptions(w.task.Context(), W.DockerListOptions{
+func (w *Watcher) getEventCh(dockerWatcher watcher.DockerWatcher) (eventTask task.Task, eventCh <-chan events.Event, errCh <-chan E.NestedError) {
+	eventTask = w.task.Subtask("watcher for %s", w.ContainerID)
+	eventCh, errCh = dockerWatcher.EventsWithOptions(eventTask.Context(), W.DockerListOptions{
 		Filters: W.NewDockerFilter(
 			W.DockerFilterContainer,
 			W.DockerrFilterContainer(w.ContainerID),
@@ -194,34 +196,47 @@ func (w *Watcher) watchUntilCancel() {
 			W.DockerFilterStop,
 			W.DockerFilterDie,
 			W.DockerFilterKill,
+			W.DockerFilterDestroy,
 			W.DockerFilterPause,
 			W.DockerFilterUnpause,
 		),
 	})
+	return
+}

-	defer func() {
-		w.cancel()
-		w.ticker.Stop()
-		w.client.Close()
-		watcherMap.Delete(w.ContainerID)
-		w.task.Finished()
-	}()
+// watchUntilDestroy waits for the container to be created, started, or unpaused,
+// and then reset the idle timer.
+//
+// When the container is stopped, paused,
+// or killed, the idle timer is stopped and the ContainerRunning flag is set to false.
+//
+// When the idle timer fires, the container is stopped according to the
+// stop method.
+//
+// it exits only if the context is canceled, the container is destroyed,
+// errors occured on docker client, or route provider died (mainly caused by config reload).
+func (w *Watcher) watchUntilDestroy() error {
+	dockerWatcher := W.NewDockerWatcherWithClient(w.client)
+	eventTask, dockerEventCh, dockerEventErrCh := w.getEventCh(dockerWatcher)

 	for {
 		select {
 		case <-w.task.Context().Done():
-			w.l.Debug("stopped by context done")
-			return
-		case <-w.refCount.Zero():
-			w.l.Debug("stopped by zero ref count")
-			return
+			cause := context.Cause(w.task.Context())
+			w.l.Debugf("watcher stopped by context done: %s", cause)
+			return cause
 		case err := <-dockerEventErrCh:
 			if err != nil && err.IsNot(context.Canceled) {
 				w.l.Error(E.FailWith("docker watcher", err))
-				return
+				return err.Error()
 			}
 		case e := <-dockerEventCh:
 			switch {
+			case e.Action == events.ActionContainerDestroy:
+				w.ContainerRunning = false
+				w.ready.Store(false)
+				w.l.Info("watcher stopped by container destruction")
+				return errors.New("container destroyed")
 			// create / start / unpause
 			case e.Action.IsContainerWake():
 				w.ContainerRunning = true
@@ -229,18 +244,31 @@ func (w *Watcher) watchUntilCancel() {
 				w.l.Info("container awaken")
 			case e.Action.IsContainerSleep(): // stop / pause / kil
 				w.ContainerRunning = false
-				w.ticker.Stop()
 				w.ready.Store(false)
+				w.ticker.Stop()
 			default:
 				w.l.Errorf("unexpected docker event: %s", e)
 			}
+			// container name changed should also change the container id
+			if w.ContainerName != e.ActorName {
+				w.l.Debugf("container renamed %s -> %s", w.ContainerName, e.ActorName)
+				w.ContainerName = e.ActorName
+			}
+			if w.ContainerID != e.ActorID {
+				w.l.Debugf("container id changed %s -> %s", w.ContainerID, e.ActorID)
+				w.ContainerID = e.ActorID
+				// recreate event stream
+				eventTask.Finish("recreate event stream")
+				eventTask, dockerEventCh, dockerEventErrCh = w.getEventCh(dockerWatcher)
+			}
 		case <-w.ticker.C:
-			w.l.Debug("idle timeout")
 			w.ticker.Stop()
-			if err := w.stopByMethod(); err != nil && err.IsNot(context.Canceled) {
-				w.l.Error(E.FailWith("stop", err).Extraf("stop method: %s", w.StopMethod))
-			} else {
-				w.l.Info("stopped by idle timeout")
+			if w.ContainerRunning {
+				if err := w.stopByMethod(); err != nil && !errors.Is(err, context.Canceled) {
+					w.l.Errorf("container stop with method %q failed with error: %v", w.StopMethod, err)
+				} else {
+					w.l.Info("container stopped by idle timeout")
+				}
 			}
 		}
 	}
--- a/internal/docker/inspect.go
+++ b/internal/docker/inspect.go
@@ -2,6 +2,7 @@ package docker

 import (
 	"context"
+	"errors"
 	"time"

 	E "github.com/yusing/go-proxy/internal/error"
@@ -19,7 +20,7 @@ func Inspect(dockerHost string, containerID string) (*Container, E.NestedError)
 }

 func (c Client) Inspect(containerID string) (*Container, E.NestedError) {
-	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	ctx, cancel := context.WithTimeoutCause(context.Background(), 3*time.Second, errors.New("docker container inspect timeout"))
 	defer cancel()

 	json, err := c.ContainerInspect(ctx, containerID)