feat: idle sleep for proxmox LXCs

internal/idlewatcher/common.go

@@ -0,0 +1,13 @@
+package idlewatcher
+
+import "context"
+
+func (w *Watcher) cancelled(reqCtx context.Context) bool {
+	select {
+	case <-reqCtx.Done():
+		w.l.Debug().AnErr("cause", context.Cause(reqCtx)).Msg("wake canceled")
+		return true
+	default:
+		return false
+	}
+}

internal/idlewatcher/container.go

@@ -1,60 +0,0 @@
-package idlewatcher
-
-import (
-	"context"
-	"errors"
-
-	"github.com/docker/docker/api/types/container"
-)
-
-type (
-	containerMeta struct {
-		ContainerID, ContainerName string
-	}
-	containerState struct {
-		running bool
-		ready   bool
-		err     error
-	}
-)
-
-func (w *Watcher) ContainerID() string {
-	return w.route.ContainerInfo().ContainerID
-}
-
-func (w *Watcher) ContainerName() string {
-	return w.route.ContainerInfo().ContainerName
-}
-
-func (w *Watcher) containerStop(ctx context.Context) error {
-	return w.client.ContainerStop(ctx, w.ContainerID(), container.StopOptions{
-		Signal:  string(w.Config().StopSignal),
-		Timeout: &w.Config().StopTimeout,
-	})
-}
-
-func (w *Watcher) containerPause(ctx context.Context) error {
-	return w.client.ContainerPause(ctx, w.ContainerID())
-}
-
-func (w *Watcher) containerKill(ctx context.Context) error {
-	return w.client.ContainerKill(ctx, w.ContainerID(), string(w.Config().StopSignal))
-}
-
-func (w *Watcher) containerUnpause(ctx context.Context) error {
-	return w.client.ContainerUnpause(ctx, w.ContainerID())
-}
-
-func (w *Watcher) containerStart(ctx context.Context) error {
-	return w.client.ContainerStart(ctx, w.ContainerID(), container.StartOptions{})
-}
-
-func (w *Watcher) containerStatus() (string, error) {
-	ctx, cancel := context.WithTimeoutCause(w.task.Context(), dockerReqTimeout, errors.New("docker request timeout"))
-	defer cancel()
-	json, err := w.client.ContainerInspect(ctx, w.ContainerID())
-	if err != nil {
-		return "", err
-	}
-	return json.State.Status, nil
-}

internal/idlewatcher/debug.go

@@ -0,0 +1,40 @@
+package idlewatcher
+
+import (
+	"iter"
+	"strconv"
+
+	"github.com/yusing/go-proxy/internal/utils/strutils"
+)
+
+type watcherDebug struct {
+	*Watcher
+}
+
+func (w watcherDebug) MarshalMap() map[string]any {
+	state := w.state.Load()
+	return map[string]any{
+		"name": w.Name(),
+		"state": map[string]string{
+			"status": string(state.status),
+			"ready":  strconv.FormatBool(state.ready),
+			"err":    fmtErr(state.err),
+		},
+		"expires":    strutils.FormatTime(w.expires()),
+		"last_reset": strutils.FormatTime(w.lastReset.Load()),
+		"config":     w.cfg,
+	}
+}
+
+func Watchers() iter.Seq2[string, watcherDebug] {
+	return func(yield func(string, watcherDebug) bool) {
+		watcherMapMu.RLock()
+		defer watcherMapMu.RUnlock()
+
+		for k, w := range watcherMap {
+			if !yield(k, watcherDebug{w}) {
+				return
+			}
+		}
+	}
+}

internal/idlewatcher/handle_http.go

@@ -42,20 +42,6 @@ func (w *Watcher) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func (w *Watcher) cancelled(reqCtx context.Context, rw http.ResponseWriter) bool {
-	select {
-	case <-reqCtx.Done():
-		w.WakeDebug().Str("cause", context.Cause(reqCtx).Error()).Msg("canceled")
-		return true
-	case <-w.task.Context().Done():
-		w.WakeDebug().Str("cause", w.task.FinishCause().Error()).Msg("canceled")
-		http.Error(rw, "Service unavailable", http.StatusServiceUnavailable)
-		return true
-	default:
-		return false
-	}
-}
-
 func isFaviconPath(path string) bool {
 	return path == "/favicon.ico"
 }
@@ -70,13 +56,13 @@ func (w *Watcher) wakeFromHTTP(rw http.ResponseWriter, r *http.Request) (shouldN
 
 	// handle favicon request
 	if isFaviconPath(r.URL.Path) {
-		r.URL.RawQuery = "alias=" + w.route.TargetName()
+		r.URL.RawQuery = "alias=" + w.rp.TargetName
 		favicon.GetFavIcon(rw, r)
 		return false
 	}
 
 	// Check if start endpoint is configured and request path matches
-	if w.Config().StartEndpoint != "" && r.URL.Path != w.Config().StartEndpoint {
+	if w.cfg.StartEndpoint != "" && r.URL.Path != w.cfg.StartEndpoint {
 		http.Error(rw, "Forbidden: Container can only be started via configured start endpoint", http.StatusForbidden)
 		return false
 	}
@@ -95,44 +81,48 @@ func (w *Watcher) wakeFromHTTP(rw http.ResponseWriter, r *http.Request) (shouldN
 		rw.Header().Add("Cache-Control", "must-revalidate")
 		rw.Header().Add("Connection", "close")
 		if _, err := rw.Write(body); err != nil {
-			w.Err(err).Msg("error writing http response")
+			return false
 		}
 		return false
 	}
 
-	ctx, cancel := context.WithTimeoutCause(r.Context(), w.Config().WakeTimeout, errors.New("wake timeout"))
+	ctx, cancel := context.WithTimeoutCause(r.Context(), w.cfg.WakeTimeout, errors.New("wake timeout"))
 	defer cancel()
 
-	if w.cancelled(ctx, rw) {
+	if w.cancelled(ctx) {
+		gphttp.ServerError(rw, r, context.Cause(ctx), http.StatusServiceUnavailable)
 		return false
 	}
 
-	w.WakeTrace().Msg("signal received")
+	w.l.Trace().Msg("signal received")
 	err := w.wakeIfStopped()
 	if err != nil {
-		w.WakeError(err)
-		http.Error(rw, "Error waking container", http.StatusInternalServerError)
+		gphttp.ServerError(rw, r, err)
 		return false
 	}
 
+	var ready bool
+
 	for {
-		if w.cancelled(ctx, rw) {
+		w.resetIdleTimer()
+
+		if w.cancelled(ctx) {
+			gphttp.ServerError(rw, r, context.Cause(ctx), http.StatusServiceUnavailable)
 			return false
 		}
 
-		ready, err := w.checkUpdateState()
+		w, ready, err = checkUpdateState(w.Key())
 		if err != nil {
-			http.Error(rw, "Error waking container", http.StatusInternalServerError)
+			gphttp.ServerError(rw, r, err)
 			return false
 		}
 		if ready {
-			w.resetIdleTimer()
 			if isCheckRedirect {
-				w.Debug().Msgf("redirecting to %s ...", w.hc.URL())
+				w.l.Debug().Stringer("url", w.hc.URL()).Msg("container is ready, redirecting")
 				rw.WriteHeader(http.StatusOK)
 				return false
 			}
-			w.Debug().Msgf("passing through to %s ...", w.hc.URL())
+			w.l.Debug().Stringer("url", w.hc.URL()).Msg("container is ready, passing through")
 			return true
 		}
 

internal/idlewatcher/handle_stream.go

@@ -3,11 +3,10 @@ package idlewatcher
 import (
 	"context"
 	"errors"
-	"fmt"
 	"net"
 	"time"
 
-	"github.com/yusing/go-proxy/internal/net/types"
+	gpnet "github.com/yusing/go-proxy/internal/net/types"
 )
 
 // Setup implements types.Stream.
@@ -21,19 +20,19 @@ func (w *Watcher) Setup() error {
 }
 
 // Accept implements types.Stream.
-func (w *Watcher) Accept() (conn types.StreamConn, err error) {
+func (w *Watcher) Accept() (conn gpnet.StreamConn, err error) {
 	conn, err = w.stream.Accept()
 	if err != nil {
 		return
 	}
 	if wakeErr := w.wakeFromStream(); wakeErr != nil {
-		w.WakeError(wakeErr)
+		w.l.Err(wakeErr).Msg("error waking container")
 	}
 	return
 }
 
 // Handle implements types.Stream.
-func (w *Watcher) Handle(conn types.StreamConn) error {
+func (w *Watcher) Handle(conn gpnet.StreamConn) error {
 	if err := w.wakeFromStream(); err != nil {
 		return err
 	}
@@ -53,35 +52,29 @@ func (w *Watcher) wakeFromStream() error {
 		return nil
 	}
 
-	w.WakeDebug().Msg("wake signal received")
-	wakeErr := w.wakeIfStopped()
-	if wakeErr != nil {
-		wakeErr = fmt.Errorf("%s failed: %w", w.String(), wakeErr)
-		w.WakeError(wakeErr)
-		return wakeErr
+	w.l.Debug().Msg("wake signal received")
+	err := w.wakeIfStopped()
+	if err != nil {
+		return err
 	}
 
-	ctx, cancel := context.WithTimeoutCause(w.task.Context(), w.Config().WakeTimeout, errors.New("wake timeout"))
+	ctx, cancel := context.WithTimeoutCause(w.task.Context(), w.cfg.WakeTimeout, errors.New("wake timeout"))
 	defer cancel()
 
+	var ready bool
+
 	for {
-		select {
-		case <-w.task.Context().Done():
-			cause := w.task.FinishCause()
-			w.WakeDebug().Str("cause", cause.Error()).Msg("canceled")
-			return cause
-		case <-ctx.Done():
-			cause := context.Cause(ctx)
-			w.WakeDebug().Str("cause", cause.Error()).Msg("timeout")
-			return cause
-		default:
+		if w.cancelled(ctx) {
+			return context.Cause(ctx)
 		}
 
-		if ready, err := w.checkUpdateState(); err != nil {
+		w, ready, err = checkUpdateState(w.Key())
+		if err != nil {
 			return err
-		} else if ready {
+		}
+		if ready {
 			w.resetIdleTimer()
-			w.Debug().Msg("container is ready, passing through to " + w.hc.URL().String())
+			w.l.Debug().Stringer("url", w.hc.URL()).Msg("container is ready, passing through")
 			return nil
 		}
 

internal/idlewatcher/health.go

@@ -0,0 +1,122 @@
+package idlewatcher
+
+import (
+	"errors"
+	"time"
+
+	"github.com/yusing/go-proxy/internal/gperr"
+	idlewatcher "github.com/yusing/go-proxy/internal/idlewatcher/types"
+	"github.com/yusing/go-proxy/internal/task"
+	"github.com/yusing/go-proxy/internal/watcher/health"
+)
+
+// Start implements health.HealthMonitor.
+func (w *Watcher) Start(parent task.Parent) gperr.Error {
+	w.task.OnCancel("route_cleanup", func() {
+		parent.Finish(w.task.FinishCause())
+	})
+	return nil
+}
+
+// Task implements health.HealthMonitor.
+func (w *Watcher) Task() *task.Task {
+	return w.task
+}
+
+// Finish implements health.HealthMonitor.
+func (w *Watcher) Finish(reason any) {
+	if w.stream != nil {
+		w.stream.Close()
+	}
+}
+
+// Name implements health.HealthMonitor.
+func (w *Watcher) Name() string {
+	return w.cfg.ContainerName()
+}
+
+// String implements health.HealthMonitor.
+func (w *Watcher) String() string {
+	return w.Name()
+}
+
+// Uptime implements health.HealthMonitor.
+func (w *Watcher) Uptime() time.Duration {
+	return 0
+}
+
+// Latency implements health.HealthMonitor.
+func (w *Watcher) Latency() time.Duration {
+	return 0
+}
+
+// Status implements health.HealthMonitor.
+func (w *Watcher) Status() health.Status {
+	state := w.state.Load()
+	if state.err != nil {
+		return health.StatusError
+	}
+	if state.ready {
+		return health.StatusHealthy
+	}
+	if state.status == idlewatcher.ContainerStatusRunning {
+		return health.StatusStarting
+	}
+	return health.StatusNapping
+}
+
+func checkUpdateState(key string) (w *Watcher, ready bool, err error) {
+	watcherMapMu.RLock()
+	w, ok := watcherMap[key]
+	if !ok {
+		watcherMapMu.RUnlock()
+		return nil, false, errors.New("watcher not found")
+	}
+	watcherMapMu.RUnlock()
+
+	// already ready
+	if w.ready() {
+		return w, true, nil
+	}
+
+	if !w.running() {
+		return w, false, nil
+	}
+
+	// the new container info not yet updated
+	if w.hc.URL().Host == "" {
+		return w, false, nil
+	}
+
+	res, err := w.hc.CheckHealth()
+	if err != nil {
+		w.setError(err)
+		return w, false, err
+	}
+
+	if res.Healthy {
+		w.setReady()
+		return w, true, nil
+	}
+	w.setStarting()
+	return w, false, nil
+}
+
+// MarshalMap implements health.HealthMonitor.
+func (w *Watcher) MarshalMap() map[string]any {
+	url := w.hc.URL()
+	if url.Port() == "0" {
+		url = nil
+	}
+	var detail string
+	if err := w.error(); err != nil {
+		detail = err.Error()
+	}
+	return (&health.JSONRepresentation{
+		Name:   w.Name(),
+		Status: w.Status(),
+		Config: dummyHealthCheckConfig,
+		URL:    url,
+		Detail: detail,
+	}).MarshalMap()
+}

internal/idlewatcher/loading_page.go

@@ -19,11 +19,11 @@ var loadingPage []byte
 var loadingPageTmpl = template.Must(template.New("loading_page").Parse(string(loadingPage)))
 
 func (w *Watcher) makeLoadingPageBody() []byte {
-	msg := w.ContainerName() + " is starting..."
+	msg := w.cfg.ContainerName() + " is starting..."
 
 	data := new(templateData)
 	data.CheckRedirectHeader = httpheaders.HeaderGoDoxyCheckRedirect
-	data.Title = w.route.HomepageItem().Name
+	data.Title = w.cfg.ContainerName()
 	data.Message = msg
 
 	buf := bytes.NewBuffer(make([]byte, len(loadingPage)+len(data.Title)+len(data.Message)+len(httpheaders.HeaderGoDoxyCheckRedirect)))

internal/idlewatcher/provider/docker.go

@@ -0,0 +1,90 @@
+package provider
+
+import (
+	"context"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/yusing/go-proxy/internal/docker"
+	"github.com/yusing/go-proxy/internal/gperr"
+	idlewatcher "github.com/yusing/go-proxy/internal/idlewatcher/types"
+	"github.com/yusing/go-proxy/internal/watcher"
+)
+
+type DockerProvider struct {
+	client      *docker.SharedClient
+	watcher     *watcher.DockerWatcher
+	containerID string
+}
+
+var startOptions = container.StartOptions{}
+
+func NewDockerProvider(dockerHost, containerID string) (idlewatcher.Provider, error) {
+	client, err := docker.NewClient(dockerHost)
+	if err != nil {
+		return nil, err
+	}
+	return &DockerProvider{
+		client:      client,
+		watcher:     watcher.NewDockerWatcher(dockerHost),
+		containerID: containerID,
+	}, nil
+}
+
+func (p *DockerProvider) ContainerPause(ctx context.Context) error {
+	return p.client.ContainerPause(ctx, p.containerID)
+}
+
+func (p *DockerProvider) ContainerUnpause(ctx context.Context) error {
+	return p.client.ContainerUnpause(ctx, p.containerID)
+}
+
+func (p *DockerProvider) ContainerStart(ctx context.Context) error {
+	return p.client.ContainerStart(ctx, p.containerID, startOptions)
+}
+
+func (p *DockerProvider) ContainerStop(ctx context.Context, signal idlewatcher.Signal, timeout int) error {
+	return p.client.ContainerStop(ctx, p.containerID, container.StopOptions{
+		Signal:  string(signal),
+		Timeout: &timeout,
+	})
+}
+
+func (p *DockerProvider) ContainerKill(ctx context.Context, signal idlewatcher.Signal) error {
+	return p.client.ContainerKill(ctx, p.containerID, string(signal))
+}
+
+func (p *DockerProvider) ContainerStatus(ctx context.Context) (idlewatcher.ContainerStatus, error) {
+	status, err := p.client.ContainerInspect(ctx, p.containerID)
+	if err != nil {
+		return idlewatcher.ContainerStatusError, err
+	}
+	switch status.State.Status {
+	case "running":
+		return idlewatcher.ContainerStatusRunning, nil
+	case "exited", "dead", "restarting":
+		return idlewatcher.ContainerStatusStopped, nil
+	case "paused":
+		return idlewatcher.ContainerStatusPaused, nil
+	}
+	return idlewatcher.ContainerStatusError, idlewatcher.ErrUnexpectedContainerStatus.Subject(status.State.Status)
+}
+
+func (p *DockerProvider) Watch(ctx context.Context) (eventCh <-chan watcher.Event, errCh <-chan gperr.Error) {
+	return p.watcher.EventsWithOptions(ctx, watcher.DockerListOptions{
+		Filters: watcher.NewDockerFilter(
+			watcher.DockerFilterContainer,
+			watcher.DockerFilterContainerNameID(p.containerID),
+			watcher.DockerFilterStart,
+			watcher.DockerFilterStop,
+			watcher.DockerFilterDie,
+			watcher.DockerFilterKill,
+			watcher.DockerFilterDestroy,
+			watcher.DockerFilterPause,
+			watcher.DockerFilterUnpause,
+		),
+	})
+}
+
+func (p *DockerProvider) Close() {
+	p.client.Close()
+}

internal/idlewatcher/provider/proxmox.go

@@ -0,0 +1,129 @@
+package provider
+
+import (
+	"context"
+	"strconv"
+	"time"
+
+	"github.com/yusing/go-proxy/internal/gperr"
+	idlewatcher "github.com/yusing/go-proxy/internal/idlewatcher/types"
+	"github.com/yusing/go-proxy/internal/proxmox"
+	"github.com/yusing/go-proxy/internal/watcher"
+	"github.com/yusing/go-proxy/internal/watcher/events"
+)
+
+type ProxmoxProvider struct {
+	*proxmox.Node
+	vmid    int
+	lxcName string
+	running bool
+}
+
+const proxmoxStateCheckInterval = 1 * time.Second
+
+var ErrNodeNotFound = gperr.New("node not found in pool")
+
+func NewProxmoxProvider(nodeName string, vmid int) (idlewatcher.Provider, error) {
+	node, ok := proxmox.Nodes.Get(nodeName)
+	if !ok {
+		return nil, ErrNodeNotFound.Subject(nodeName).
+			Withf("available nodes: %s", proxmox.AvailableNodeNames())
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	lxcName, err := node.LXCName(ctx, vmid)
+	if err != nil {
+		return nil, err
+	}
+	return &ProxmoxProvider{Node: node, vmid: vmid, lxcName: lxcName}, nil
+}
+
+func (p *ProxmoxProvider) ContainerPause(ctx context.Context) error {
+	return p.LXCAction(ctx, p.vmid, proxmox.LXCSuspend)
+}
+
+func (p *ProxmoxProvider) ContainerUnpause(ctx context.Context) error {
+	return p.LXCAction(ctx, p.vmid, proxmox.LXCResume)
+}
+
+func (p *ProxmoxProvider) ContainerStart(ctx context.Context) error {
+	return p.LXCAction(ctx, p.vmid, proxmox.LXCStart)
+}
+
+func (p *ProxmoxProvider) ContainerStop(ctx context.Context, _ idlewatcher.Signal, _ int) error {
+	return p.LXCAction(ctx, p.vmid, proxmox.LXCShutdown)
+}
+
+func (p *ProxmoxProvider) ContainerKill(ctx context.Context, _ idlewatcher.Signal) error {
+	return p.LXCAction(ctx, p.vmid, proxmox.LXCShutdown)
+}
+
+func (p *ProxmoxProvider) ContainerStatus(ctx context.Context) (idlewatcher.ContainerStatus, error) {
+	status, err := p.LXCStatus(ctx, p.vmid)
+	if err != nil {
+		return idlewatcher.ContainerStatusError, err
+	}
+	switch status {
+	case proxmox.LXCStatusRunning:
+		return idlewatcher.ContainerStatusRunning, nil
+	case proxmox.LXCStatusStopped:
+		return idlewatcher.ContainerStatusStopped, nil
+	}
+	return idlewatcher.ContainerStatusError, idlewatcher.ErrUnexpectedContainerStatus.Subject(string(status))
+}
+
+func (p *ProxmoxProvider) Watch(ctx context.Context) (<-chan watcher.Event, <-chan gperr.Error) {
+	eventCh := make(chan watcher.Event)
+	errCh := make(chan gperr.Error)
+
+	go func() {
+		defer close(eventCh)
+		defer close(errCh)
+
+		var err error
+		p.running, err = p.LXCIsRunning(ctx, p.vmid)
+		if err != nil {
+			errCh <- gperr.Wrap(err)
+			return
+		}
+
+		ticker := time.NewTicker(proxmoxStateCheckInterval)
+		defer ticker.Stop()
+
+		event := watcher.Event{
+			Type:      events.EventTypeDocker,
+			ActorID:   strconv.Itoa(p.vmid),
+			ActorName: p.lxcName,
+		}
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				status, err := p.ContainerStatus(ctx)
+				if err != nil {
+					errCh <- gperr.Wrap(err)
+					return
+				}
+				running := status == idlewatcher.ContainerStatusRunning
+				if p.running != running {
+					p.running = running
+					if running {
+						event.Action = events.ActionContainerStart
+					} else {
+						event.Action = events.ActionContainerStop
+					}
+					eventCh <- event
+				}
+			}
+		}
+	}()
+
+	return eventCh, errCh
+}
+
+func (p *ProxmoxProvider) Close() {
+	// noop
+}

internal/idlewatcher/state.go

@@ -1,7 +1,9 @@
 package idlewatcher
 
+import idlewatcher "github.com/yusing/go-proxy/internal/idlewatcher/types"
+
 func (w *Watcher) running() bool {
-	return w.state.Load().running
+	return w.state.Load().status == idlewatcher.ContainerStatusRunning
 }
 
 func (w *Watcher) ready() bool {
@@ -14,26 +16,29 @@ func (w *Watcher) error() error {
 
 func (w *Watcher) setReady() {
 	w.state.Store(&containerState{
-		running: true,
-		ready:   true,
+		status: idlewatcher.ContainerStatusRunning,
+		ready:  true,
 	})
 }
 
 func (w *Watcher) setStarting() {
 	w.state.Store(&containerState{
-		running: true,
-		ready:   false,
+		status: idlewatcher.ContainerStatusRunning,
+		ready:  false,
 	})
 }
 
-func (w *Watcher) setNapping() {
-	w.setError(nil)
+func (w *Watcher) setNapping(status idlewatcher.ContainerStatus) {
+	w.state.Store(&containerState{
+		status: status,
+		ready:  false,
+	})
 }
 
 func (w *Watcher) setError(err error) {
 	w.state.Store(&containerState{
-		running: false,
-		ready:   false,
-		err:     err,
+		status: idlewatcher.ContainerStatusError,
+		ready:  false,
+		err:    err,
 	})
 }

internal/idlewatcher/types/config.go

@@ -1,110 +1,128 @@
 package idlewatcher
 
 import (
-	"errors"
 	"net/url"
+	"strconv"
 	"strings"
 	"time"
 
-	"github.com/yusing/go-proxy/internal/docker"
 	"github.com/yusing/go-proxy/internal/gperr"
 )
 
 type (
 	Config struct {
-		IdleTimeout   time.Duration `json:"idle_timeout,omitempty"`
-		WakeTimeout   time.Duration `json:"wake_timeout,omitempty"`
-		StopTimeout   int           `json:"stop_timeout,omitempty"` // docker api takes integer seconds for timeout argument
-		StopMethod    StopMethod    `json:"stop_method,omitempty"`
+		Proxmox *ProxmoxConfig `json:"proxmox,omitempty"`
+		Docker  *DockerConfig  `json:"docker,omitempty"`
+
+		IdleTimeout   time.Duration `json:"idle_timeout"`
+		WakeTimeout   time.Duration `json:"wake_timeout"`
+		StopTimeout   time.Duration `json:"stop_timeout"`
+		StopMethod    StopMethod    `json:"stop_method"`
 		StopSignal    Signal        `json:"stop_signal,omitempty"`
 		StartEndpoint string        `json:"start_endpoint,omitempty"` // Optional path that must be hit to start container
 	}
 	StopMethod string
 	Signal     string
+
+	DockerConfig struct {
+		DockerHost    string `json:"docker_host" validate:"required"`
+		ContainerID   string `json:"container_id" validate:"required"`
+		ContainerName string `json:"container_name" validate:"required"`
+	}
+	ProxmoxConfig struct {
+		Node string `json:"node" validate:"required"`
+		VMID int    `json:"vmid" validate:"required"`
+	}
 )
 
 const (
+	WakeTimeoutDefault = 30 * time.Second
+	StopTimeoutDefault = 1 * time.Minute
+
 	StopMethodPause StopMethod = "pause"
 	StopMethodStop  StopMethod = "stop"
 	StopMethodKill  StopMethod = "kill"
 )
 
-var validSignals = map[string]struct{}{
-	"":       {},
-	"SIGINT": {}, "SIGTERM": {}, "SIGHUP": {}, "SIGQUIT": {},
-	"INT": {}, "TERM": {}, "HUP": {}, "QUIT": {},
+func (c *Config) Key() string {
+	if c.Docker != nil {
+		return c.Docker.ContainerID
+	}
+	return c.Proxmox.Node + ":" + strconv.Itoa(c.Proxmox.VMID)
 }
 
-func ValidateConfig(cont *docker.Container) (*Config, gperr.Error) {
-	if cont == nil || cont.IdleTimeout == "" {
-		return nil, nil
+func (c *Config) ContainerName() string {
+	if c.Docker != nil {
+		return c.Docker.ContainerName
 	}
-
-	errs := gperr.NewBuilder("invalid idlewatcher config")
-
-	idleTimeout := gperr.Collect(errs, validateDurationPostitive, cont.IdleTimeout)
-	wakeTimeout := gperr.Collect(errs, validateDurationPostitive, cont.WakeTimeout)
-	stopTimeout := gperr.Collect(errs, validateDurationPostitive, cont.StopTimeout)
-	stopMethod := gperr.Collect(errs, validateStopMethod, cont.StopMethod)
-	signal := gperr.Collect(errs, validateSignal, cont.StopSignal)
-	startEndpoint := gperr.Collect(errs, validateStartEndpoint, cont.StartEndpoint)
-
-	if errs.HasError() {
-		return nil, errs.Error()
-	}
-
-	return &Config{
-		IdleTimeout:   idleTimeout,
-		WakeTimeout:   wakeTimeout,
-		StopTimeout:   int(stopTimeout.Seconds()),
-		StopMethod:    stopMethod,
-		StopSignal:    signal,
-		StartEndpoint: startEndpoint,
-	}, nil
+	return "lxc " + strconv.Itoa(c.Proxmox.VMID)
 }
 
-func validateDurationPostitive(value string) (time.Duration, error) {
-	d, err := time.ParseDuration(value)
-	if err != nil {
-		return 0, err
+func (c *Config) Validate() gperr.Error {
+	if c.IdleTimeout == 0 { // no idle timeout means no idle watcher
+		return nil
 	}
-	if d < 0 {
-		return 0, errors.New("duration must be positive")
-	}
-	return d, nil
+	errs := gperr.NewBuilder("idlewatcher config validation error")
+	errs.AddRange(
+		c.validateProvider(),
+		c.validateTimeouts(),
+		c.validateStopMethod(),
+		c.validateStopSignal(),
+		c.validateStartEndpoint(),
+	)
+	return errs.Error()
 }
 
-func validateSignal(s string) (Signal, error) {
-	if _, ok := validSignals[s]; ok {
-		return Signal(s), nil
+func (c *Config) validateProvider() error {
+	if c.Docker == nil && c.Proxmox == nil {
+		return gperr.New("missing idlewatcher provider config")
 	}
-	return "", errors.New("invalid signal " + s)
+	return nil
 }
 
-func validateStopMethod(s string) (StopMethod, error) {
-	sm := StopMethod(s)
-	switch sm {
+func (c *Config) validateTimeouts() error {
+	if c.WakeTimeout == 0 {
+		c.WakeTimeout = WakeTimeoutDefault
+	}
+	if c.StopTimeout == 0 {
+		c.StopTimeout = StopTimeoutDefault
+	}
+	return nil
+}
+
+func (c *Config) validateStopMethod() error {
+	switch c.StopMethod {
+	case "":
+		c.StopMethod = StopMethodStop
+		return nil
 	case StopMethodPause, StopMethodStop, StopMethodKill:
-		return sm, nil
+		return nil
 	default:
-		return "", errors.New("invalid stop method " + s)
+		return gperr.New("invalid stop method").Subject(string(c.StopMethod))
 	}
 }
 
-func validateStartEndpoint(s string) (string, error) {
-	if s == "" {
-		return "", nil
+func (c *Config) validateStopSignal() error {
+	switch c.StopSignal {
+	case "", "SIGINT", "SIGTERM", "SIGQUIT", "SIGHUP", "INT", "TERM", "QUIT", "HUP":
+		return nil
+	default:
+		return gperr.New("invalid stop signal").Subject(string(c.StopSignal))
+	}
+}
+
+func (c *Config) validateStartEndpoint() error {
+	if c.StartEndpoint == "" {
+		return nil
 	}
 	// checks needed as of Go 1.6 because of change https://github.com/golang/go/commit/617c93ce740c3c3cc28cdd1a0d712be183d0b328#diff-6c2d018290e298803c0c9419d8739885L195
 	// emulate browser and strip the '#' suffix prior to validation. see issue-#237
-	if i := strings.Index(s, "#"); i > -1 {
-		s = s[:i]
+	if i := strings.Index(c.StartEndpoint, "#"); i > -1 {
+		c.StartEndpoint = c.StartEndpoint[:i]
 	}
-	if len(s) == 0 {
-		return "", errors.New("start endpoint must not be empty if defined")
+	if len(c.StartEndpoint) == 0 {
+		return gperr.New("start endpoint must not be empty if defined")
 	}
-	if _, err := url.ParseRequestURI(s); err != nil {
-		return "", err
-	}
-	return s, nil
+	_, err := url.ParseRequestURI(c.StartEndpoint)
+	return err
 }

internal/idlewatcher/types/config_test.go

@@ -35,9 +35,10 @@ func TestValidateStartEndpoint(t *testing.T) {
 	}
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
-			s, err := validateStartEndpoint(tc.input)
+			cfg := Config{StartEndpoint: tc.input}
+			err := cfg.validateStartEndpoint()
 			if err == nil {
-				ExpectEqual(t, s, tc.input)
+				ExpectEqual(t, cfg.StartEndpoint, tc.input)
 			}
 			if (err != nil) != tc.wantErr {
 				t.Errorf("validateStartEndpoint() error = %v, wantErr %t", err, tc.wantErr)

internal/idlewatcher/types/container_status.go

@@ -0,0 +1,14 @@
+package idlewatcher
+
+import "github.com/yusing/go-proxy/internal/gperr"
+
+type ContainerStatus string
+
+const (
+	ContainerStatusError   ContainerStatus = "error"
+	ContainerStatusRunning ContainerStatus = "running"
+	ContainerStatusPaused  ContainerStatus = "paused"
+	ContainerStatusStopped ContainerStatus = "stopped"
+)
+
+var ErrUnexpectedContainerStatus = gperr.New("unexpected container status")

internal/idlewatcher/types/provider.go

@@ -0,0 +1,19 @@
+package idlewatcher
+
+import (
+	"context"
+
+	"github.com/yusing/go-proxy/internal/gperr"
+	"github.com/yusing/go-proxy/internal/watcher/events"
+)
+
+type Provider interface {
+	ContainerPause(ctx context.Context) error
+	ContainerUnpause(ctx context.Context) error
+	ContainerStart(ctx context.Context) error
+	ContainerStop(ctx context.Context, signal Signal, timeout int) error
+	ContainerKill(ctx context.Context, signal Signal) error
+	ContainerStatus(ctx context.Context) (ContainerStatus, error)
+	Watch(ctx context.Context) (eventCh <-chan events.Event, errCh <-chan gperr.Error)
+	Close()
+}

internal/idlewatcher/waker.go

@@ -1,172 +0,0 @@
-package idlewatcher
-
-import (
-	"time"
-
-	"github.com/yusing/go-proxy/internal/gperr"
-	idlewatcher "github.com/yusing/go-proxy/internal/idlewatcher/types"
-	"github.com/yusing/go-proxy/internal/net/gphttp/reverseproxy"
-	net "github.com/yusing/go-proxy/internal/net/types"
-	route "github.com/yusing/go-proxy/internal/route/types"
-	"github.com/yusing/go-proxy/internal/task"
-	U "github.com/yusing/go-proxy/internal/utils"
-	"github.com/yusing/go-proxy/internal/watcher/health"
-	"github.com/yusing/go-proxy/internal/watcher/health/monitor"
-)
-
-type (
-	Waker = idlewatcher.Waker
-	waker struct {
-		_ U.NoCopy
-
-		rp     *reverseproxy.ReverseProxy
-		stream net.Stream
-		hc     health.HealthChecker
-	}
-)
-
-const (
-	idleWakerCheckInterval = 100 * time.Millisecond
-	idleWakerCheckTimeout  = time.Second
-)
-
-// TODO: support stream
-
-func newWaker(parent task.Parent, route route.Route, rp *reverseproxy.ReverseProxy, stream net.Stream) (Waker, gperr.Error) {
-	hcCfg := route.HealthCheckConfig()
-	hcCfg.Timeout = idleWakerCheckTimeout
-
-	waker := &waker{
-		rp:     rp,
-		stream: stream,
-	}
-	watcher, err := registerWatcher(parent, route, waker)
-	if err != nil {
-		return nil, gperr.Errorf("register watcher: %w", err)
-	}
-
-	switch {
-	case route.IsAgent():
-		waker.hc = monitor.NewAgentProxiedMonitor(route.Agent(), hcCfg, monitor.AgentTargetFromURL(route.TargetURL()))
-	case rp != nil:
-		waker.hc = monitor.NewHTTPHealthChecker(route.TargetURL(), hcCfg)
-	case stream != nil:
-		waker.hc = monitor.NewRawHealthChecker(route.TargetURL(), hcCfg)
-	default:
-		panic("both nil")
-	}
-
-	return watcher, nil
-}
-
-// lifetime should follow route provider.
-func NewHTTPWaker(parent task.Parent, route route.Route, rp *reverseproxy.ReverseProxy) (Waker, gperr.Error) {
-	return newWaker(parent, route, rp, nil)
-}
-
-func NewStreamWaker(parent task.Parent, route route.Route, stream net.Stream) (Waker, gperr.Error) {
-	return newWaker(parent, route, nil, stream)
-}
-
-// Start implements health.HealthMonitor.
-func (w *Watcher) Start(parent task.Parent) gperr.Error {
-	w.task.OnCancel("route_cleanup", func() {
-		parent.Finish(w.task.FinishCause())
-	})
-	return nil
-}
-
-// Task implements health.HealthMonitor.
-func (w *Watcher) Task() *task.Task {
-	return w.task
-}
-
-// Finish implements health.HealthMonitor.
-func (w *Watcher) Finish(reason any) {
-	if w.stream != nil {
-		w.stream.Close()
-	}
-}
-
-// Name implements health.HealthMonitor.
-func (w *Watcher) Name() string {
-	return w.String()
-}
-
-// String implements health.HealthMonitor.
-func (w *Watcher) String() string {
-	return w.ContainerName()
-}
-
-// Uptime implements health.HealthMonitor.
-func (w *Watcher) Uptime() time.Duration {
-	return 0
-}
-
-// Latency implements health.HealthMonitor.
-func (w *Watcher) Latency() time.Duration {
-	return 0
-}
-
-// Status implements health.HealthMonitor.
-func (w *Watcher) Status() health.Status {
-	state := w.state.Load()
-	if state.err != nil {
-		return health.StatusError
-	}
-	if state.ready {
-		return health.StatusHealthy
-	}
-	if state.running {
-		return health.StatusStarting
-	}
-	return health.StatusNapping
-}
-
-func (w *Watcher) checkUpdateState() (ready bool, err error) {
-	// already ready
-	if w.ready() {
-		return true, nil
-	}
-
-	if !w.running() {
-		return false, nil
-	}
-
-	// the new container info not yet updated
-	if w.hc.URL().Host == "" {
-		return false, nil
-	}
-
-	res, err := w.hc.CheckHealth()
-	if err != nil {
-		w.setError(err)
-		return false, err
-	}
-
-	if res.Healthy {
-		w.setReady()
-		return true, nil
-	}
-	w.setStarting()
-	return false, nil
-}
-
-// MarshalJSON implements health.HealthMonitor.
-func (w *Watcher) MarshalJSON() ([]byte, error) {
-	var url *net.URL
-	if w.hc.URL().Port() != "0" {
-		url = w.hc.URL()
-	}
-	var detail string
-	if err := w.error(); err != nil {
-		detail = err.Error()
-	}
-	return (&monitor.JSONRepresentation{
-		Name:   w.Name(),
-		Status: w.Status(),
-		Config: w.hc.Config(),
-		URL:    url,
-		Detail: detail,
-	}).MarshalJSON()
-}

internal/idlewatcher/watcher.go

@@ -7,195 +7,236 @@ import (
 	"time"
 
 	"github.com/rs/zerolog"
-	"github.com/yusing/go-proxy/internal/docker"
 	"github.com/yusing/go-proxy/internal/gperr"
+	"github.com/yusing/go-proxy/internal/idlewatcher/provider"
 	idlewatcher "github.com/yusing/go-proxy/internal/idlewatcher/types"
 	"github.com/yusing/go-proxy/internal/logging"
+	"github.com/yusing/go-proxy/internal/net/gphttp/reverseproxy"
+	net "github.com/yusing/go-proxy/internal/net/types"
 	route "github.com/yusing/go-proxy/internal/route/types"
 	"github.com/yusing/go-proxy/internal/task"
 	U "github.com/yusing/go-proxy/internal/utils"
 	"github.com/yusing/go-proxy/internal/utils/atomic"
-	"github.com/yusing/go-proxy/internal/watcher"
 	"github.com/yusing/go-proxy/internal/watcher/events"
+	"github.com/yusing/go-proxy/internal/watcher/health"
+	"github.com/yusing/go-proxy/internal/watcher/health/monitor"
 )
 
 type (
+	routeHelper struct {
+		rp     *reverseproxy.ReverseProxy
+		stream net.Stream
+		hc     health.HealthChecker
+	}
+
+	containerState struct {
+		status idlewatcher.ContainerStatus
+		ready  bool
+		err    error
+	}
+
 	Watcher struct {
 		_ U.NoCopy
+		routeHelper
 
-		zerolog.Logger
+		l zerolog.Logger
 
-		*waker
+		cfg *idlewatcher.Config
 
-		route route.Route
+		provider idlewatcher.Provider
 
-		client *docker.SharedClient
-		state  atomic.Value[*containerState]
+		state     atomic.Value[*containerState]
+		lastReset atomic.Value[time.Time]
 
-		stopByMethod StopCallback // send a docker command w.r.t. `stop_method`
-		ticker       *time.Ticker
-		lastReset    time.Time
-		task         *task.Task
+		ticker *time.Ticker
+		task   *task.Task
 	}
 
 	StopCallback func() error
 )
 
+const ContextKey = "idlewatcher.watcher"
+
 var (
 	watcherMap   = make(map[string]*Watcher)
 	watcherMapMu sync.RWMutex
-
-	errShouldNotReachHere = errors.New("should not reach here")
 )
 
-const dockerReqTimeout = 3 * time.Second
+const (
+	idleWakerCheckInterval = 100 * time.Millisecond
+	idleWakerCheckTimeout  = time.Second
+)
 
-func registerWatcher(parent task.Parent, route route.Route, waker *waker) (*Watcher, error) {
-	cfg := route.IdlewatcherConfig()
-	cont := route.ContainerInfo()
-	key := cont.ContainerID
+var dummyHealthCheckConfig = &health.HealthCheckConfig{
+	Interval: idleWakerCheckInterval,
+	Timeout:  idleWakerCheckTimeout,
+}
+
+var (
+	causeReload           = gperr.New("reloaded")
+	causeContainerDestroy = gperr.New("container destroyed")
+)
+
+const reqTimeout = 3 * time.Second
+
+// TODO: fix stream type
+func NewWatcher(parent task.Parent, r route.Route) (*Watcher, error) {
+	cfg := r.IdlewatcherConfig()
+	key := cfg.Key()
+
+	watcherMapMu.RLock()
+	// if the watcher already exists, finish it
+	w, exists := watcherMap[key]
+	if exists {
+		if w.cfg == cfg {
+			// same address, likely two routes from the same container
+			return w, nil
+		}
+		w.task.Finish(causeReload)
+	}
+	watcherMapMu.RUnlock()
+
+	w = &Watcher{
+		ticker: time.NewTicker(cfg.IdleTimeout),
+		cfg:    cfg,
+		routeHelper: routeHelper{
+			hc: monitor.NewMonitor(r),
+		},
+	}
+
+	var p idlewatcher.Provider
+	var providerType string
+	var err error
+	switch {
+	case cfg.Docker != nil:
+		p, err = provider.NewDockerProvider(cfg.Docker.DockerHost, cfg.Docker.ContainerID)
+		providerType = "docker"
+	default:
+		p, err = provider.NewProxmoxProvider(cfg.Proxmox.Node, cfg.Proxmox.VMID)
+		providerType = "proxmox"
+	}
+
+	if err != nil {
+		return nil, err
+	}
+	w.provider = p
+	w.l = logging.With().
+		Str("provider", providerType).
+		Str("container", cfg.ContainerName()).
+		Logger()
+
+	switch r := r.(type) {
+	case route.ReverseProxyRoute:
+		w.rp = r.ReverseProxy()
+	case route.StreamRoute:
+		w.stream = r
+	default:
+		return nil, gperr.New("unexpected route type")
+	}
+
+	ctx, cancel := context.WithTimeout(parent.Context(), reqTimeout)
+	defer cancel()
+	status, err := w.provider.ContainerStatus(ctx)
+	if err != nil {
+		w.provider.Close()
+		return nil, gperr.Wrap(err, "failed to get container status")
+	}
+
+	switch p := w.provider.(type) {
+	case *provider.ProxmoxProvider:
+		shutdownTimeout := max(time.Second, cfg.StopTimeout-idleWakerCheckTimeout)
+		err = p.LXCSetShutdownTimeout(ctx, cfg.Proxmox.VMID, shutdownTimeout)
+		if err != nil {
+			w.l.Warn().Err(err).Msg("failed to set shutdown timeout")
+		}
+	}
+
+	w.state.Store(&containerState{status: status})
+
+	w.task = parent.Subtask("idlewatcher."+r.TargetName(), true)
 
 	watcherMapMu.Lock()
 	defer watcherMapMu.Unlock()
-	w, ok := watcherMap[key]
-	if !ok {
-		client, err := docker.NewClient(cont.DockerHost)
-		if err != nil {
-			return nil, err
-		}
-
-		w = &Watcher{
-			Logger: logging.With().Str("name", cont.ContainerName).Logger(),
-			client: client,
-			task:   parent.Subtask("idlewatcher." + cont.ContainerName),
-			ticker: time.NewTicker(cfg.IdleTimeout),
-		}
-	}
-
-	// FIXME: possible race condition here
-	w.waker = waker
-	w.route = route
-	w.ticker.Reset(cfg.IdleTimeout)
-
-	if cont.Running {
-		w.setStarting()
-	} else {
-		w.setNapping()
-	}
-
-	if !ok {
-		w.stopByMethod = w.getStopCallback()
-		watcherMap[key] = w
-
-		go func() {
-			cause := w.watchUntilDestroy()
-
+	watcherMap[key] = w
+	go func() {
+		cause := w.watchUntilDestroy()
+		if cause.Is(causeContainerDestroy) {
 			watcherMapMu.Lock()
 			defer watcherMapMu.Unlock()
 			delete(watcherMap, key)
+			w.l.Info().Msg("idlewatcher stopped")
+		} else if !cause.Is(causeReload) {
+			gperr.LogError("idlewatcher stopped unexpectedly", cause, &w.l)
+		}
 
-			w.ticker.Stop()
-			w.client.Close()
-			w.task.Finish(cause)
-		}()
-	}
-
+		w.ticker.Stop()
+		w.provider.Close()
+		w.task.Finish(cause)
+	}()
+	w.l.Info().Msg("idlewatcher started")
 	return w, nil
 }
 
-func (w *Watcher) Config() *idlewatcher.Config {
-	return w.route.IdlewatcherConfig()
+func (w *Watcher) Key() string {
+	return w.cfg.Key()
 }
 
 func (w *Watcher) Wake() error {
 	return w.wakeIfStopped()
 }
 
-// WakeDebug logs a debug message related to waking the container.
-func (w *Watcher) WakeDebug() *zerolog.Event {
-	//nolint:zerologlint
-	return w.Debug().Str("action", "wake")
-}
-
-func (w *Watcher) WakeTrace() *zerolog.Event {
-	//nolint:zerologlint
-	return w.Trace().Str("action", "wake")
-}
-
-func (w *Watcher) WakeError(err error) {
-	w.Err(err).Str("action", "wake").Msg("error")
-}
-
 func (w *Watcher) wakeIfStopped() error {
-	if w.running() {
+	state := w.state.Load()
+	if state.status == idlewatcher.ContainerStatusRunning {
+		w.l.Debug().Msg("container is already running")
 		return nil
 	}
 
-	status, err := w.containerStatus()
-	if err != nil {
-		return err
+	ctx, cancel := context.WithTimeout(w.task.Context(), w.cfg.WakeTimeout)
+	defer cancel()
+	switch state.status {
+	case idlewatcher.ContainerStatusStopped:
+		w.l.Info().Msg("starting container")
+		return w.provider.ContainerStart(ctx)
+	case idlewatcher.ContainerStatusPaused:
+		w.l.Info().Msg("unpausing container")
+		return w.provider.ContainerUnpause(ctx)
+	default:
+		return gperr.Errorf("unexpected container status: %s", state.status)
+	}
+}
+
+func (w *Watcher) stopByMethod() error {
+	if !w.running() {
+		return nil
 	}
 
-	ctx, cancel := context.WithTimeout(w.task.Context(), w.Config().WakeTimeout)
+	cfg := w.cfg
+	ctx, cancel := context.WithTimeout(w.task.Context(), cfg.StopTimeout)
 	defer cancel()
 
-	// !Hard coded here since theres no constants from Docker API
-	switch status {
-	case "exited", "dead":
-		return w.containerStart(ctx)
-	case "paused":
-		return w.containerUnpause(ctx)
-	case "running":
-		return nil
-	default:
-		return gperr.Errorf("unexpected container status: %s", status)
-	}
-}
-
-func (w *Watcher) getStopCallback() StopCallback {
-	var cb func(context.Context) error
-	switch w.Config().StopMethod {
+	switch cfg.StopMethod {
 	case idlewatcher.StopMethodPause:
-		cb = w.containerPause
+		return w.provider.ContainerPause(ctx)
 	case idlewatcher.StopMethodStop:
-		cb = w.containerStop
+		return w.provider.ContainerStop(ctx, cfg.StopSignal, int(cfg.StopTimeout.Seconds()))
 	case idlewatcher.StopMethodKill:
-		cb = w.containerKill
+		return w.provider.ContainerKill(ctx, cfg.StopSignal)
 	default:
-		panic(errShouldNotReachHere)
-	}
-	return func() error {
-		ctx, cancel := context.WithTimeout(w.task.Context(), time.Duration(w.Config().StopTimeout)*time.Second)
-		defer cancel()
-		return cb(ctx)
+		return gperr.Errorf("unexpected stop method: %q", cfg.StopMethod)
 	}
 }
 
 func (w *Watcher) resetIdleTimer() {
-	w.Trace().Msg("reset idle timer")
-	w.ticker.Reset(w.Config().IdleTimeout)
-	w.lastReset = time.Now()
+	w.ticker.Reset(w.cfg.IdleTimeout)
+	w.lastReset.Store(time.Now())
 }
 
 func (w *Watcher) expires() time.Time {
-	return w.lastReset.Add(w.Config().IdleTimeout)
-}
-
-func (w *Watcher) getEventCh(ctx context.Context, dockerWatcher *watcher.DockerWatcher) (eventCh <-chan events.Event, errCh <-chan gperr.Error) {
-	eventCh, errCh = dockerWatcher.EventsWithOptions(ctx, watcher.DockerListOptions{
-		Filters: watcher.NewDockerFilter(
-			watcher.DockerFilterContainer,
-			watcher.DockerFilterContainerNameID(w.route.ContainerInfo().ContainerID),
-			watcher.DockerFilterStart,
-			watcher.DockerFilterStop,
-			watcher.DockerFilterDie,
-			watcher.DockerFilterKill,
-			watcher.DockerFilterDestroy,
-			watcher.DockerFilterPause,
-			watcher.DockerFilterUnpause,
-		),
-	})
-	return
+	if !w.running() {
+		return time.Time{}
+	}
+	return w.lastReset.Load().Add(w.cfg.IdleTimeout)
 }
 
 // watchUntilDestroy waits for the container to be created, started, or unpaused,
@@ -209,55 +250,34 @@ func (w *Watcher) getEventCh(ctx context.Context, dockerWatcher *watcher.DockerW
 //
 // it exits only if the context is canceled, the container is destroyed,
 // errors occurred on docker client, or route provider died (mainly caused by config reload).
-func (w *Watcher) watchUntilDestroy() (returnCause error) {
-	eventCtx, eventCancel := context.WithCancel(w.task.Context())
-	defer eventCancel()
-
-	dockerWatcher := watcher.NewDockerWatcher(w.client.DaemonHost())
-	dockerEventCh, dockerEventErrCh := w.getEventCh(eventCtx, dockerWatcher)
+func (w *Watcher) watchUntilDestroy() (returnCause gperr.Error) {
+	eventCh, errCh := w.provider.Watch(w.Task().Context())
 
 	for {
 		select {
 		case <-w.task.Context().Done():
-			return w.task.FinishCause()
-		case err := <-dockerEventErrCh:
-			if !err.Is(context.Canceled) {
-				gperr.LogError("idlewatcher error", err, &w.Logger)
-			}
+			return gperr.Wrap(w.task.FinishCause())
+		case err := <-errCh:
 			return err
-		case e := <-dockerEventCh:
+		case e := <-eventCh:
+			w.l.Debug().Stringer("action", e.Action).Msg("state changed")
+			if e.Action == events.ActionContainerDestroy {
+				return causeContainerDestroy
+			}
+			w.resetIdleTimer()
 			switch {
-			case e.Action == events.ActionContainerDestroy:
-				w.setError(errors.New("container destroyed"))
-				w.Info().Str("reason", "container destroyed").Msg("watcher stopped")
-				return errors.New("container destroyed")
-			// create / start / unpause
-			case e.Action.IsContainerWake():
+			case e.Action.IsContainerStart(): // create / start / unpause
 				w.setStarting()
-				w.resetIdleTimer()
-				w.Info().Msg("awaken")
-			case e.Action.IsContainerSleep(): // stop / pause / kil
-				w.setNapping()
-				w.resetIdleTimer()
+				w.l.Info().Msg("awaken")
+			case e.Action.IsContainerStop(): // stop / kill / die
+				w.setNapping(idlewatcher.ContainerStatusStopped)
+				w.ticker.Stop()
+			case e.Action.IsContainerPause(): // pause
+				w.setNapping(idlewatcher.ContainerStatusPaused)
 				w.ticker.Stop()
 			default:
-				w.Error().Msg("unexpected docker event: " + e.String())
+				w.l.Error().Stringer("action", e.Action).Msg("unexpected container action")
 			}
-			// container name changed should also change the container id
-			// if w.ContainerName != e.ActorName {
-			// 	w.Debug().Msgf("renamed %s -> %s", w.ContainerName, e.ActorName)
-			// 	w.ContainerName = e.ActorName
-			// }
-			// if w.ContainerID != e.ActorID {
-			// 	w.Debug().Msgf("id changed %s -> %s", w.ContainerID, e.ActorID)
-			// 	w.ContainerID = e.ActorID
-			// 	// recreate event stream
-			// 	eventCancel()
-
-			// 	eventCtx, eventCancel = context.WithCancel(w.task.Context())
-			// 	defer eventCancel()
-			// 	dockerEventCh, dockerEventErrCh = w.getEventCh(eventCtx, dockerWatcher)
-			// }
 		case <-w.ticker.C:
 			w.ticker.Stop()
 			if w.running() {
@@ -269,11 +289,18 @@ func (w *Watcher) watchUntilDestroy() (returnCause error) {
 					if errors.Is(err, context.DeadlineExceeded) {
 						err = errors.New("timeout waiting for container to stop, please set a higher value for `stop_timeout`")
 					}
-					w.Err(err).Msgf("container stop with method %q failed", w.Config().StopMethod)
+					w.l.Err(err).Msgf("container stop with method %q failed", w.cfg.StopMethod)
 				default:
-					w.Info().Str("reason", "idle timeout").Msg("container stopped")
+					w.l.Info().Str("reason", "idle timeout").Msg("container stopped")
 				}
 			}
 		}
 	}
 }
+
+func fmtErr(err error) string {
+	if err == nil {
+		return ""
+	}
+	return err.Error()
+}