improved idlewatcher support for API-like services, fixed idlewaker proxying to zero port

.github/workflows/docker-image.yml

@@ -11,7 +11,7 @@ env:
 jobs:
     build:
         name: Build multi-platform Docker image
-        runs-on: self-hosted
+        runs-on: ubuntu-22.04
 
         permissions:
             contents: read
@@ -85,7 +85,7 @@ jobs:
                   if-no-files-found: error
                   retention-days: 1
     merge:
-        runs-on: self-hosted
+        runs-on: ubuntu-22.04
         needs:
             - build
         permissions:

.gitignore

@@ -1,4 +1,5 @@
 compose.yml
+*.compose.yml
 
 config*/
 certs*/
@@ -20,3 +21,4 @@ todo.md
 .*.swp
 .aider*
 mtrace.json
+.env

Dockerfile

@@ -1,5 +1,5 @@
 # Stage 1: Builder
-FROM golang:1.23.1-alpine AS builder
+FROM golang:1.23.2-alpine AS builder
 RUN apk add --no-cache tzdata make
 
 WORKDIR /src

Makefile

@@ -30,6 +30,9 @@ get:
 debug:
 	make build && sudo GOPROXY_DEBUG=1 bin/go-proxy
 
+mtrace:
+	bin/go-proxy debug-ls-mtrace > mtrace.json
+
 run-test:
 	make build && sudo GOPROXY_TEST=1 bin/go-proxy
 

README.md

@@ -72,13 +72,16 @@ _Join our [Discord](https://discord.gg/umReR62nRd) for help and discussions_
 
 4.  Setup `docker-socket-proxy` other docker nodes _(if any)_ (see [Multi docker nodes setup](https://github.com/yusing/go-proxy/wiki/Configurations#multi-docker-nodes-setup)) and then them inside `config.yml`
 
-5.  Done. You may now do some extra configuration
+5.  Run go-proxy `docker compose up -d` 
+    then list all routes to see if further configurations are needed:
+    `docker exec go-proxy /app/go-proxy ls-routes`
+
+6.  You may now do some extra configuration
     -   With text editor (e.g. Visual Studio Code)
     -   With Web UI via `http://localhost:3000` or `https://gp.y.z`
     -   For more info, [See Wiki]([wiki](https://github.com/yusing/go-proxy/wiki))
 
 [🔼Back to top](#table-of-content)
-|
 
 ### Use JSON Schema in VSCode
 

compose.example.yml

@@ -14,7 +14,7 @@ services:
         # Make sure the value is same as `GOPROXY_API_ADDR` below (if you have changed it)
         #
         # environment:
-        #     NEXT_PUBLIC_GOPROXY_API_ADDR: 127.0.0.1:8888
+        #     GOPROXY_API_ADDR: 127.0.0.1:8888
     app:
         image: ghcr.io/yusing/go-proxy:latest
         container_name: go-proxy

examples/microbin.yml

@@ -7,7 +7,7 @@ services:
         limits:
           memory: 256M
     env_file: .env
-    image: docker.i.sh/danielszabo99/microbin:latest
+    image: danielszabo99/microbin:latest
     ports:
       - 8080
     restart: unless-stopped

go.mod

@@ -1,13 +1,13 @@
 module github.com/yusing/go-proxy
 
-go 1.23.1
+go 1.23.2
 
 require (
 	github.com/coder/websocket v1.8.12
 	github.com/docker/cli v27.3.1+incompatible
 	github.com/docker/docker v27.3.1+incompatible
 	github.com/fsnotify/fsnotify v1.7.0
-	github.com/go-acme/lego/v4 v4.19.0
+	github.com/go-acme/lego/v4 v4.19.2
 	github.com/puzpuzpuz/xsync/v3 v3.4.0
 	github.com/santhosh-tekuri/jsonschema v1.2.4
 	github.com/sirupsen/logrus v1.9.3

go.sum

@@ -29,8 +29,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
-github.com/go-acme/lego/v4 v4.19.0 h1:c7YabBOwoa2URsPiCNGQsdzQnbd8Y23B4/66gxh4H7c=
-github.com/go-acme/lego/v4 v4.19.0/go.mod h1:wtDe3dDkmV4/oI2nydpNXSJpvV10J9RCyZ6MbYxNtlQ=
+github.com/go-acme/lego/v4 v4.19.2 h1:Y8hrmMvWETdqzzkRly7m98xtPJJivWFsgWi8fcvZo+Y=
+github.com/go-acme/lego/v4 v4.19.2/go.mod h1:wtDe3dDkmV4/oI2nydpNXSJpvV10J9RCyZ6MbYxNtlQ=
 github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E=
 github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=

internal/docker/idlewatcher/html/loading_page.html

@@ -66,22 +66,23 @@
     <body>
         <script>
             window.onload = async function () {
-                let result = await fetch(window.location.href, {
+                let resp = await fetch(window.location.href, {
                     headers: {
-                        {{ range $key, $value := .RequestHeaders }}
-                        '{{ $key }}' : {{ $value }}
-                        {{ end }}
+                        "{{.CheckRedirectHeader}}": "1",
                     },
-                }).then((resp) => resp.text())
-                    .catch((err) => {
-                        document.getElementById("message").innerText = err;
-                    });
-                if (result) {
-                    document.documentElement.innerHTML = result
+                });
+                if (resp.ok) {
+                    window.location.href = resp.url;
+                } else {
+                    document.getElementById("message").innerText =
+                        await resp.text();
+                    document
+                        .getElementById("spinner")
+                        .classList.replace("spinner", "error");
                 }
             };
         </script>
-        <div class="{{.SpinnerClass}}"></div>
-        <div class="message">{{.Message}}</div>
+        <div id="spinner" class="spinner"></div>
+        <div id="message" class="message">{{.Message}}</div>
     </body>
 </html>

internal/docker/idlewatcher/http.go

@@ -4,84 +4,35 @@ import (
 	"bytes"
 	_ "embed"
 	"fmt"
-	"io"
-	"net/http"
 	"strings"
 	"text/template"
 )
 
 type templateData struct {
-	Title          string
-	Message        string
-	RequestHeaders http.Header
-	SpinnerClass   string
+	CheckRedirectHeader string
+	Title               string
+	Message             string
 }
 
 //go:embed html/loading_page.html
 var loadingPage []byte
 var loadingPageTmpl = template.Must(template.New("loading_page").Parse(string(loadingPage)))
 
-const (
-	htmlContentType = "text/html; charset=utf-8"
+const headerCheckRedirect = "X-GoProxy-Check-Redirect"
 
-	errPrefix = "\u1000"
-
-	headerGoProxyTargetURL = "X-GoProxy-Target"
-	headerContentType      = "Content-Type"
-
-	spinnerClassSpinner   = "spinner"
-	spinnerClassErrorSign = "error"
-)
-
-func (w *watcher) makeSuccResp(redirectURL string, resp *http.Response) (*http.Response, error) {
-	h := make(http.Header)
-	h.Set("Location", redirectURL)
-	h.Set("Content-Length", "0")
-	h.Set(headerContentType, htmlContentType)
-	return &http.Response{
-		StatusCode: http.StatusTemporaryRedirect,
-		Header:     h,
-		Body:       http.NoBody,
-		TLS:        resp.TLS,
-	}, nil
-}
-
-func (w *watcher) makeErrResp(errFmt string, args ...any) (*http.Response, error) {
-	return w.makeResp(errPrefix+errFmt, args...)
-}
-
-func (w *watcher) makeResp(format string, args ...any) (*http.Response, error) {
+func (w *watcher) makeRespBody(format string, args ...any) []byte {
 	msg := fmt.Sprintf(format, args...)
 
 	data := new(templateData)
+	data.CheckRedirectHeader = headerCheckRedirect
 	data.Title = w.ContainerName
 	data.Message = strings.ReplaceAll(msg, "\n", "<br>")
 	data.Message = strings.ReplaceAll(data.Message, " ", "&ensp;")
-	data.RequestHeaders = make(http.Header)
-	data.RequestHeaders.Add(headerGoProxyTargetURL, "window.location.href")
-	if strings.HasPrefix(data.Message, errPrefix) {
-		data.Message = strings.TrimLeft(data.Message, errPrefix)
-		data.SpinnerClass = spinnerClassErrorSign
-	} else {
-		data.SpinnerClass = spinnerClassSpinner
-	}
 
 	buf := bytes.NewBuffer(make([]byte, 128)) // more than enough
 	err := loadingPageTmpl.Execute(buf, data)
-	if err != nil { // should never happen
+	if err != nil { // should never happen in production
 		panic(err)
 	}
-	return &http.Response{
-		StatusCode: http.StatusAccepted,
-		Header: http.Header{
-			headerContentType: {htmlContentType},
-			"Cache-Control": {
-				"no-cache",
-				"no-store",
-				"must-revalidate",
-			},
-		},
-		Body:          io.NopCloser(buf),
-		ContentLength: int64(buf.Len()),
-	}, nil
+	return buf.Bytes()
 }

internal/docker/idlewatcher/round_trip.go

@@ -1,82 +0,0 @@
-package idlewatcher
-
-import (
-	"context"
-	"net/http"
-)
-
-type (
-	roundTripper struct {
-		patched roundTripFunc
-	}
-	roundTripFunc func(*http.Request) (*http.Response, error)
-)
-
-func (rt roundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
-	return rt.patched(req)
-}
-
-func (w *watcher) roundTrip(origRoundTrip roundTripFunc, req *http.Request) (*http.Response, error) {
-	// target site is ready, passthrough
-	if w.ready.Load() {
-		return origRoundTrip(req)
-	}
-
-	// initial request
-	targetUrl := req.Header.Get(headerGoProxyTargetURL)
-	if targetUrl == "" {
-		return w.makeResp(
-			"%s is starting... Please wait",
-			w.ContainerName,
-		)
-	}
-
-	w.l.Debug("serving event")
-
-	// stream request
-	rtDone := make(chan *http.Response, 1)
-	ctx, cancel := context.WithTimeout(req.Context(), w.WakeTimeout)
-	defer cancel()
-
-	// loop original round trip until success in a goroutine
-	go func() {
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case <-w.ctx.Done():
-				return
-			default:
-				// wake the container and reset idle timer
-				select {
-				case w.wakeCh <- struct{}{}:
-				default:
-				}
-				resp, err := origRoundTrip(req)
-				if err == nil {
-					w.ready.Store(true)
-					rtDone <- resp
-					return
-				}
-			}
-		}
-	}()
-
-	for {
-		select {
-		case resp := <-rtDone:
-			return w.makeSuccResp(targetUrl, resp)
-		case err := <-w.wakeDone:
-			if err != nil {
-				return w.makeErrResp("error waking up %s\n%s", w.ContainerName, err.Error())
-			}
-		case <-ctx.Done():
-			if ctx.Err() == context.DeadlineExceeded {
-				return w.makeErrResp("Timed out waiting for %s to fully wake", w.ContainerName)
-			}
-			return w.makeErrResp("idlewatcher has stopped\n%s", w.ctx.Err())
-		case <-w.ctx.Done():
-			return w.makeErrResp("idlewatcher has stopped\n%s", w.ctx.Err())
-		}
-	}
-}

internal/docker/idlewatcher/waker.go

@@ -0,0 +1,136 @@
+package idlewatcher
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net/http"
+	"strconv"
+	"time"
+
+	gphttp "github.com/yusing/go-proxy/internal/net/http"
+)
+
+type Waker struct {
+	*watcher
+
+	client *http.Client
+	rp     *gphttp.ReverseProxy
+}
+
+func NewWaker(w *watcher, rp *gphttp.ReverseProxy) *Waker {
+	tr := &http.Transport{}
+	if w.NoTLSVerify {
+		tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	}
+	orig := rp.ServeHTTP
+	// workaround for stopped containers port become zero
+	rp.ServeHTTP = func(rw http.ResponseWriter, r *http.Request) {
+		if rp.TargetURL.Port() == "0" {
+			port, ok := portHistoryMap.Load(w.Alias)
+			if !ok {
+				w.l.Errorf("port history not found for %s", w.Alias)
+				http.Error(rw, "internal server error", http.StatusInternalServerError)
+				return
+			}
+			rp.TargetURL.Host = fmt.Sprintf("%s:%v", rp.TargetURL.Hostname(), port)
+		}
+		orig(rw, r)
+	}
+	return &Waker{
+		watcher: w,
+		client: &http.Client{
+			Timeout:   1 * time.Second,
+			Transport: tr,
+		},
+		rp: rp,
+	}
+}
+
+func (w *Waker) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
+	w.wake(w.rp.ServeHTTP, rw, r)
+}
+
+func (w *Waker) wake(next http.HandlerFunc, rw http.ResponseWriter, r *http.Request) {
+	w.resetIdleTimer()
+
+	// pass through if container is ready
+	if w.ready.Load() {
+		next(rw, r)
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(r.Context(), w.WakeTimeout)
+	defer cancel()
+
+	accept := gphttp.GetAccept(r.Header)
+	acceptHTML := r.Method == http.MethodGet && accept.AcceptHTML()
+
+	isCheckRedirect := r.Header.Get(headerCheckRedirect) != ""
+	if !isCheckRedirect && acceptHTML {
+		// Send a loading response to the client
+		body := w.makeRespBody("%s waking up...", w.ContainerName)
+		rw.Header().Set("Content-Type", "text/html; charset=utf-8")
+		rw.Header().Set("Content-Length", strconv.Itoa(len(body)))
+		rw.Header().Add("Cache-Control", "no-cache")
+		rw.Header().Add("Cache-Control", "no-store")
+		rw.Header().Add("Cache-Control", "must-revalidate")
+		rw.Write(body)
+		return
+	}
+
+	// wake the container and reset idle timer
+	// also wait for another wake request
+	w.wakeCh <- struct{}{}
+
+	if <-w.wakeDone != nil {
+		http.Error(rw, "Error sending wake request", http.StatusInternalServerError)
+		return
+	}
+
+	// maybe another request came in while we were waiting for the wake
+	if w.ready.Load() {
+		if isCheckRedirect {
+			rw.WriteHeader(http.StatusOK)
+		} else {
+			next(rw, r)
+		}
+		return
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			http.Error(rw, "Waking timed out", http.StatusGatewayTimeout)
+			return
+		default:
+		}
+
+		wakeReq, err := http.NewRequestWithContext(
+			ctx,
+			http.MethodHead,
+			w.URL.String(),
+			nil,
+		)
+		if err != nil {
+			w.l.Errorf("new request err to %s: %s", r.URL, err)
+			http.Error(rw, "Internal server error", http.StatusInternalServerError)
+			return
+		}
+
+		wakeResp, err := w.client.Do(wakeReq)
+		if err == nil && wakeResp.StatusCode != http.StatusServiceUnavailable {
+			w.ready.Store(true)
+			w.l.Debug("awaken")
+			if isCheckRedirect {
+				rw.WriteHeader(http.StatusOK)
+			} else {
+				next(rw, r)
+			}
+			return
+		}
+
+		// retry until the container is ready or timeout
+		time.Sleep(100 * time.Millisecond)
+	}
+}

internal/docker/idlewatcher/watcher.go

@@ -2,7 +2,6 @@ package idlewatcher
 
 import (
 	"context"
-	"net/http"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -13,6 +12,7 @@ import (
 	E "github.com/yusing/go-proxy/internal/error"
 	P "github.com/yusing/go-proxy/internal/proxy"
 	PT "github.com/yusing/go-proxy/internal/proxy/fields"
+	F "github.com/yusing/go-proxy/internal/utils/functional"
 	W "github.com/yusing/go-proxy/internal/watcher"
 )
 
@@ -27,6 +27,7 @@ type (
 
 		wakeCh   chan struct{}
 		wakeDone chan E.NestedError
+		ticker   *time.Ticker
 
 		ctx      context.Context
 		cancel   context.CancelFunc
@@ -45,9 +46,11 @@ var (
 	mainLoopCancel context.CancelFunc
 	mainLoopWg     sync.WaitGroup
 
-	watcherMap   = make(map[string]*watcher)
+	watcherMap   = F.NewMapOf[string, *watcher]()
 	watcherMapMu sync.Mutex
 
+	portHistoryMap = F.NewMapOf[PT.Alias, string]()
+
 	newWatcherCh = make(chan *watcher)
 
 	logger = logrus.WithField("module", "idle_watcher")
@@ -65,7 +68,11 @@ func Register(entry *P.ReverseProxyEntry) (*watcher, E.NestedError) {
 
 	key := entry.ContainerID
 
-	if w, ok := watcherMap[key]; ok {
+	if entry.URL.Port() != "0" {
+		portHistoryMap.Store(entry.Alias, entry.URL.Port())
+	}
+
+	if w, ok := watcherMap.Load(key); ok {
 		w.refCount.Add(1)
 		w.ReverseProxyEntry = entry
 		return w, nil
@@ -80,14 +87,15 @@ func Register(entry *P.ReverseProxyEntry) (*watcher, E.NestedError) {
 		ReverseProxyEntry: entry,
 		client:            client,
 		refCount:          &sync.WaitGroup{},
-		wakeCh:            make(chan struct{}),
+		wakeCh:            make(chan struct{}, 1),
 		wakeDone:          make(chan E.NestedError),
+		ticker:            time.NewTicker(entry.IdleTimeout),
 		l:                 logger.WithField("container", entry.ContainerName),
 	}
 	w.refCount.Add(1)
 	w.stopByMethod = w.getStopCallback()
 
-	watcherMap[key] = w
+	watcherMap.Store(key, w)
 
 	go func() {
 		newWatcherCh <- w
@@ -96,10 +104,8 @@ func Register(entry *P.ReverseProxyEntry) (*watcher, E.NestedError) {
 	return w, nil
 }
 
-func Unregister(entry *P.ReverseProxyEntry) {
-	if w, ok := watcherMap[entry.ContainerID]; ok {
-		w.refCount.Add(-1)
-	}
+func (w *watcher) Unregister() {
+	w.refCount.Add(-1)
 }
 
 func Start() {
@@ -119,8 +125,7 @@ func Start() {
 				w.watchUntilCancel()
 				w.refCount.Wait() // wait for 0 ref count
 
-				w.client.Close()
-				delete(watcherMap, w.ContainerID)
+				watcherMap.Delete(w.ContainerID)
 				w.l.Debug("unregistered")
 				mainLoopWg.Done()
 			}()
@@ -133,12 +138,6 @@ func Stop() {
 	mainLoopWg.Wait()
 }
 
-func (w *watcher) PatchRoundTripper(rtp http.RoundTripper) roundTripper {
-	return roundTripper{patched: func(r *http.Request) (*http.Response, error) {
-		return w.roundTrip(rtp.RoundTrip, r)
-	}}
-}
-
 func (w *watcher) containerStop() error {
 	return w.client.ContainerStop(w.ctx, w.ContainerID, container.StopOptions{
 		Signal:  string(w.StopSignal),
@@ -216,10 +215,14 @@ func (w *watcher) getStopCallback() StopCallback {
 	}
 }
 
+func (w *watcher) resetIdleTimer() {
+	w.ticker.Reset(w.IdleTimeout)
+}
+
 func (w *watcher) watchUntilCancel() {
 	defer close(w.wakeCh)
 
-	w.ctx, w.cancel = context.WithCancel(context.Background())
+	w.ctx, w.cancel = context.WithCancel(mainLoopCtx)
 
 	dockerWatcher := W.NewDockerWatcherWithClient(w.client)
 	dockerEventCh, dockerEventErrCh := dockerWatcher.EventsWithOptions(w.ctx, W.DockerListOptions{
@@ -234,14 +237,11 @@ func (w *watcher) watchUntilCancel() {
 			W.DockerFilterUnpause,
 		),
 	})
-
-	ticker := time.NewTicker(w.IdleTimeout)
-	defer ticker.Stop()
+	defer w.ticker.Stop()
+	defer w.client.Close()
 
 	for {
 		select {
-		case <-mainLoopCtx.Done():
-			w.cancel()
 		case <-w.ctx.Done():
 			w.l.Debug("stopped")
 			return
@@ -254,31 +254,28 @@ func (w *watcher) watchUntilCancel() {
 			// create / start / unpause
 			case e.Action.IsContainerWake():
 				w.ContainerRunning = true
-				ticker.Reset(w.IdleTimeout)
+				w.resetIdleTimer()
 				w.l.Info(e)
-			default: // stop / pause / kill
+			default: // stop / pause / kil
 				w.ContainerRunning = false
-				ticker.Stop()
+				w.ticker.Stop()
 				w.ready.Store(false)
 				w.l.Info(e)
 			}
-		case <-ticker.C:
+		case <-w.ticker.C:
 			w.l.Debug("idle timeout")
-			ticker.Stop()
+			w.ticker.Stop()
 			if err := w.stopByMethod(); err != nil && err.IsNot(context.Canceled) {
 				w.l.Error(E.FailWith("stop", err).Extraf("stop method: %s", w.StopMethod))
 			}
 		case <-w.wakeCh:
 			w.l.Debug("wake signal received")
-			ticker.Reset(w.IdleTimeout)
+			w.resetIdleTimer()
 			err := w.wakeIfStopped()
-			if err != nil && err.IsNot(context.Canceled) {
+			if err != nil {
 				w.l.Error(E.FailWith("wake", err))
 			}
-			select {
-			case w.wakeDone <- err: // this is passed to roundtrip
-			default:
-			}
+			w.wakeDone <- err
 		}
 	}
 }

internal/net/http/content_type.go

@@ -6,6 +6,7 @@ import (
 )
 
 type ContentType string
+type AcceptContentType []ContentType
 
 func GetContentType(h http.Header) ContentType {
 	ct := h.Get("Content-Type")
@@ -19,6 +20,18 @@ func GetContentType(h http.Header) ContentType {
 	return ContentType(ct)
 }
 
+func GetAccept(h http.Header) AcceptContentType {
+	var accepts []ContentType
+	for _, v := range h["Accept"] {
+		ct, _, err := mime.ParseMediaType(v)
+		if err != nil {
+			continue
+		}
+		accepts = append(accepts, ContentType(ct))
+	}
+	return accepts
+}
+
 func (ct ContentType) IsHTML() bool {
 	return ct == "text/html" || ct == "application/xhtml+xml"
 }
@@ -30,3 +43,34 @@ func (ct ContentType) IsJSON() bool {
 func (ct ContentType) IsPlainText() bool {
 	return ct == "text/plain"
 }
+
+func (act AcceptContentType) IsEmpty() bool {
+	return len(act) == 0
+}
+
+func (act AcceptContentType) AcceptHTML() bool {
+	for _, v := range act {
+		if v.IsHTML() || v == "text/*" || v == "*/*" {
+			return true
+		}
+	}
+	return false
+}
+
+func (act AcceptContentType) AcceptJSON() bool {
+	for _, v := range act {
+		if v.IsJSON() || v == "*/*" {
+			return true
+		}
+	}
+	return false
+}
+
+func (act AcceptContentType) AcceptPlainText() bool {
+	for _, v := range act {
+		if v.IsPlainText() || v == "text/*" || v == "*/*" {
+			return true
+		}
+	}
+	return false
+}

internal/net/http/content_type_test.go

@@ -0,0 +1,41 @@
+package http
+
+import (
+	"net/http"
+	"testing"
+
+	. "github.com/yusing/go-proxy/internal/utils/testing"
+)
+
+func TestContentTypes(t *testing.T) {
+	ExpectTrue(t, GetContentType(http.Header{"Content-Type": {"text/html"}}).IsHTML())
+	ExpectTrue(t, GetContentType(http.Header{"Content-Type": {"text/html; charset=utf-8"}}).IsHTML())
+	ExpectTrue(t, GetContentType(http.Header{"Content-Type": {"application/xhtml+xml"}}).IsHTML())
+	ExpectFalse(t, GetContentType(http.Header{"Content-Type": {"text/plain"}}).IsHTML())
+
+	ExpectTrue(t, GetContentType(http.Header{"Content-Type": {"application/json"}}).IsJSON())
+	ExpectTrue(t, GetContentType(http.Header{"Content-Type": {"application/json; charset=utf-8"}}).IsJSON())
+	ExpectFalse(t, GetContentType(http.Header{"Content-Type": {"text/html"}}).IsJSON())
+
+	ExpectTrue(t, GetContentType(http.Header{"Content-Type": {"text/plain"}}).IsPlainText())
+	ExpectTrue(t, GetContentType(http.Header{"Content-Type": {"text/plain; charset=utf-8"}}).IsPlainText())
+	ExpectFalse(t, GetContentType(http.Header{"Content-Type": {"text/html"}}).IsPlainText())
+}
+
+func TestAcceptContentTypes(t *testing.T) {
+	ExpectTrue(t, GetAccept(http.Header{"Accept": {"text/html", "text/plain"}}).AcceptPlainText())
+	ExpectTrue(t, GetAccept(http.Header{"Accept": {"text/html", "text/plain; charset=utf-8"}}).AcceptPlainText())
+	ExpectTrue(t, GetAccept(http.Header{"Accept": {"text/html", "text/plain"}}).AcceptHTML())
+	ExpectTrue(t, GetAccept(http.Header{"Accept": {"application/json"}}).AcceptJSON())
+	ExpectTrue(t, GetAccept(http.Header{"Accept": {"*/*"}}).AcceptPlainText())
+	ExpectTrue(t, GetAccept(http.Header{"Accept": {"*/*"}}).AcceptHTML())
+	ExpectTrue(t, GetAccept(http.Header{"Accept": {"*/*"}}).AcceptJSON())
+	ExpectTrue(t, GetAccept(http.Header{"Accept": {"text/*"}}).AcceptPlainText())
+	ExpectTrue(t, GetAccept(http.Header{"Accept": {"text/*"}}).AcceptHTML())
+
+	ExpectFalse(t, GetAccept(http.Header{"Accept": {"text/plain"}}).AcceptHTML())
+	ExpectFalse(t, GetAccept(http.Header{"Accept": {"text/plain; charset=utf-8"}}).AcceptHTML())
+	ExpectFalse(t, GetAccept(http.Header{"Accept": {"text/html"}}).AcceptPlainText())
+	ExpectFalse(t, GetAccept(http.Header{"Accept": {"text/html"}}).AcceptJSON())
+	ExpectFalse(t, GetAccept(http.Header{"Accept": {"text/*"}}).AcceptJSON())
+}

internal/net/http/middleware/custom_error_page.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"github.com/yusing/go-proxy/internal/api/v1/error_page"
-	gpHTTP "github.com/yusing/go-proxy/internal/net/http"
+	gphttp "github.com/yusing/go-proxy/internal/net/http"
 )
 
 var CustomErrorPage = &Middleware{
@@ -21,8 +21,8 @@ var CustomErrorPage = &Middleware{
 	},
 	modifyResponse: func(resp *Response) error {
 		// only handles non-success status code and html/plain content type
-		contentType := gpHTTP.GetContentType(resp.Header)
-		if !gpHTTP.IsSuccess(resp.StatusCode) && (contentType.IsHTML() || contentType.IsPlainText()) {
+		contentType := gphttp.GetContentType(resp.Header)
+		if !gphttp.IsSuccess(resp.StatusCode) && (contentType.IsHTML() || contentType.IsPlainText()) {
 			errorPage, ok := error_page.GetErrorPageByStatus(resp.StatusCode)
 			if ok {
 				errPageLogger.Debugf("error page for status %d loaded", resp.StatusCode)
@@ -46,8 +46,8 @@ func ServeStaticErrorPageFile(w http.ResponseWriter, r *http.Request) bool {
 	if path != "" && path[0] != '/' {
 		path = "/" + path
 	}
-	if strings.HasPrefix(path, gpHTTP.StaticFilePathPrefix) {
-		filename := path[len(gpHTTP.StaticFilePathPrefix):]
+	if strings.HasPrefix(path, gphttp.StaticFilePathPrefix) {
+		filename := path[len(gphttp.StaticFilePathPrefix):]
 		file, ok := error_page.GetStaticFile(filename)
 		if !ok {
 			errPageLogger.Errorf("unable to load resource %s", filename)

internal/net/http/middleware/forward_auth.go

@@ -14,7 +14,7 @@ import (
 	"time"
 
 	E "github.com/yusing/go-proxy/internal/error"
-	gpHTTP "github.com/yusing/go-proxy/internal/net/http"
+	gphttp "github.com/yusing/go-proxy/internal/net/http"
 )
 
 type (
@@ -58,7 +58,7 @@ func NewForwardAuthfunc(optsRaw OptionsRaw) (*Middleware, E.NestedError) {
 	if ok {
 		tr = tr.Clone()
 	} else {
-		tr = gpHTTP.DefaultTransport.Clone()
+		tr = gphttp.DefaultTransport.Clone()
 	}
 
 	fa.client = http.Client{
@@ -72,7 +72,7 @@ func NewForwardAuthfunc(optsRaw OptionsRaw) (*Middleware, E.NestedError) {
 }
 
 func (fa *forwardAuth) forward(next http.HandlerFunc, w ResponseWriter, req *Request) {
-	gpHTTP.RemoveHop(req.Header)
+	gphttp.RemoveHop(req.Header)
 
 	faReq, err := http.NewRequestWithContext(
 		req.Context(),
@@ -86,10 +86,10 @@ func (fa *forwardAuth) forward(next http.HandlerFunc, w ResponseWriter, req *Req
 		return
 	}
 
-	gpHTTP.CopyHeader(faReq.Header, req.Header)
-	gpHTTP.RemoveHop(faReq.Header)
+	gphttp.CopyHeader(faReq.Header, req.Header)
+	gphttp.RemoveHop(faReq.Header)
 
-	faReq.Header = gpHTTP.FilterHeaders(faReq.Header, fa.AuthResponseHeaders)
+	faReq.Header = gphttp.FilterHeaders(faReq.Header, fa.AuthResponseHeaders)
 	fa.setAuthHeaders(req, faReq)
 	fa.m.AddTraceRequest("forward auth request", faReq)
 
@@ -110,8 +110,8 @@ func (fa *forwardAuth) forward(next http.HandlerFunc, w ResponseWriter, req *Req
 
 	if faResp.StatusCode < http.StatusOK || faResp.StatusCode >= http.StatusMultipleChoices {
 		fa.m.AddTraceResponse("forward auth response", faResp)
-		gpHTTP.CopyHeader(w.Header(), faResp.Header)
-		gpHTTP.RemoveHop(w.Header())
+		gphttp.CopyHeader(w.Header(), faResp.Header)
+		gphttp.RemoveHop(w.Header())
 
 		redirectURL, err := faResp.Location()
 		if err != nil {
@@ -148,7 +148,7 @@ func (fa *forwardAuth) forward(next http.HandlerFunc, w ResponseWriter, req *Req
 		return
 	}
 
-	next.ServeHTTP(gpHTTP.NewModifyResponseWriter(w, req, func(resp *Response) error {
+	next.ServeHTTP(gphttp.NewModifyResponseWriter(w, req, func(resp *Response) error {
 		fa.setAuthCookies(resp, authCookies)
 		return nil
 	}), req)

internal/net/http/middleware/middleware.go

@@ -6,15 +6,15 @@ import (
 	"net/http"
 
 	E "github.com/yusing/go-proxy/internal/error"
-	gpHTTP "github.com/yusing/go-proxy/internal/net/http"
+	gphttp "github.com/yusing/go-proxy/internal/net/http"
 	U "github.com/yusing/go-proxy/internal/utils"
 )
 
 type (
 	Error = E.NestedError
 
-	ReverseProxy   = gpHTTP.ReverseProxy
-	ProxyRequest   = gpHTTP.ProxyRequest
+	ReverseProxy   = gphttp.ReverseProxy
+	ProxyRequest   = gphttp.ProxyRequest
 	Request        = http.Request
 	Response       = http.Response
 	ResponseWriter = http.ResponseWriter

internal/net/http/middleware/test_utils.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/yusing/go-proxy/internal/common"
 	E "github.com/yusing/go-proxy/internal/error"
-	gpHTTP "github.com/yusing/go-proxy/internal/net/http"
+	gphttp "github.com/yusing/go-proxy/internal/net/http"
 )
 
 //go:embed test_data/sample_headers.json
@@ -110,7 +110,7 @@ func newMiddlewareTest(middleware *Middleware, args *testArgs) (*TestResult, E.N
 	} else {
 		proxyURL, _ = url.Parse("https://" + testHost) // dummy url, no actual effect
 	}
-	rp := gpHTTP.NewReverseProxy(proxyURL, rr)
+	rp := gphttp.NewReverseProxy(proxyURL, rr)
 	mid, setOptErr := middleware.WithOptionsClone(args.middlewareOpt)
 	if setOptErr != nil {
 		return nil, setOptErr

internal/net/http/middleware/trace.go

@@ -5,7 +5,7 @@ import (
 	"sync"
 	"time"
 
-	gpHTTP "github.com/yusing/go-proxy/internal/net/http"
+	gphttp "github.com/yusing/go-proxy/internal/net/http"
 	U "github.com/yusing/go-proxy/internal/utils"
 )
 
@@ -36,7 +36,7 @@ func (tr *Trace) WithRequest(req *Request) *Trace {
 		return nil
 	}
 	tr.URL = req.RequestURI
-	tr.ReqHeaders = gpHTTP.HeaderToMap(req.Header)
+	tr.ReqHeaders = gphttp.HeaderToMap(req.Header)
 	return tr
 }
 
@@ -45,8 +45,8 @@ func (tr *Trace) WithResponse(resp *Response) *Trace {
 		return nil
 	}
 	tr.URL = resp.Request.RequestURI
-	tr.ReqHeaders = gpHTTP.HeaderToMap(resp.Request.Header)
-	tr.RespHeaders = gpHTTP.HeaderToMap(resp.Header)
+	tr.ReqHeaders = gphttp.HeaderToMap(resp.Request.Header)
+	tr.RespHeaders = gphttp.HeaderToMap(resp.Header)
 	tr.RespStatus = resp.StatusCode
 	return tr
 }

internal/net/http/reverse_proxy_mod.go

@@ -69,36 +69,6 @@ type ProxyRequest struct {
 // 1xx responses are forwarded to the client if the underlying
 // transport supports ClientTrace.Got1xxResponse.
 type ReverseProxy struct {
-	// Director is a function which modifies
-	// the request into a new request to be sent
-	// using Transport. Its response is then copied
-	// back to the original client unmodified.
-	// Director must not access the provided Request
-	// after returning.
-	//
-	// By default, the X-Forwarded-For header is set to the
-	// value of the client IP address. If an X-Forwarded-For
-	// header already exists, the client IP is appended to the
-	// existing values. As a special case, if the header
-	// exists in the Request.Header map but has a nil value
-	// (such as when set by the Director func), the X-Forwarded-For
-	// header is not modified.
-	//
-	// To prevent IP spoofing, be sure to delete any pre-existing
-	// X-Forwarded-For header coming from the client or
-	// an untrusted proxy.
-	//
-	// Hop-by-hop headers are removed from the request after
-	// Director returns, which can remove headers added by
-	// Director. Use a Rewrite function instead to ensure
-	// modifications to the request are preserved.
-	//
-	// Unparsable query parameters are removed from the outbound
-	// request if Request.Form is set after Director returns.
-	//
-	// At most one of Rewrite or Director may be set.
-	Director func(*http.Request)
-
 	// The transport used to perform proxy requests.
 	// If nil, http.DefaultTransport is used.
 	Transport http.RoundTripper
@@ -115,6 +85,8 @@ type ReverseProxy struct {
 	ModifyResponse func(*http.Response) error
 
 	ServeHTTP http.HandlerFunc
+
+	TargetURL *url.URL
 }
 
 func singleJoiningSlash(a, b string) string {
@@ -176,12 +148,7 @@ func NewReverseProxy(target *url.URL, transport http.RoundTripper) *ReverseProxy
 	if transport == nil {
 		panic("nil transport")
 	}
-	rp := &ReverseProxy{
-		Director: func(req *http.Request) {
-			rewriteRequestURL(req, target)
-		},
-		Transport: transport,
-	}
+	rp := &ReverseProxy{Transport: transport, TargetURL: target}
 	rp.ServeHTTP = rp.serveHTTP
 	return rp
 }
@@ -296,7 +263,7 @@ func (p *ReverseProxy) serveHTTP(rw http.ResponseWriter, req *http.Request) {
 		outreq.Header = make(http.Header) // Issue 33142: historical behavior was to always allocate
 	}
 
-	p.Director(outreq)
+	rewriteRequestURL(outreq, p.TargetURL)
 	outreq.Close = false
 
 	reqUpType := UpgradeType(outreq.Header)

internal/route/http.go

@@ -26,11 +26,8 @@ type (
 		PathPatterns PT.PathPatterns `json:"path_patterns"`
 
 		entry   *P.ReverseProxyEntry
-		mux     http.Handler
-		handler *ReverseProxy
-
-		regIdleWatcher   func() E.NestedError
-		unregIdleWatcher func()
+		handler http.Handler
+		rp      *ReverseProxy
 	}
 
 	URL          url.URL
@@ -63,8 +60,6 @@ func SetFindMuxDomains(domains []string) {
 
 func NewHTTPRoute(entry *P.ReverseProxyEntry) (*HTTPRoute, E.NestedError) {
 	var trans *http.Transport
-	var regIdleWatcher func() E.NestedError
-	var unregIdleWatcher func()
 
 	if entry.NoTLSVerify {
 		trans = DefaultTransportNoTLS.Clone()
@@ -81,37 +76,15 @@ func NewHTTPRoute(entry *P.ReverseProxyEntry) (*HTTPRoute, E.NestedError) {
 		}
 	}
 
-	if entry.UseIdleWatcher() {
-		// allow time for response header up to `WakeTimeout`
-		if entry.WakeTimeout > trans.ResponseHeaderTimeout {
-			trans.ResponseHeaderTimeout = entry.WakeTimeout
-		}
-		regIdleWatcher = func() E.NestedError {
-			watcher, err := idlewatcher.Register(entry)
-			if err.HasError() {
-				return err
-			}
-			// patch round-tripper
-			rp.Transport = watcher.PatchRoundTripper(trans)
-			return nil
-		}
-		unregIdleWatcher = func() {
-			idlewatcher.Unregister(entry)
-			rp.Transport = trans
-		}
-	}
-
 	httpRoutesMu.Lock()
 	defer httpRoutesMu.Unlock()
 
 	r := &HTTPRoute{
-		Alias:            entry.Alias,
-		TargetURL:        (*URL)(entry.URL),
-		PathPatterns:     entry.PathPatterns,
-		entry:            entry,
-		handler:          rp,
-		regIdleWatcher:   regIdleWatcher,
-		unregIdleWatcher: unregIdleWatcher,
+		Alias:        entry.Alias,
+		TargetURL:    (*URL)(entry.URL),
+		PathPatterns: entry.PathPatterns,
+		entry:        entry,
+		rp:           rp,
 	}
 	return r, nil
 }
@@ -121,60 +94,55 @@ func (r *HTTPRoute) String() string {
 }
 
 func (r *HTTPRoute) Start() E.NestedError {
-	if r.mux != nil {
+	if r.handler != nil {
 		return nil
 	}
 
 	httpRoutesMu.Lock()
 	defer httpRoutesMu.Unlock()
 
-	if r.regIdleWatcher != nil {
-		if err := r.regIdleWatcher(); err.HasError() {
-			r.unregIdleWatcher = nil
+	if r.entry.UseIdleWatcher() {
+		watcher, err := idlewatcher.Register(r.entry)
+		if err != nil {
 			return err
 		}
-	}
-
-	if !r.entry.UseIdleWatcher() && (r.entry.URL.Port() == "0" ||
-		r.entry.IsDocker() && !r.entry.ContainerRunning) {
-		// TODO: if it use idlewatcher, set mux to dummy mux
+		r.handler = idlewatcher.NewWaker(watcher, r.rp)
+	} else if r.entry.URL.Port() == "0" ||
+		r.entry.IsDocker() && !r.entry.ContainerRunning {
 		return nil
-	}
-
-	if len(r.PathPatterns) == 1 && r.PathPatterns[0] == "/" {
-		r.mux = ReverseProxyHandler{r.handler}
+	} else if len(r.PathPatterns) == 1 && r.PathPatterns[0] == "/" {
+		r.handler = ReverseProxyHandler{r.rp}
 	} else {
 		mux := http.NewServeMux()
 		for _, p := range r.PathPatterns {
-			mux.HandleFunc(string(p), r.handler.ServeHTTP)
+			mux.HandleFunc(string(p), r.rp.ServeHTTP)
 		}
-		r.mux = mux
+		r.handler = mux
 	}
 
 	httpRoutes.Store(string(r.Alias), r)
 	return nil
 }
 
-func (r *HTTPRoute) Stop() E.NestedError {
-	if r.mux == nil {
-		return nil
+func (r *HTTPRoute) Stop() (_ E.NestedError) {
+	if r.handler == nil {
+		return
 	}
 
 	httpRoutesMu.Lock()
 	defer httpRoutesMu.Unlock()
 
-	if r.unregIdleWatcher != nil {
-		r.unregIdleWatcher()
-		r.unregIdleWatcher = nil
+	if waker, ok := r.handler.(*idlewatcher.Waker); ok {
+		waker.Unregister()
 	}
 
-	r.mux = nil
+	r.handler = nil
 	httpRoutes.Delete(string(r.Alias))
-	return nil
+	return
 }
 
 func (r *HTTPRoute) Started() bool {
-	return r.mux != nil
+	return r.handler != nil
 }
 
 func (u *URL) String() string {
@@ -214,7 +182,7 @@ func findMuxAnyDomain(host string) (http.Handler, error) {
 	}
 	sd := strings.Join(hostSplit[:n-2], ".")
 	if r, ok := httpRoutes.Load(sd); ok {
-		return r.mux, nil
+		return r.handler, nil
 	}
 	return nil, fmt.Errorf("no such route: %s", sd)
 }
@@ -236,7 +204,7 @@ func findMuxByDomains(domains []string) func(host string) (http.Handler, error)
 			return nil, fmt.Errorf("%s does not match any base domain", host)
 		}
 		if r, ok := httpRoutes.Load(subdomain); ok {
-			return r.mux, nil
+			return r.handler, nil
 		}
 		return nil, fmt.Errorf("no such route: %s", subdomain)
 	}