Feat/fileserver (#60)

* cleanup code for URL type

* fix makefile for trace mode

* refactor, merge Entry, RawEntry and Route into one. 

* Implement fileserver.

* refactor: rename HTTPRoute to ReverseProxyRoute to avoid confusion

* refactor: move metrics logger to middleware package

- fix prometheus metrics for load balanced routes
  - route will now fail when health monitor fail to start

* fix extra output of ls-* commands by defer initializaing stuff, speed up start time

* add test for path traversal attack, small fix on FileServer.Start method

* rename rule.on.bypass to pass

* refactor and fixed map-to-map  deserialization

* updated route loading logic

* schemas: add "add_prefix" option to modify_request middleware


* updated route JSONMarshalling

---------

Co-authored-by: yusing <yusing@6uo.me>

internal/net/http/accesslog/access_logger_test.go

@@ -9,7 +9,6 @@ import (
 	"testing"
 	"time"
 
-	E "github.com/yusing/go-proxy/internal/error"
 	. "github.com/yusing/go-proxy/internal/net/http/accesslog"
 	"github.com/yusing/go-proxy/internal/task"
 	. "github.com/yusing/go-proxy/internal/utils/testing"
@@ -30,7 +29,7 @@ const (
 
 var (
 	testTask = task.RootTask("test", false)
-	testURL  = E.Must(url.Parse("http://" + host + uri))
+	testURL  = Must(url.Parse("http://" + host + uri))
 	req      = &http.Request{
 		RemoteAddr: remote,
 		Method:     method,

internal/net/http/loadbalancer/types/server.go

@@ -4,7 +4,7 @@ import (
 	"net/http"
 
 	idlewatcher "github.com/yusing/go-proxy/internal/docker/idlewatcher/types"
-	"github.com/yusing/go-proxy/internal/net/types"
+	net "github.com/yusing/go-proxy/internal/net/types"
 	U "github.com/yusing/go-proxy/internal/utils"
 	F "github.com/yusing/go-proxy/internal/utils/functional"
 	"github.com/yusing/go-proxy/internal/watcher/health"
@@ -15,7 +15,7 @@ type (
 		_ U.NoCopy
 
 		name   string
-		url    types.URL
+		url    *net.URL
 		weight Weight
 
 		http.Handler `json:"-"`
@@ -26,7 +26,7 @@ type (
 		http.Handler
 		health.HealthMonitor
 		Name() string
-		URL() types.URL
+		URL() *net.URL
 		Weight() Weight
 		SetWeight(weight Weight)
 		TryWake() error
@@ -37,7 +37,7 @@ type (
 
 var NewServerPool = F.NewMap[Pool]
 
-func NewServer(name string, url types.URL, weight Weight, handler http.Handler, healthMon health.HealthMonitor) Server {
+func NewServer(name string, url *net.URL, weight Weight, handler http.Handler, healthMon health.HealthMonitor) Server {
 	srv := &server{
 		name:          name,
 		url:           url,
@@ -59,7 +59,7 @@ func (srv *server) Name() string {
 	return srv.name
 }
 
-func (srv *server) URL() types.URL {
+func (srv *server) URL() *net.URL {
 	return srv.url
 }
 

internal/net/http/middleware/metrics_logger/metrics_logger.go

@@ -0,0 +1,44 @@
+package metricslogger
+
+import (
+	"net"
+	"net/http"
+
+	"github.com/yusing/go-proxy/internal/metrics"
+)
+
+type MetricsLogger struct {
+	ServiceName string `json:"service_name"`
+}
+
+func NewMetricsLogger(serviceName string) *MetricsLogger {
+	return &MetricsLogger{serviceName}
+}
+
+func (m *MetricsLogger) GetHandler(next http.Handler) http.HandlerFunc {
+	return func(rw http.ResponseWriter, req *http.Request) {
+		m.ServeHTTP(rw, req, next.ServeHTTP)
+	}
+}
+
+func (m *MetricsLogger) ServeHTTP(rw http.ResponseWriter, req *http.Request, next http.HandlerFunc) {
+	visitorIP, _, err := net.SplitHostPort(req.RemoteAddr)
+	if err != nil {
+		visitorIP = req.RemoteAddr
+	}
+
+	// req.RemoteAddr had been modified by middleware (if any)
+	lbls := &metrics.HTTPRouteMetricLabels{
+		Service: m.ServiceName,
+		Method:  req.Method,
+		Host:    req.Host,
+		Visitor: visitorIP,
+		Path:    req.URL.Path,
+	}
+
+	next.ServeHTTP(newHTTPMetricLogger(rw, lbls), req)
+}
+
+func (m *MetricsLogger) ResetMetrics() {
+	metrics.GetRouteMetrics().UnregisterService(m.ServiceName)
+}

internal/net/http/middleware/metrics_logger/metrics_response_writer.go

@@ -0,0 +1,47 @@
+package metricslogger
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/yusing/go-proxy/internal/metrics"
+)
+
+type httpMetricLogger struct {
+	http.ResponseWriter
+	timestamp time.Time
+	labels    *metrics.HTTPRouteMetricLabels
+}
+
+// WriteHeader implements http.ResponseWriter.
+func (l *httpMetricLogger) WriteHeader(status int) {
+	l.ResponseWriter.WriteHeader(status)
+	duration := time.Since(l.timestamp)
+	go func() {
+		m := metrics.GetRouteMetrics()
+		m.HTTPReqTotal.Inc()
+		m.HTTPReqElapsed.With(l.labels).Set(float64(duration.Milliseconds()))
+
+		// ignore 1xx
+		switch {
+		case status >= 500:
+			m.HTTP5xx.With(l.labels).Inc()
+		case status >= 400:
+			m.HTTP4xx.With(l.labels).Inc()
+		case status >= 200:
+			m.HTTP2xx3xx.With(l.labels).Inc()
+		}
+	}()
+}
+
+func (l *httpMetricLogger) Unwrap() http.ResponseWriter {
+	return l.ResponseWriter
+}
+
+func newHTTPMetricLogger(w http.ResponseWriter, labels *metrics.HTTPRouteMetricLabels) *httpMetricLogger {
+	return &httpMetricLogger{
+		ResponseWriter: w,
+		timestamp:      time.Now(),
+		labels:         labels,
+	}
+}

internal/net/http/middleware/middleware.go

@@ -196,34 +196,6 @@ func (m *Middleware) ServeHTTP(next http.HandlerFunc, w http.ResponseWriter, r *
 	next(w, r)
 }
 
-// TODO: check conflict or duplicates.
-func compileMiddlewares(middlewaresMap map[string]OptionsRaw) ([]*Middleware, E.Error) {
-	middlewares := make([]*Middleware, 0, len(middlewaresMap))
-
-	errs := E.NewBuilder("middlewares compile error")
-	invalidOpts := E.NewBuilder("options compile error")
-
-	for name, opts := range middlewaresMap {
-		m, err := Get(name)
-		if err != nil {
-			errs.Add(err)
-			continue
-		}
-
-		m, err = m.New(opts)
-		if err != nil {
-			invalidOpts.Add(err.Subject(name))
-			continue
-		}
-		middlewares = append(middlewares, m)
-	}
-
-	if invalidOpts.HasError() {
-		errs.Add(invalidOpts.Error())
-	}
-	return middlewares, errs.Error()
-}
-
 func PatchReverseProxy(rp *ReverseProxy, middlewaresMap map[string]OptionsRaw) (err E.Error) {
 	var middlewares []*Middleware
 	middlewares, err = compileMiddlewares(middlewaresMap)

internal/net/http/middleware/middleware_builder.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"sort"
 
 	E "github.com/yusing/go-proxy/internal/error"
 	"gopkg.in/yaml.v3"
@@ -39,6 +40,43 @@ func BuildMiddlewaresFromYAML(source string, data []byte, eb *E.Builder) map[str
 	return middlewares
 }
 
+func compileMiddlewares(middlewaresMap map[string]OptionsRaw) ([]*Middleware, E.Error) {
+	middlewares := make([]*Middleware, 0, len(middlewaresMap))
+
+	errs := E.NewBuilder("middlewares compile error")
+	invalidOpts := E.NewBuilder("options compile error")
+
+	for name, opts := range middlewaresMap {
+		m, err := Get(name)
+		if err != nil {
+			errs.Add(err)
+			continue
+		}
+
+		m, err = m.New(opts)
+		if err != nil {
+			invalidOpts.Add(err.Subject(name))
+			continue
+		}
+		middlewares = append(middlewares, m)
+	}
+
+	if invalidOpts.HasError() {
+		errs.Add(invalidOpts.Error())
+	}
+	sort.Sort(ByPriority(middlewares))
+	return middlewares, errs.Error()
+}
+
+func BuildMiddlewareFromMap(name string, middlewaresMap map[string]OptionsRaw) (*Middleware, E.Error) {
+	compiled, err := compileMiddlewares(middlewaresMap)
+	if err != nil {
+		return nil, err
+	}
+	return NewMiddlewareChain(name, compiled), nil
+}
+
+// TODO: check conflict or duplicates.
 func BuildMiddlewareFromChainRaw(name string, defs []map[string]any) (*Middleware, E.Error) {
 	chainErr := E.NewBuilder("")
 	chain := make([]*Middleware, 0, len(defs))

internal/net/http/middleware/middleware_builder_test.go

@@ -16,7 +16,7 @@ func TestBuild(t *testing.T) {
 	errs := E.NewBuilder("")
 	middlewares := BuildMiddlewaresFromYAML("", testMiddlewareCompose, errs)
 	ExpectNoError(t, errs.Error())
-	E.Must(json.MarshalIndent(middlewares, "", "  "))
+	Must(json.MarshalIndent(middlewares, "", "  "))
 	// t.Log(string(data))
 	// TODO: test
 }

internal/net/http/middleware/test_utils.go

@@ -12,6 +12,7 @@ import (
 	E "github.com/yusing/go-proxy/internal/error"
 	"github.com/yusing/go-proxy/internal/net/http/reverseproxy"
 	"github.com/yusing/go-proxy/internal/net/types"
+	. "github.com/yusing/go-proxy/internal/utils/testing"
 )
 
 //go:embed test_data/sample_headers.json
@@ -79,11 +80,11 @@ type TestResult struct {
 
 type testArgs struct {
 	middlewareOpt OptionsRaw
-	upstreamURL   types.URL
+	upstreamURL   *types.URL
 
 	realRoundTrip bool
 
-	reqURL    types.URL
+	reqURL    *types.URL
 	reqMethod string
 	headers   http.Header
 	body      []byte
@@ -94,14 +95,14 @@ type testArgs struct {
 }
 
 func (args *testArgs) setDefaults() {
-	if args.reqURL.Nil() {
-		args.reqURL = E.Must(types.ParseURL("https://example.com"))
+	if args.reqURL == nil {
+		args.reqURL = Must(types.ParseURL("https://example.com"))
 	}
 	if args.reqMethod == "" {
 		args.reqMethod = http.MethodGet
 	}
-	if args.upstreamURL.Nil() {
-		args.upstreamURL = E.Must(types.ParseURL("https://10.0.0.1:8443")) // dummy url, no actual effect
+	if args.upstreamURL == nil {
+		args.upstreamURL = Must(types.ParseURL("https://10.0.0.1:8443")) // dummy url, no actual effect
 	}
 	if args.respHeaders == nil {
 		args.respHeaders = http.Header{}

internal/net/http/reverseproxy/reverse_proxy_mod.go

@@ -23,12 +23,9 @@ import (
 	"net/url"
 	"strings"
 	"sync"
-	"time"
 
 	"github.com/rs/zerolog"
-	"github.com/yusing/go-proxy/internal/common"
 	"github.com/yusing/go-proxy/internal/logging"
-	"github.com/yusing/go-proxy/internal/metrics"
 	gphttp "github.com/yusing/go-proxy/internal/net/http"
 	"github.com/yusing/go-proxy/internal/net/http/accesslog"
 	"github.com/yusing/go-proxy/internal/net/types"
@@ -96,38 +93,7 @@ type ReverseProxy struct {
 	HandlerFunc http.HandlerFunc
 
 	TargetName string
-	TargetURL  types.URL
-}
-
-type httpMetricLogger struct {
-	http.ResponseWriter
-	timestamp time.Time
-	labels    *metrics.HTTPRouteMetricLabels
-}
-
-// WriteHeader implements http.ResponseWriter.
-func (l *httpMetricLogger) WriteHeader(status int) {
-	l.ResponseWriter.WriteHeader(status)
-	duration := time.Since(l.timestamp)
-	go func() {
-		m := metrics.GetRouteMetrics()
-		m.HTTPReqTotal.Inc()
-		m.HTTPReqElapsed.With(l.labels).Set(float64(duration.Milliseconds()))
-
-		// ignore 1xx
-		switch {
-		case status >= 500:
-			m.HTTP5xx.With(l.labels).Inc()
-		case status >= 400:
-			m.HTTP4xx.With(l.labels).Inc()
-		case status >= 200:
-			m.HTTP2xx3xx.With(l.labels).Inc()
-		}
-	}()
-}
-
-func (l *httpMetricLogger) Unwrap() http.ResponseWriter {
-	return l.ResponseWriter
+	TargetURL  *types.URL
 }
 
 func singleJoiningSlash(a, b string) string {
@@ -167,7 +133,7 @@ func joinURLPath(a, b *url.URL) (path, rawpath string) {
 // URLs to the scheme, host, and base path provided in target. If the
 // target's path is "/base" and the incoming request was for "/dir",
 // the target request will be for /base/dir.
-func NewReverseProxy(name string, target types.URL, transport http.RoundTripper) *ReverseProxy {
+func NewReverseProxy(name string, target *types.URL, transport http.RoundTripper) *ReverseProxy {
 	if transport == nil {
 		panic("nil transport")
 	}
@@ -181,15 +147,11 @@ func NewReverseProxy(name string, target types.URL, transport http.RoundTripper)
 	return rp
 }
 
-func (p *ReverseProxy) UnregisterMetrics() {
-	metrics.GetRouteMetrics().UnregisterService(p.TargetName)
-}
-
 func (p *ReverseProxy) rewriteRequestURL(req *http.Request) {
 	targetQuery := p.TargetURL.RawQuery
 	req.URL.Scheme = p.TargetURL.Scheme
 	req.URL.Host = p.TargetURL.Host
-	req.URL.Path, req.URL.RawPath = joinURLPath(p.TargetURL.URL, req.URL)
+	req.URL.Path, req.URL.RawPath = joinURLPath(&p.TargetURL.URL, req.URL)
 	if targetQuery == "" || req.URL.RawQuery == "" {
 		req.URL.RawQuery = targetQuery + req.URL.RawQuery
 	} else {
@@ -255,28 +217,6 @@ func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 }
 
 func (p *ReverseProxy) handler(rw http.ResponseWriter, req *http.Request) {
-	visitorIP, _, err := net.SplitHostPort(req.RemoteAddr)
-	if err != nil {
-		visitorIP = req.RemoteAddr
-	}
-
-	if common.PrometheusEnabled {
-		t := time.Now()
-		// req.RemoteAddr had been modified by middleware (if any)
-		lbls := &metrics.HTTPRouteMetricLabels{
-			Service: p.TargetName,
-			Method:  req.Method,
-			Host:    req.Host,
-			Visitor: visitorIP,
-			Path:    req.URL.Path,
-		}
-		rw = &httpMetricLogger{
-			ResponseWriter: rw,
-			timestamp:      t,
-			labels:         lbls,
-		}
-	}
-
 	transport := p.Transport
 
 	ctx := req.Context()
@@ -360,7 +300,11 @@ func (p *ReverseProxy) handler(rw http.ResponseWriter, req *http.Request) {
 	// separated list and fold multiple headers into one.
 	prior, ok := outreq.Header[gphttp.HeaderXForwardedFor]
 	omit := ok && prior == nil // Issue 38079: nil now means don't populate the header
-	xff := visitorIP
+
+	xff, _, err := net.SplitHostPort(req.RemoteAddr)
+	if err != nil {
+		xff = req.RemoteAddr
+	}
 	if len(prior) > 0 {
 		xff = strings.Join(prior, ", ") + ", " + xff
 	}

internal/net/types/url.go

@@ -2,13 +2,16 @@ package types
 
 import (
 	urlPkg "net/url"
+
+	"github.com/yusing/go-proxy/internal/utils"
 )
 
 type URL struct {
-	*urlPkg.URL
+	_ utils.NoCopy
+	urlPkg.URL
 }
 
-func MustParseURL(url string) URL {
+func MustParseURL(url string) *URL {
 	u, err := ParseURL(url)
 	if err != nil {
 		panic(err)
@@ -16,40 +19,38 @@ func MustParseURL(url string) URL {
 	return u
 }
 
-func ParseURL(url string) (URL, error) {
-	u, err := urlPkg.Parse(url)
+func ParseURL(url string) (*URL, error) {
+	u := &URL{}
+	return u, u.Parse(url)
+}
+
+func NewURL(url *urlPkg.URL) *URL {
+	return &URL{URL: *url}
+}
+
+func (u *URL) Parse(url string) error {
+	uu, err := urlPkg.Parse(url)
 	if err != nil {
-		return URL{}, err
+		return err
 	}
-	return URL{URL: u}, nil
+	u.URL = *uu
+	return nil
 }
 
-func NewURL(url *urlPkg.URL) URL {
-	return URL{url}
-}
-
-func (u URL) Nil() bool {
-	return u.URL == nil
-}
-
-func (u URL) String() string {
-	if u.URL == nil {
+func (u *URL) String() string {
+	if u == nil {
 		return "nil"
 	}
 	return u.URL.String()
 }
 
-func (u URL) MarshalJSON() (text []byte, err error) {
-	if u.URL == nil {
+func (u *URL) MarshalJSON() (text []byte, err error) {
+	if u == nil {
 		return []byte("null"), nil
 	}
 	return []byte("\"" + u.URL.String() + "\""), nil
 }
 
-func (u URL) Equals(other *URL) bool {
-	return u.URL == other.URL || u.String() == other.String()
-}
-
-func (u URL) JoinPath(path string) URL {
-	return URL{u.URL.JoinPath(path)}
+func (u *URL) Equals(other *URL) bool {
+	return u.String() == other.String()
 }