diff --git a/agent/cmd/main.go b/agent/cmd/main.go index 6b08caa5..0ce89364 100644 --- a/agent/cmd/main.go +++ b/agent/cmd/main.go @@ -12,9 +12,9 @@ import ( "github.com/yusing/godoxy/internal/metrics/systeminfo" httpServer "github.com/yusing/godoxy/internal/net/gphttp/server" "github.com/yusing/godoxy/internal/task" - "github.com/yusing/godoxy/internal/utils/strutils" "github.com/yusing/godoxy/pkg" socketproxy "github.com/yusing/godoxy/socketproxy/pkg" + strutils "github.com/yusing/goutils/strings" ) func main() { diff --git a/agent/go.mod b/agent/go.mod index f25b01fe..e79d59e5 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -19,9 +19,8 @@ require ( github.com/rs/zerolog v1.34.0 github.com/stretchr/testify v1.11.1 github.com/yusing/godoxy v0.18.6 - github.com/yusing/godoxy/internal/utils v0.0.0 github.com/yusing/godoxy/socketproxy v0.0.0-00010101000000-000000000000 - github.com/yusing/goutils v0.0.0-20250922091446-1c6a11717d72 + github.com/yusing/goutils v0.2.1 ) require ( @@ -89,7 +88,8 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.3.0 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect - github.com/yusing/ds v0.1.0 // indirect + github.com/yusing/ds v0.2.0 // indirect + github.com/yusing/godoxy/internal/utils v0.1.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect diff --git a/agent/go.sum b/agent/go.sum index e27b9e16..f7a7423e 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -206,10 +206,10 @@ github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2W github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yusing/ds v0.1.0 h1:aiZs7jPMN3MEChUsddMYjpZFHhhAmkxrwRyIUnGy5AU= -github.com/yusing/ds v0.1.0/go.mod h1:KC785+mtt+Bau0LLR+slExDaUjeiqLT1k9Or6Rpryh4= -github.com/yusing/goutils v0.0.0-20250922091446-1c6a11717d72 h1:NHYq8ZqoLSCJYzZaIOn8mJ28/Ac7GEFgeZgcfBoamWE= -github.com/yusing/goutils v0.0.0-20250922091446-1c6a11717d72/go.mod h1:XROGErdAT8UxsbLQbpejGhrTpknWEEgy+9HVjq7ULBI= +github.com/yusing/ds v0.2.0 h1:lPhDU5eA2uvquVrBrzLCrQXRJJgSXlUYA53TbuK2sQY= +github.com/yusing/ds v0.2.0/go.mod h1:XhKV4l7cZwBbbl7lRzNC9zX27zvCM0frIwiuD40ULRk= +github.com/yusing/goutils v0.2.1 h1:KjoCrNO0otthaPCZPfQY+5GKsqs5+J77CxP+TNHYa/Y= +github.com/yusing/goutils v0.2.1/go.mod h1:v6RZsMRdzcts4udSg0vqUIFvaD0OaUMPTwYJZ4XnQYo= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= diff --git a/agent/pkg/agent/http_requests.go b/agent/pkg/agent/http_requests.go index a6c2c2fd..da918bce 100644 --- a/agent/pkg/agent/http_requests.go +++ b/agent/pkg/agent/http_requests.go @@ -6,8 +6,7 @@ import ( "net/http" "github.com/gorilla/websocket" - "github.com/yusing/godoxy/internal/net/gphttp/reverseproxy" - nettypes "github.com/yusing/godoxy/internal/net/types" + "github.com/yusing/goutils/http/reverseproxy" ) func (cfg *AgentConfig) Do(ctx context.Context, method, endpoint string, body io.Reader) (*http.Response, error) { @@ -74,7 +73,7 @@ func (cfg *AgentConfig) Websocket(ctx context.Context, endpoint string) (*websoc // It will create a new request with the same context, method, and body, but with the agent host and scheme, and the endpoint // If the request has a query, it will be added to the proxy request's URL func (cfg *AgentConfig) ReverseProxy(w http.ResponseWriter, req *http.Request, endpoint string) { - rp := reverseproxy.NewReverseProxy("agent", nettypes.NewURL(AgentURL), cfg.Transport()) + rp := reverseproxy.NewReverseProxy("agent", AgentURL, cfg.Transport()) req.URL.Host = AgentHost req.URL.Scheme = "https" req.URL.Path = endpoint diff --git a/agent/pkg/certs/zip.go b/agent/pkg/certs/zip.go index 4dca3a0c..abff82a9 100644 --- a/agent/pkg/certs/zip.go +++ b/agent/pkg/certs/zip.go @@ -6,7 +6,7 @@ import ( "io" "path/filepath" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) const AgentCertsBasePath = "certs" diff --git a/cmd/pprof_prof.go b/cmd/pprof_prof.go index 3e581f8b..ba17aab4 100644 --- a/cmd/pprof_prof.go +++ b/cmd/pprof_prof.go @@ -10,7 +10,7 @@ import ( "time" "github.com/rs/zerolog/log" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) func initProfiling() { diff --git a/go.mod b/go.mod index 8e4cd3f4..068a83cc 100644 --- a/go.mod +++ b/go.mod @@ -17,12 +17,14 @@ require ( github.com/coreos/go-oidc/v3 v3.15.0 // oidc authentication github.com/docker/docker v28.4.0+incompatible // docker daemon github.com/fsnotify/fsnotify v1.9.0 // file watcher + github.com/gin-gonic/gin v1.11.0 // api server github.com/go-acme/lego/v4 v4.26.0 // acme client github.com/go-playground/validator/v10 v10.27.0 // validator github.com/gobwas/glob v0.2.3 // glob matcher for route rules github.com/gorilla/websocket v1.5.3 // websocket for API and agent github.com/gotify/server/v2 v2.7.3 // reference the Message struct for json response github.com/lithammer/fuzzysearch v1.1.8 // fuzzy search for searching icons and filtering metrics + github.com/pires/go-proxyproto v0.8.1 // proxy protocol support github.com/puzpuzpuz/xsync/v4 v4.2.0 // lock free map for concurrent operations github.com/rs/zerolog v1.34.0 // logging github.com/shirou/gopsutil/v4 v4.25.8 // system info metrics @@ -40,13 +42,15 @@ require ( github.com/golang-jwt/jwt/v5 v5.3.0 github.com/luthermonson/go-proxmox v0.2.3 github.com/oschwald/maxminddb-golang v1.13.1 - github.com/quic-go/quic-go v0.54.1 + github.com/quic-go/quic-go v0.54.1 // http3 support github.com/samber/slog-zerolog/v2 v2.7.3 github.com/spf13/afero v1.15.0 github.com/stretchr/testify v1.11.1 + github.com/yusing/ds v0.2.0 github.com/yusing/godoxy/agent v0.0.0-20250926130035-55c1c918ba95 github.com/yusing/godoxy/internal/dnsproviders v0.0.0-20250926130035-55c1c918ba95 - github.com/yusing/godoxy/internal/utils v0.0.0 + github.com/yusing/godoxy/internal/utils v0.1.0 + github.com/yusing/goutils v0.2.1 ) require ( @@ -210,13 +214,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) -require ( - github.com/gin-gonic/gin v1.11.0 - github.com/pires/go-proxyproto v0.8.1 - github.com/yusing/ds v0.1.0 - github.com/yusing/goutils v0.0.0-20250922091446-1c6a11717d72 -) - require ( github.com/akamai/AkamaiOPEN-edgegrid-golang/v11 v11.1.0 // indirect github.com/aziontech/azionapi-go-sdk v0.143.0 // indirect diff --git a/go.sum b/go.sum index 2868289e..824ac291 100644 --- a/go.sum +++ b/go.sum @@ -1646,10 +1646,10 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yusing/ds v0.1.0 h1:aiZs7jPMN3MEChUsddMYjpZFHhhAmkxrwRyIUnGy5AU= -github.com/yusing/ds v0.1.0/go.mod h1:KC785+mtt+Bau0LLR+slExDaUjeiqLT1k9Or6Rpryh4= -github.com/yusing/goutils v0.0.0-20250922091446-1c6a11717d72 h1:NHYq8ZqoLSCJYzZaIOn8mJ28/Ac7GEFgeZgcfBoamWE= -github.com/yusing/goutils v0.0.0-20250922091446-1c6a11717d72/go.mod h1:XROGErdAT8UxsbLQbpejGhrTpknWEEgy+9HVjq7ULBI= +github.com/yusing/ds v0.2.0 h1:lPhDU5eA2uvquVrBrzLCrQXRJJgSXlUYA53TbuK2sQY= +github.com/yusing/ds v0.2.0/go.mod h1:XhKV4l7cZwBbbl7lRzNC9zX27zvCM0frIwiuD40ULRk= +github.com/yusing/goutils v0.2.1 h1:KjoCrNO0otthaPCZPfQY+5GKsqs5+J77CxP+TNHYa/Y= +github.com/yusing/goutils v0.2.1/go.mod h1:v6RZsMRdzcts4udSg0vqUIFvaD0OaUMPTwYJZ4XnQYo= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= diff --git a/internal/api/v1/agent/list.go b/internal/api/v1/agent/list.go index 4fa5d742..e616c378 100644 --- a/internal/api/v1/agent/list.go +++ b/internal/api/v1/agent/list.go @@ -6,8 +6,8 @@ import ( "github.com/gin-gonic/gin" "github.com/yusing/godoxy/agent/pkg/agent" - "github.com/yusing/godoxy/internal/net/gphttp/httpheaders" "github.com/yusing/godoxy/internal/net/gphttp/websocket" + "github.com/yusing/goutils/http/httpheaders" ) // @x-id "list" diff --git a/internal/api/v1/docker/info.go b/internal/api/v1/docker/info.go index d1ae16c8..995100d8 100644 --- a/internal/api/v1/docker/info.go +++ b/internal/api/v1/docker/info.go @@ -7,7 +7,7 @@ import ( dockerSystem "github.com/docker/docker/api/types/system" "github.com/gin-gonic/gin" "github.com/yusing/godoxy/internal/gperr" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) type containerStats struct { diff --git a/internal/api/v1/docker/utils.go b/internal/api/v1/docker/utils.go index 7d84596c..f2086edb 100644 --- a/internal/api/v1/docker/utils.go +++ b/internal/api/v1/docker/utils.go @@ -11,8 +11,8 @@ import ( config "github.com/yusing/godoxy/internal/config/types" "github.com/yusing/godoxy/internal/docker" "github.com/yusing/godoxy/internal/gperr" - "github.com/yusing/godoxy/internal/net/gphttp/httpheaders" "github.com/yusing/godoxy/internal/net/gphttp/websocket" + "github.com/yusing/goutils/http/httpheaders" ) type ( diff --git a/internal/api/v1/health.go b/internal/api/v1/health.go index 78f84151..453fe8de 100644 --- a/internal/api/v1/health.go +++ b/internal/api/v1/health.go @@ -5,9 +5,9 @@ import ( "time" "github.com/gin-gonic/gin" - "github.com/yusing/godoxy/internal/net/gphttp/httpheaders" "github.com/yusing/godoxy/internal/net/gphttp/websocket" "github.com/yusing/godoxy/internal/route/routes" + "github.com/yusing/goutils/http/httpheaders" ) type HealthMap = map[string]routes.HealthInfo // @name HealthMap diff --git a/internal/api/v1/homepage/items.go b/internal/api/v1/homepage/items.go index e3eda5f6..fc0dd011 100644 --- a/internal/api/v1/homepage/items.go +++ b/internal/api/v1/homepage/items.go @@ -12,9 +12,9 @@ import ( "github.com/lithammer/fuzzysearch/fuzzy" apitypes "github.com/yusing/godoxy/internal/api/types" "github.com/yusing/godoxy/internal/homepage" - "github.com/yusing/godoxy/internal/net/gphttp/httpheaders" "github.com/yusing/godoxy/internal/net/gphttp/websocket" "github.com/yusing/godoxy/internal/route/routes" + "github.com/yusing/goutils/http/httpheaders" ) type HomepageItemsRequest struct { diff --git a/internal/api/v1/metrics/all_system_info.go b/internal/api/v1/metrics/all_system_info.go index bc6e679b..d80b8bd5 100644 --- a/internal/api/v1/metrics/all_system_info.go +++ b/internal/api/v1/metrics/all_system_info.go @@ -17,9 +17,9 @@ import ( "github.com/yusing/godoxy/internal/gperr" "github.com/yusing/godoxy/internal/metrics/period" "github.com/yusing/godoxy/internal/metrics/systeminfo" - "github.com/yusing/godoxy/internal/net/gphttp/httpheaders" "github.com/yusing/godoxy/internal/net/gphttp/websocket" - "github.com/yusing/godoxy/internal/utils/synk" + "github.com/yusing/goutils/http/httpheaders" + "github.com/yusing/goutils/synk" ) var ( diff --git a/internal/api/v1/metrics/system_info.go b/internal/api/v1/metrics/system_info.go index 0817f386..58dfa1a6 100644 --- a/internal/api/v1/metrics/system_info.go +++ b/internal/api/v1/metrics/system_info.go @@ -10,7 +10,7 @@ import ( apitypes "github.com/yusing/godoxy/internal/api/types" "github.com/yusing/godoxy/internal/metrics/period" "github.com/yusing/godoxy/internal/metrics/systeminfo" - "github.com/yusing/godoxy/internal/net/gphttp/httpheaders" + "github.com/yusing/goutils/http/httpheaders" ) type SystemInfoRequest struct { diff --git a/internal/api/v1/route/providers.go b/internal/api/v1/route/providers.go index d5265241..da454704 100644 --- a/internal/api/v1/route/providers.go +++ b/internal/api/v1/route/providers.go @@ -6,8 +6,8 @@ import ( "github.com/gin-gonic/gin" config "github.com/yusing/godoxy/internal/config/types" - "github.com/yusing/godoxy/internal/net/gphttp/httpheaders" "github.com/yusing/godoxy/internal/net/gphttp/websocket" + "github.com/yusing/goutils/http/httpheaders" ) // @x-id "providers" diff --git a/internal/api/v1/route/routes.go b/internal/api/v1/route/routes.go index 9a11dedb..ad77ea9b 100644 --- a/internal/api/v1/route/routes.go +++ b/internal/api/v1/route/routes.go @@ -6,11 +6,11 @@ import ( "time" "github.com/gin-gonic/gin" - "github.com/yusing/godoxy/internal/net/gphttp/httpheaders" "github.com/yusing/godoxy/internal/net/gphttp/websocket" "github.com/yusing/godoxy/internal/route" "github.com/yusing/godoxy/internal/route/routes" "github.com/yusing/godoxy/internal/types" + "github.com/yusing/goutils/http/httpheaders" ) type RouteType route.Route // @name Route diff --git a/internal/api/v1/stats.go b/internal/api/v1/stats.go index af320722..c3b25867 100644 --- a/internal/api/v1/stats.go +++ b/internal/api/v1/stats.go @@ -6,9 +6,9 @@ import ( "github.com/gin-gonic/gin" config "github.com/yusing/godoxy/internal/config/types" - "github.com/yusing/godoxy/internal/net/gphttp/httpheaders" "github.com/yusing/godoxy/internal/net/gphttp/websocket" "github.com/yusing/godoxy/internal/types" + "github.com/yusing/goutils/http/httpheaders" ) type StatsResponse struct { diff --git a/internal/auth/userpass.go b/internal/auth/userpass.go index 00bde533..cdbf1ec3 100644 --- a/internal/auth/userpass.go +++ b/internal/auth/userpass.go @@ -10,7 +10,7 @@ import ( "github.com/yusing/godoxy/internal/common" "github.com/yusing/godoxy/internal/gperr" "github.com/yusing/godoxy/internal/net/gphttp" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" "golang.org/x/crypto/bcrypt" ) diff --git a/internal/auth/utils.go b/internal/auth/utils.go index 2d7bcd32..c8e42b9c 100644 --- a/internal/auth/utils.go +++ b/internal/auth/utils.go @@ -8,7 +8,7 @@ import ( "github.com/yusing/godoxy/internal/common" "github.com/yusing/godoxy/internal/gperr" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) var ( diff --git a/internal/autocert/provider.go b/internal/autocert/provider.go index b2e03a04..e7014a06 100644 --- a/internal/autocert/provider.go +++ b/internal/autocert/provider.go @@ -21,7 +21,7 @@ import ( "github.com/yusing/godoxy/internal/gperr" "github.com/yusing/godoxy/internal/notif" "github.com/yusing/godoxy/internal/task" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) type ( diff --git a/internal/autocert/setup.go b/internal/autocert/setup.go index 6fbdf58b..88bbcd53 100644 --- a/internal/autocert/setup.go +++ b/internal/autocert/setup.go @@ -5,7 +5,7 @@ import ( "os" "github.com/rs/zerolog/log" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) func (p *Provider) Setup() (err error) { diff --git a/internal/config/config.go b/internal/config/config.go index 19f5e730..6a3050a0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -26,9 +26,9 @@ import ( proxy "github.com/yusing/godoxy/internal/route/provider" "github.com/yusing/godoxy/internal/serialization" "github.com/yusing/godoxy/internal/task" - "github.com/yusing/godoxy/internal/utils/strutils/ansi" "github.com/yusing/godoxy/internal/watcher" "github.com/yusing/godoxy/internal/watcher/events" + "github.com/yusing/goutils/strings/ansi" ) type Config struct { diff --git a/internal/dnsproviders/go.mod b/internal/dnsproviders/go.mod index b8c35c20..5634cede 100644 --- a/internal/dnsproviders/go.mod +++ b/internal/dnsproviders/go.mod @@ -147,8 +147,8 @@ require ( github.com/volcengine/volc-sdk-golang v1.0.221 // indirect github.com/vultr/govultr/v3 v3.24.0 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect - github.com/yusing/godoxy/internal/utils v0.0.0 // indirect - github.com/yusing/goutils v0.0.0-20250922091446-1c6a11717d72 // indirect + github.com/yusing/godoxy/internal/utils v0.1.0 // indirect + github.com/yusing/goutils v0.2.1 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect go.opentelemetry.io/otel v1.38.0 // indirect diff --git a/internal/dnsproviders/go.sum b/internal/dnsproviders/go.sum index e6f1d6c0..3e019736 100644 --- a/internal/dnsproviders/go.sum +++ b/internal/dnsproviders/go.sum @@ -1513,8 +1513,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yusing/goutils v0.0.0-20250922091446-1c6a11717d72 h1:NHYq8ZqoLSCJYzZaIOn8mJ28/Ac7GEFgeZgcfBoamWE= -github.com/yusing/goutils v0.0.0-20250922091446-1c6a11717d72/go.mod h1:XROGErdAT8UxsbLQbpejGhrTpknWEEgy+9HVjq7ULBI= +github.com/yusing/goutils v0.2.1 h1:KjoCrNO0otthaPCZPfQY+5GKsqs5+J77CxP+TNHYa/Y= +github.com/yusing/goutils v0.2.1/go.mod h1:v6RZsMRdzcts4udSg0vqUIFvaD0OaUMPTwYJZ4XnQYo= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= diff --git a/internal/docker/container_helper.go b/internal/docker/container_helper.go index 3f444e78..7dab7609 100644 --- a/internal/docker/container_helper.go +++ b/internal/docker/container_helper.go @@ -6,7 +6,7 @@ import ( "github.com/docker/docker/api/types/container" "github.com/yusing/ds/ordered" "github.com/yusing/godoxy/internal/types" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) type containerHelper struct { diff --git a/internal/docker/label.go b/internal/docker/label.go index 49df702a..21c4391f 100644 --- a/internal/docker/label.go +++ b/internal/docker/label.go @@ -7,7 +7,7 @@ import ( "github.com/goccy/go-yaml" "github.com/yusing/godoxy/internal/gperr" "github.com/yusing/godoxy/internal/types" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) var ErrInvalidLabel = gperr.New("invalid label") diff --git a/internal/gperr/error_test.go b/internal/gperr/error_test.go index 9b39823e..f0919324 100644 --- a/internal/gperr/error_test.go +++ b/internal/gperr/error_test.go @@ -5,8 +5,8 @@ import ( "strings" "testing" - "github.com/yusing/godoxy/internal/utils/strutils/ansi" expect "github.com/yusing/godoxy/internal/utils/testing" + "github.com/yusing/goutils/strings/ansi" ) func TestBaseString(t *testing.T) { diff --git a/internal/gperr/hint.go b/internal/gperr/hint.go index 8fc27356..ddc59279 100644 --- a/internal/gperr/hint.go +++ b/internal/gperr/hint.go @@ -1,6 +1,6 @@ package gperr -import "github.com/yusing/godoxy/internal/utils/strutils/ansi" +import "github.com/yusing/goutils/strings/ansi" type Hint struct { Prefix string diff --git a/internal/gperr/subject.go b/internal/gperr/subject.go index e6c2d6ff..088be5f6 100644 --- a/internal/gperr/subject.go +++ b/internal/gperr/subject.go @@ -6,7 +6,7 @@ import ( "errors" "slices" - "github.com/yusing/godoxy/internal/utils/strutils/ansi" + "github.com/yusing/goutils/strings/ansi" ) //nolint:errname diff --git a/internal/homepage/favicon.go b/internal/homepage/favicon.go index ae8434c1..af7d8385 100644 --- a/internal/homepage/favicon.go +++ b/internal/homepage/favicon.go @@ -14,7 +14,7 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/vincent-petithory/dataurl" gphttp "github.com/yusing/godoxy/internal/net/gphttp" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) type FetchResult struct { diff --git a/internal/homepage/integrations/qbittorrent/transfer_info.go b/internal/homepage/integrations/qbittorrent/transfer_info.go index c31f086d..69703531 100644 --- a/internal/homepage/integrations/qbittorrent/transfer_info.go +++ b/internal/homepage/integrations/qbittorrent/transfer_info.go @@ -4,7 +4,7 @@ import ( "context" "github.com/yusing/godoxy/internal/homepage/widgets" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) const endpointTransferInfo = "/api/v2/transfer/info" diff --git a/internal/homepage/list_icons.go b/internal/homepage/list_icons.go index 8344a11f..f3cc0819 100644 --- a/internal/homepage/list_icons.go +++ b/internal/homepage/list_icons.go @@ -16,7 +16,7 @@ import ( "github.com/yusing/godoxy/internal/common" "github.com/yusing/godoxy/internal/serialization" "github.com/yusing/godoxy/internal/task" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) type ( diff --git a/internal/idlewatcher/debug.go b/internal/idlewatcher/debug.go index 0ff21b30..5be84fe2 100644 --- a/internal/idlewatcher/debug.go +++ b/internal/idlewatcher/debug.go @@ -5,7 +5,7 @@ import ( "iter" "strconv" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) type watcherDebug struct { diff --git a/internal/idlewatcher/handle_http.go b/internal/idlewatcher/handle_http.go index 393814f1..d3213ca3 100644 --- a/internal/idlewatcher/handle_http.go +++ b/internal/idlewatcher/handle_http.go @@ -7,7 +7,7 @@ import ( api "github.com/yusing/godoxy/internal/api/v1" gphttp "github.com/yusing/godoxy/internal/net/gphttp" - "github.com/yusing/godoxy/internal/net/gphttp/httpheaders" + "github.com/yusing/goutils/http/httpheaders" ) type ForceCacheControl struct { @@ -107,7 +107,8 @@ func (w *Watcher) wakeFromHTTP(rw http.ResponseWriter, r *http.Request) (shouldN w.l.Trace().Msg("signal received") err := w.Wake(ctx) if err != nil { - gphttp.ServerError(rw, r, err) + http.Error(rw, "Internal Server Error", http.StatusInternalServerError) + gphttp.LogError(r).Msg(fmt.Sprintf("failed to wake: %v", err)) return false } diff --git a/internal/idlewatcher/loading_page.go b/internal/idlewatcher/loading_page.go index 0d0e22c1..8407080e 100644 --- a/internal/idlewatcher/loading_page.go +++ b/internal/idlewatcher/loading_page.go @@ -5,7 +5,7 @@ import ( _ "embed" "text/template" - "github.com/yusing/godoxy/internal/net/gphttp/httpheaders" + "github.com/yusing/goutils/http/httpheaders" ) type templateData struct { diff --git a/internal/idlewatcher/watcher.go b/internal/idlewatcher/watcher.go index b7052e17..c73cbdb4 100644 --- a/internal/idlewatcher/watcher.go +++ b/internal/idlewatcher/watcher.go @@ -15,7 +15,6 @@ import ( "github.com/yusing/godoxy/internal/gperr" "github.com/yusing/godoxy/internal/idlewatcher/provider" idlewatcher "github.com/yusing/godoxy/internal/idlewatcher/types" - "github.com/yusing/godoxy/internal/net/gphttp/reverseproxy" nettypes "github.com/yusing/godoxy/internal/net/types" "github.com/yusing/godoxy/internal/route/routes" "github.com/yusing/godoxy/internal/task" @@ -24,6 +23,7 @@ import ( "github.com/yusing/godoxy/internal/utils/atomic" "github.com/yusing/godoxy/internal/watcher/events" "github.com/yusing/godoxy/internal/watcher/health/monitor" + "github.com/yusing/goutils/http/reverseproxy" "golang.org/x/sync/errgroup" "golang.org/x/sync/singleflight" ) diff --git a/internal/logging/accesslog/access_logger.go b/internal/logging/accesslog/access_logger.go index 52ae36e4..334a4607 100644 --- a/internal/logging/accesslog/access_logger.go +++ b/internal/logging/accesslog/access_logger.go @@ -12,9 +12,9 @@ import ( "github.com/yusing/godoxy/internal/gperr" maxmind "github.com/yusing/godoxy/internal/maxmind/types" "github.com/yusing/godoxy/internal/task" - "github.com/yusing/godoxy/internal/utils" - "github.com/yusing/godoxy/internal/utils/strutils" - "github.com/yusing/godoxy/internal/utils/synk" + ioutils "github.com/yusing/goutils/io" + strutils "github.com/yusing/goutils/strings" + "github.com/yusing/goutils/synk" "golang.org/x/time/rate" ) @@ -26,7 +26,7 @@ type ( rawWriter io.Writer closer []io.Closer supportRotate []supportRotate - writer *utils.BufferedWriter + writer *ioutils.BufferedWriter writeLock sync.Mutex closed bool @@ -119,7 +119,7 @@ func NewAccessLoggerWithIO(parent task.Parent, writer WriterWithName, anyCfg Any task: parent.Subtask("accesslog."+writer.Name(), true), cfg: cfg, rawWriter: writer, - writer: utils.NewBufferedWriter(writer, MinBufferSize), + writer: ioutils.NewBufferedWriter(writer, MinBufferSize), bufSize: MinBufferSize, errRateLimiter: rate.NewLimiter(rate.Every(errRateLimit), errBurst), logger: log.With().Str("file", writer.Name()).Logger(), diff --git a/internal/logging/accesslog/back_scanner_test.go b/internal/logging/accesslog/back_scanner_test.go index ef42ed90..08cab2fa 100644 --- a/internal/logging/accesslog/back_scanner_test.go +++ b/internal/logging/accesslog/back_scanner_test.go @@ -10,8 +10,8 @@ import ( "github.com/spf13/afero" "github.com/yusing/godoxy/internal/task" - "github.com/yusing/godoxy/internal/utils/strutils" expect "github.com/yusing/godoxy/internal/utils/testing" + strutils "github.com/yusing/goutils/strings" ) func TestBackScanner(t *testing.T) { diff --git a/internal/logging/accesslog/filter.go b/internal/logging/accesslog/filter.go index 2d6f1f09..0659904b 100644 --- a/internal/logging/accesslog/filter.go +++ b/internal/logging/accesslog/filter.go @@ -7,7 +7,7 @@ import ( "github.com/yusing/godoxy/internal/gperr" nettypes "github.com/yusing/godoxy/internal/net/types" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) type ( diff --git a/internal/logging/accesslog/filter_test.go b/internal/logging/accesslog/filter_test.go index 7c39ecb2..995ff895 100644 --- a/internal/logging/accesslog/filter_test.go +++ b/internal/logging/accesslog/filter_test.go @@ -7,8 +7,8 @@ import ( . "github.com/yusing/godoxy/internal/logging/accesslog" nettypes "github.com/yusing/godoxy/internal/net/types" - "github.com/yusing/godoxy/internal/utils/strutils" expect "github.com/yusing/godoxy/internal/utils/testing" + strutils "github.com/yusing/goutils/strings" ) func TestStatusCodeFilter(t *testing.T) { diff --git a/internal/logging/accesslog/retention.go b/internal/logging/accesslog/retention.go index 560f4814..d5bd4e40 100644 --- a/internal/logging/accesslog/retention.go +++ b/internal/logging/accesslog/retention.go @@ -5,7 +5,7 @@ import ( "strconv" "github.com/yusing/godoxy/internal/gperr" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) type Retention struct { diff --git a/internal/logging/accesslog/rotate.go b/internal/logging/accesslog/rotate.go index 57057795..d81d43f6 100644 --- a/internal/logging/accesslog/rotate.go +++ b/internal/logging/accesslog/rotate.go @@ -8,8 +8,8 @@ import ( "github.com/rs/zerolog" "github.com/yusing/godoxy/internal/gperr" "github.com/yusing/godoxy/internal/utils" - "github.com/yusing/godoxy/internal/utils/strutils" - "github.com/yusing/godoxy/internal/utils/synk" + strutils "github.com/yusing/goutils/strings" + "github.com/yusing/goutils/synk" ) type supportRotate interface { diff --git a/internal/logging/accesslog/rotate_test.go b/internal/logging/accesslog/rotate_test.go index ce96260e..f3c7bf04 100644 --- a/internal/logging/accesslog/rotate_test.go +++ b/internal/logging/accesslog/rotate_test.go @@ -9,8 +9,8 @@ import ( . "github.com/yusing/godoxy/internal/logging/accesslog" "github.com/yusing/godoxy/internal/task" "github.com/yusing/godoxy/internal/utils" - "github.com/yusing/godoxy/internal/utils/strutils" expect "github.com/yusing/godoxy/internal/utils/testing" + strutils "github.com/yusing/goutils/strings" ) var ( diff --git a/internal/logging/accesslog/status_code_range.go b/internal/logging/accesslog/status_code_range.go index c943e772..07ba7974 100644 --- a/internal/logging/accesslog/status_code_range.go +++ b/internal/logging/accesslog/status_code_range.go @@ -4,7 +4,7 @@ import ( "strconv" "github.com/yusing/godoxy/internal/gperr" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) type StatusCodeRange struct { diff --git a/internal/logging/logging.go b/internal/logging/logging.go index bbeb0035..af860d25 100644 --- a/internal/logging/logging.go +++ b/internal/logging/logging.go @@ -8,7 +8,7 @@ import ( "github.com/rs/zerolog" "github.com/yusing/godoxy/internal/common" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" zerologlog "github.com/rs/zerolog/log" ) diff --git a/internal/metrics/period/handler.go b/internal/metrics/period/handler.go index 5dffcab8..5cf0212e 100644 --- a/internal/metrics/period/handler.go +++ b/internal/metrics/period/handler.go @@ -8,8 +8,8 @@ import ( "github.com/gin-gonic/gin" apitypes "github.com/yusing/godoxy/internal/api/types" metricsutils "github.com/yusing/godoxy/internal/metrics/utils" - "github.com/yusing/godoxy/internal/net/gphttp/httpheaders" "github.com/yusing/godoxy/internal/net/gphttp/websocket" + "github.com/yusing/goutils/http/httpheaders" ) type ResponseType[AggregateT any] struct { diff --git a/internal/net/gphttp/httpheaders/csp.go b/internal/net/gphttp/httpheaders/csp.go deleted file mode 100644 index d77b595b..00000000 --- a/internal/net/gphttp/httpheaders/csp.go +++ /dev/null @@ -1,69 +0,0 @@ -package httpheaders - -import ( - "net/http" - "strings" -) - -// AppendCSP appends a CSP header to specific directives in the response writer. -// -// Directives other than the ones in cspDirectives will be kept as is. -// -// It will replace 'none' with the sources. -// -// It will append 'self' to the sources if it's not already present. -func AppendCSP(w http.ResponseWriter, r *http.Request, cspDirectives []string, sources []string) { - csp := make(map[string]string) - cspValues := r.Header.Values("Content-Security-Policy") - if len(cspValues) == 1 { - cspValues = strings.Split(cspValues[0], ";") - for i, cspString := range cspValues { - cspValues[i] = strings.TrimSpace(cspString) - } - } - - for _, cspString := range cspValues { - parts := strings.SplitN(cspString, " ", 2) - if len(parts) == 2 { - csp[parts[0]] = parts[1] - } - } - - for _, directive := range cspDirectives { - value, ok := csp[directive] - if !ok { - value = "'self'" - } - switch value { - case "'self'": - csp[directive] = value + " " + strings.Join(sources, " ") - case "'none'": - csp[directive] = strings.Join(sources, " ") - default: - for _, source := range sources { - if !strings.Contains(value, source) { - value += " " + source - } - } - if !strings.Contains(value, "'self'") { - value = "'self' " + value - } - csp[directive] = value - } - } - - values := make([]string, 0, len(csp)) - for directive, value := range csp { - values = append(values, directive+" "+value) - } - - // Remove existing CSP header, case insensitive - for k := range w.Header() { - if strings.EqualFold(k, "Content-Security-Policy") { - delete(w.Header(), k) - } - } - - // Set new CSP header - w.Header()["Content-Security-Policy"] = values -} diff --git a/internal/net/gphttp/httpheaders/csp_test.go b/internal/net/gphttp/httpheaders/csp_test.go deleted file mode 100644 index 95b33c22..00000000 --- a/internal/net/gphttp/httpheaders/csp_test.go +++ /dev/null @@ -1,168 +0,0 @@ -package httpheaders - -import ( - "net/http" - "net/http/httptest" - "strings" - "testing" -) - -func TestAppendCSP(t *testing.T) { - tests := []struct { - name string - initialHeaders map[string][]string - sources []string - directives []string - expectedCSP map[string]string - }{ - { - name: "No CSP header", - initialHeaders: map[string][]string{}, - sources: []string{}, - directives: []string{"default-src", "script-src", "frame-src", "style-src", "connect-src"}, - expectedCSP: map[string]string{"default-src": "'self'", "script-src": "'self'", "frame-src": "'self'", "style-src": "'self'", "connect-src": "'self'"}, - }, - { - name: "No CSP header with sources", - initialHeaders: map[string][]string{}, - sources: []string{"https://example.com"}, - directives: []string{"default-src", "script-src", "frame-src", "style-src", "connect-src"}, - expectedCSP: map[string]string{"default-src": "'self' https://example.com", "script-src": "'self' https://example.com", "frame-src": "'self' https://example.com", "style-src": "'self' https://example.com", "connect-src": "'self' https://example.com"}, - }, - { - name: "replace 'none' with sources", - initialHeaders: map[string][]string{ - "Content-Security-Policy": {"default-src 'none'"}, - }, - sources: []string{"https://example.com"}, - directives: []string{"default-src"}, - expectedCSP: map[string]string{"default-src": "https://example.com"}, - }, - { - name: "CSP header with some directives", - initialHeaders: map[string][]string{ - "Content-Security-Policy": {"default-src 'none'", "script-src 'unsafe-inline'"}, - }, - sources: []string{"https://example.com"}, - directives: []string{"script-src"}, - expectedCSP: map[string]string{ - "default-src": "'none", - "script-src": "'unsafe-inline' https://example.com", - }, - }, - { - name: "CSP header with some directives with self", - initialHeaders: map[string][]string{ - "Content-Security-Policy": {"default-src 'self'", "connect-src 'self'"}, - }, - sources: []string{"https://api.example.com"}, - directives: []string{"default-src", "connect-src"}, - expectedCSP: map[string]string{ - "default-src": "'self' https://api.example.com", - "connect-src": "'self' https://api.example.com", - }, - }, - { - name: "AppendCSP sources conflict with existing CSP header", - initialHeaders: map[string][]string{ - "Content-Security-Policy": {"default-src 'self' https://cdn.example.com", "script-src 'unsafe-inline'"}, - }, - sources: []string{"https://cdn.example.com", "https://api.example.com"}, - directives: []string{"default-src", "script-src"}, - expectedCSP: map[string]string{ - "default-src": "'self' https://cdn.example.com https://api.example.com", - "script-src": "'unsafe-inline' https://cdn.example.com https://api.example.com", - }, - }, - { - name: "Non-standard CSP directive", - initialHeaders: map[string][]string{ - "Content-Security-Policy": { - "default-src 'self'", - "script-src 'unsafe-inline'", - "img-src 'self'", // img-src is not in cspDirectives list - }, - }, - sources: []string{"https://example.com"}, - directives: []string{"default-src", "script-src"}, - expectedCSP: map[string]string{ - "default-src": "'self' https://example.com", - "script-src": "'unsafe-inline' https://example.com", - // img-src should not be present in response as it's not in cspDirectives - }, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - // Create a test request with initial headers - req := httptest.NewRequest(http.MethodGet, "/", nil) - for header, values := range tc.initialHeaders { - req.Header[header] = values - } - - // Create a test response recorder - w := httptest.NewRecorder() - - // Call the function under test - AppendCSP(w, req, tc.directives, tc.sources) - - // Check the resulting CSP headers - respHeaders := w.Header() - cspValues, exists := respHeaders["Content-Security-Policy"] - - // If we expect no CSP headers, verify none exist - if len(tc.expectedCSP) == 0 { - if exists && len(cspValues) > 0 { - t.Errorf("Expected no CSP header, but got %v", cspValues) - } - return - } - - // Verify CSP headers exist when expected - if !exists || len(cspValues) == 0 { - t.Errorf("Expected CSP header to be set, but it was not") - return - } - - // Parse the CSP response and verify each directive - foundDirectives := make(map[string]string) - for _, cspValue := range cspValues { - parts := strings.Split(cspValue, ";") - for _, part := range parts { - part = strings.TrimSpace(part) - if part == "" { - continue - } - - directiveParts := strings.SplitN(part, " ", 2) - if len(directiveParts) != 2 { - t.Errorf("Invalid CSP directive format: %s", part) - continue - } - - directive := directiveParts[0] - value := directiveParts[1] - foundDirectives[directive] = value - } - } - - // Verify expected directives - for directive, expectedValue := range tc.expectedCSP { - actualValue, ok := foundDirectives[directive] - if !ok { - t.Errorf("Expected directive %s not found in response", directive) - continue - } - - // Check if all expected sources are in the actual value - expectedSources := strings.SplitSeq(expectedValue, " ") - for source := range expectedSources { - if !strings.Contains(actualValue, source) { - t.Errorf("Directive %s missing expected source %s. Got: %s", directive, source, actualValue) - } - } - } - }) - } -} diff --git a/internal/net/gphttp/httpheaders/utils.go b/internal/net/gphttp/httpheaders/utils.go deleted file mode 100644 index 66a2b80e..00000000 --- a/internal/net/gphttp/httpheaders/utils.go +++ /dev/null @@ -1,119 +0,0 @@ -package httpheaders - -import ( - "net/http" - "net/textproto" - "strings" - - "golang.org/x/net/http/httpguts" -) - -const ( - HeaderXForwardedMethod = "X-Forwarded-Method" - HeaderXForwardedFor = "X-Forwarded-For" - HeaderXForwardedProto = "X-Forwarded-Proto" - HeaderXForwardedHost = "X-Forwarded-Host" - HeaderXForwardedPort = "X-Forwarded-Port" - HeaderXForwardedURI = "X-Forwarded-Uri" - HeaderXRealIP = "X-Real-IP" - - HeaderContentType = "Content-Type" - HeaderContentLength = "Content-Length" - - HeaderGoDoxyCheckRedirect = "X-Godoxy-Check-Redirect" -) - -// Hop-by-hop headers. These are removed when sent to the backend. -// As of RFC 7230, hop-by-hop headers are required to appear in the -// Connection header field. These are the headers defined by the -// obsoleted RFC 2616 (section 13.5.1) and are used for backward -// compatibility. -var hopHeaders = []string{ - "Connection", - "Proxy-Connection", // non-standard but still sent by libcurl and rejected by e.g. google - "Keep-Alive", - "Proxy-Authenticate", - "Proxy-Authorization", - "Te", // canonicalized version of "TE" - "Trailer", // not Trailers per URL above; https://www.rfc-editor.org/errata_search.php?eid=4522 - "Transfer-Encoding", - "Upgrade", -} - -func UpgradeType(h http.Header) string { - if !httpguts.HeaderValuesContainsToken(h["Connection"], "Upgrade") { - return "" - } - return h.Get("Upgrade") -} - -// RemoveHopByHopHeaders removes hop-by-hop headers. -func RemoveHopByHopHeaders(h http.Header) { - // RFC 7230, section 6.1: Remove headers listed in the "Connection" header. - for _, f := range h["Connection"] { - for sf := range strings.SplitSeq(f, ",") { - if sf = textproto.TrimString(sf); sf != "" { - h.Del(sf) - } - } - } - // RFC 2616, section 13.5.1: Remove a set of known hop-by-hop headers. - // This behavior is superseded by the RFC 7230 Connection header, but - // preserve it for backwards compatibility. - for _, f := range hopHeaders { - h.Del(f) - } -} - -func RemoveHop(h http.Header) { - reqUpType := UpgradeType(h) - RemoveHopByHopHeaders(h) - - if reqUpType != "" { - h.Set("Connection", "Upgrade") - h.Set("Upgrade", reqUpType) - } else { - h.Del("Connection") - } -} - -func RemoveServiceHeaders(h http.Header) { - h.Del("X-Powered-By") - h.Del("Server") -} - -func CopyHeader(dst, src http.Header) { - for k, vv := range src { - for _, v := range vv { - dst.Add(k, v) - } - } -} - -func FilterHeaders(h http.Header, allowed []string) http.Header { - if len(allowed) == 0 { - return h - } - - filtered := make(http.Header) - - for i, header := range allowed { - values := h.Values(header) - if len(values) == 0 { - continue - } - filtered[http.CanonicalHeaderKey(allowed[i])] = append([]string(nil), values...) - } - - return filtered -} - -func HeaderToMap(h http.Header) map[string]string { - result := make(map[string]string) - for k, v := range h { - if len(v) > 0 { - result[k] = v[0] // Take the first value - } - } - return result -} diff --git a/internal/net/gphttp/httpheaders/websocket.go b/internal/net/gphttp/httpheaders/websocket.go deleted file mode 100644 index 9374ec5f..00000000 --- a/internal/net/gphttp/httpheaders/websocket.go +++ /dev/null @@ -1,9 +0,0 @@ -package httpheaders - -import ( - "net/http" -) - -func IsWebsocket(h http.Header) bool { - return UpgradeType(h) == "websocket" -} diff --git a/internal/net/gphttp/loadbalancer/loadbalancer.go b/internal/net/gphttp/loadbalancer/loadbalancer.go index dd0624a7..7f67f743 100644 --- a/internal/net/gphttp/loadbalancer/loadbalancer.go +++ b/internal/net/gphttp/loadbalancer/loadbalancer.go @@ -9,10 +9,10 @@ import ( "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/yusing/godoxy/internal/gperr" - "github.com/yusing/godoxy/internal/net/gphttp/httpheaders" "github.com/yusing/godoxy/internal/task" "github.com/yusing/godoxy/internal/types" "github.com/yusing/godoxy/internal/utils/pool" + "github.com/yusing/goutils/http/httpheaders" ) // TODO: stats of each server. diff --git a/internal/net/gphttp/middleware/bypass_test.go b/internal/net/gphttp/middleware/bypass_test.go index 4bb173a5..98126767 100644 --- a/internal/net/gphttp/middleware/bypass_test.go +++ b/internal/net/gphttp/middleware/bypass_test.go @@ -12,12 +12,11 @@ import ( "github.com/yusing/godoxy/internal/entrypoint" . "github.com/yusing/godoxy/internal/net/gphttp/middleware" - "github.com/yusing/godoxy/internal/net/gphttp/reverseproxy" - nettypes "github.com/yusing/godoxy/internal/net/types" "github.com/yusing/godoxy/internal/route" routeTypes "github.com/yusing/godoxy/internal/route/types" "github.com/yusing/godoxy/internal/task" expect "github.com/yusing/godoxy/internal/utils/testing" + "github.com/yusing/goutils/http/reverseproxy" ) func noOpHandler(w http.ResponseWriter, r *http.Request) {} @@ -102,8 +101,10 @@ func (f fakeRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { } func TestReverseProxyBypass(t *testing.T) { - rp := reverseproxy.NewReverseProxy("test", nettypes.MustParseURL("http://example.com"), fakeRoundTripper{}) - err := PatchReverseProxy(rp, map[string]OptionsRaw{ + url, err := url.Parse("http://example.com") + expect.NoError(t, err) + rp := reverseproxy.NewReverseProxy("test", url, fakeRoundTripper{}) + err = PatchReverseProxy(rp, map[string]OptionsRaw{ "response": { "bypass": "path /test/* | path /api", "set_headers": map[string]string{ diff --git a/internal/net/gphttp/middleware/cloudflare_real_ip.go b/internal/net/gphttp/middleware/cloudflare_real_ip.go index dd52137c..1214f099 100644 --- a/internal/net/gphttp/middleware/cloudflare_real_ip.go +++ b/internal/net/gphttp/middleware/cloudflare_real_ip.go @@ -15,7 +15,7 @@ import ( "github.com/yusing/godoxy/internal/common" nettypes "github.com/yusing/godoxy/internal/net/types" "github.com/yusing/godoxy/internal/utils/atomic" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) type cloudflareRealIP struct { diff --git a/internal/net/gphttp/middleware/custom_error_page.go b/internal/net/gphttp/middleware/custom_error_page.go index 231617a4..06a7bd81 100644 --- a/internal/net/gphttp/middleware/custom_error_page.go +++ b/internal/net/gphttp/middleware/custom_error_page.go @@ -10,8 +10,8 @@ import ( "github.com/rs/zerolog/log" gphttp "github.com/yusing/godoxy/internal/net/gphttp" - "github.com/yusing/godoxy/internal/net/gphttp/httpheaders" "github.com/yusing/godoxy/internal/net/gphttp/middleware/errorpage" + "github.com/yusing/goutils/http/httpheaders" ) type customErrorPage struct{} diff --git a/internal/net/gphttp/middleware/forwardauth.go b/internal/net/gphttp/middleware/forwardauth.go index 1f1949ae..cd20c286 100644 --- a/internal/net/gphttp/middleware/forwardauth.go +++ b/internal/net/gphttp/middleware/forwardauth.go @@ -7,9 +7,9 @@ import ( "net/http" "time" - "github.com/yusing/godoxy/internal/net/gphttp/httpheaders" "github.com/yusing/godoxy/internal/route/routes" - "github.com/yusing/godoxy/internal/utils" + httputils "github.com/yusing/goutils/http" + "github.com/yusing/goutils/http/httpheaders" ) type ( @@ -91,7 +91,7 @@ func (m *forwardAuthMiddleware) before(w http.ResponseWriter, r *http.Request) ( defer resp.Body.Close() if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { - body, release, err := utils.ReadAllBody(resp) + body, release, err := httputils.ReadAllBody(resp) defer release() if err != nil { diff --git a/internal/net/gphttp/middleware/middleware.go b/internal/net/gphttp/middleware/middleware.go index 84cb83a3..3100698a 100644 --- a/internal/net/gphttp/middleware/middleware.go +++ b/internal/net/gphttp/middleware/middleware.go @@ -12,8 +12,8 @@ import ( "github.com/rs/zerolog/log" "github.com/yusing/godoxy/internal/gperr" gphttp "github.com/yusing/godoxy/internal/net/gphttp" - "github.com/yusing/godoxy/internal/net/gphttp/reverseproxy" "github.com/yusing/godoxy/internal/serialization" + "github.com/yusing/goutils/http/reverseproxy" ) type ( diff --git a/internal/net/gphttp/middleware/middlewares.go b/internal/net/gphttp/middleware/middlewares.go index b1e562ae..56e7342e 100644 --- a/internal/net/gphttp/middleware/middlewares.go +++ b/internal/net/gphttp/middleware/middlewares.go @@ -9,7 +9,7 @@ import ( "github.com/yusing/godoxy/internal/common" "github.com/yusing/godoxy/internal/gperr" "github.com/yusing/godoxy/internal/utils" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) // snakes and cases will be stripped on `Get` diff --git a/internal/net/gphttp/middleware/modify_html.go b/internal/net/gphttp/middleware/modify_html.go index 57b6a5c7..32b77ffd 100644 --- a/internal/net/gphttp/middleware/modify_html.go +++ b/internal/net/gphttp/middleware/modify_html.go @@ -9,8 +9,9 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/rs/zerolog/log" gphttp "github.com/yusing/godoxy/internal/net/gphttp" - "github.com/yusing/godoxy/internal/utils" - "github.com/yusing/godoxy/internal/utils/synk" + httputils "github.com/yusing/goutils/http" + ioutils "github.com/yusing/goutils/io" + "github.com/yusing/goutils/synk" "golang.org/x/net/html" ) @@ -40,7 +41,7 @@ func (m *modifyHTML) modifyResponse(resp *http.Response) error { } // NOTE: do not put it in the defer, it will be used as resp.Body - content, release, err := utils.ReadAllBody(resp) + content, release, err := httputils.ReadAllBody(resp) if err != nil { resp.Body.Close() return err @@ -71,19 +72,19 @@ func (m *modifyHTML) modifyResponse(resp *http.Response) error { } buf := bytes.NewBuffer(content[:0]) - err = buildHTML(m, doc, buf) + err = buildHTML(doc, buf) if err != nil { return err } resp.ContentLength = int64(buf.Len()) resp.Header.Set("Content-Length", strconv.Itoa(buf.Len())) resp.Header.Set("Content-Type", "text/html; charset=utf-8") - resp.Body = utils.NewHookCloser(io.NopCloser(bytes.NewReader(buf.Bytes())), release) + resp.Body = ioutils.NewHookReadCloser(io.NopCloser(bytes.NewReader(buf.Bytes())), release) return nil } // copied and modified from (*goquery.Selection).Html() -func buildHTML(m *modifyHTML, s *goquery.Document, buf *bytes.Buffer) error { +func buildHTML(s *goquery.Document, buf *bytes.Buffer) error { // Merge all head nodes into one headNodes := s.Find("head") if headNodes.Length() > 1 { diff --git a/internal/net/gphttp/middleware/real_ip.go b/internal/net/gphttp/middleware/real_ip.go index 162b0d2b..34bb1ddd 100644 --- a/internal/net/gphttp/middleware/real_ip.go +++ b/internal/net/gphttp/middleware/real_ip.go @@ -4,8 +4,8 @@ import ( "net" "net/http" - "github.com/yusing/godoxy/internal/net/gphttp/httpheaders" nettypes "github.com/yusing/godoxy/internal/net/types" + "github.com/yusing/goutils/http/httpheaders" ) // https://nginx.org/en/docs/http/ngx_http_realip_module.html diff --git a/internal/net/gphttp/middleware/real_ip_test.go b/internal/net/gphttp/middleware/real_ip_test.go index be1769e3..38a6952e 100644 --- a/internal/net/gphttp/middleware/real_ip_test.go +++ b/internal/net/gphttp/middleware/real_ip_test.go @@ -6,9 +6,9 @@ import ( "strings" "testing" - "github.com/yusing/godoxy/internal/net/gphttp/httpheaders" nettypes "github.com/yusing/godoxy/internal/net/types" . "github.com/yusing/godoxy/internal/utils/testing" + "github.com/yusing/goutils/http/httpheaders" ) func TestSetRealIPOpts(t *testing.T) { diff --git a/internal/net/gphttp/middleware/test_utils.go b/internal/net/gphttp/middleware/test_utils.go index 24e51342..2e8128ba 100644 --- a/internal/net/gphttp/middleware/test_utils.go +++ b/internal/net/gphttp/middleware/test_utils.go @@ -11,9 +11,9 @@ import ( "github.com/yusing/godoxy/internal/common" "github.com/yusing/godoxy/internal/gperr" - "github.com/yusing/godoxy/internal/net/gphttp/reverseproxy" nettypes "github.com/yusing/godoxy/internal/net/types" . "github.com/yusing/godoxy/internal/utils/testing" + "github.com/yusing/goutils/http/reverseproxy" ) //go:embed test_data/sample_headers.json @@ -152,7 +152,7 @@ func newMiddlewaresTest(middlewares []*Middleware, args *testArgs) (*TestResult, rr.parent = http.DefaultTransport } - rp := reverseproxy.NewReverseProxy("test", args.upstreamURL, rr) + rp := reverseproxy.NewReverseProxy("test", &args.upstreamURL.URL, rr) patchReverseProxy(rp, middlewares) rp.ServeHTTP(w, req) diff --git a/internal/net/gphttp/middleware/x_forwarded.go b/internal/net/gphttp/middleware/x_forwarded.go index bf6c4ef7..2997335e 100644 --- a/internal/net/gphttp/middleware/x_forwarded.go +++ b/internal/net/gphttp/middleware/x_forwarded.go @@ -5,7 +5,7 @@ import ( "net/http" "strings" - "github.com/yusing/godoxy/internal/net/gphttp/httpheaders" + "github.com/yusing/goutils/http/httpheaders" ) type ( diff --git a/internal/net/gphttp/reverseproxy/reverse_proxy.go b/internal/net/gphttp/reverseproxy/reverse_proxy.go deleted file mode 100644 index c1bea568..00000000 --- a/internal/net/gphttp/reverseproxy/reverse_proxy.go +++ /dev/null @@ -1,560 +0,0 @@ -// Copyright 2011 The Go Authors. -// Modified from the Go project under the a BSD-style License (https://cs.opensource.google/go/go/+/refs/tags/go1.23.1:src/net/http/httputil/reverseproxy.go) -// https://cs.opensource.google/go/go/+/master:LICENSE - -package reverseproxy - -// This is a small mod on net/http/httputil/reverseproxy.go -// that boosts performance in some cases -// and compatible to other modules of this project -// Copyright (c) 2024 yusing - -import ( - "bytes" - "context" - "crypto/tls" - "errors" - "fmt" - "io" - "net" - "net/http" - "net/http/httptrace" - "net/textproto" - "net/url" - "strings" - "sync" - - "github.com/quic-go/quic-go/http3" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - "github.com/yusing/godoxy/internal/logging/accesslog" - "github.com/yusing/godoxy/internal/net/gphttp/httpheaders" - nettypes "github.com/yusing/godoxy/internal/net/types" - U "github.com/yusing/godoxy/internal/utils" - "golang.org/x/net/http/httpguts" - "golang.org/x/net/http2" - - _ "unsafe" -) - -// A ProxyRequest contains a request to be rewritten by a [ReverseProxy]. -type ProxyRequest struct { - // In is the request received by the proxy. - // The Rewrite function must not modify In. - In *http.Request - - // Out is the request which will be sent by the proxy. - // The Rewrite function may modify or replace this request. - // Hop-by-hop headers are removed from this request - // before Rewrite is called. - Out *http.Request -} - -// SetXForwarded sets the X-Forwarded-For, X-Forwarded-Host, and -// X-Forwarded-Proto headers of the outbound request. -// -// - The X-Forwarded-For header is set to the client IP address. -// - The X-Forwarded-Host header is set to the host name requested -// by the client. -// - The X-Forwarded-Proto header is set to "http" or "https", depending -// on whether the inbound request was made on a TLS-enabled connection. -// -// If the outbound request contains an existing X-Forwarded-For header, -// SetXForwarded appends the client IP address to it. To append to the -// inbound request's X-Forwarded-For header (the default behavior of -// [ReverseProxy] when using a Director function), copy the header -// from the inbound request before calling SetXForwarded: -// -// rewriteFunc := func(r *httputil.ProxyRequest) { -// r.Out.Header["X-Forwarded-For"] = r.In.Header["X-Forwarded-For"] -// r.SetXForwarded() -// } - -// ReverseProxy is an HTTP Handler that takes an incoming request and -// sends it to another server, proxying the response back to the -// client. -// -// 1xx responses are forwarded to the client if the underlying -// transport supports ClientTrace.Got1xxResponse. -type ReverseProxy struct { - zerolog.Logger - - // The transport used to perform proxy requests. - Transport http.RoundTripper - - // ModifyResponse is an optional function that modifies the - // Response from the backend. It is called if the backend - // returns a response at all, with any HTTP status code. - // If the backend is unreachable, the optional ErrorHandler is - // called before ModifyResponse. - // - // If ModifyResponse returns an error, ErrorHandler is called - // with its error value. If ErrorHandler is nil, its default - // implementation is used. - ModifyResponse func(*http.Response) error - AccessLogger *accesslog.AccessLogger - - HandlerFunc http.HandlerFunc - - TargetName string - TargetURL *nettypes.URL -} - -func singleJoiningSlash(a, b string) string { - aslash := strings.HasSuffix(a, "/") - bslash := strings.HasPrefix(b, "/") - switch { - case aslash && bslash: - return a + b[1:] - case !aslash && !bslash: - return a + "/" + b - } - return a + b -} - -func joinURLPath(a, b *url.URL) (path, rawpath string) { - if a.RawPath == "" && b.RawPath == "" { - return singleJoiningSlash(a.Path, b.Path), "" - } - // Same as singleJoiningSlash, but uses EscapedPath to determine - // whether a slash should be added - apath := a.EscapedPath() - bpath := b.EscapedPath() - - aslash := strings.HasSuffix(apath, "/") - bslash := strings.HasPrefix(bpath, "/") - - switch { - case aslash && bslash: - return a.Path + b.Path[1:], apath + bpath[1:] - case !aslash && !bslash: - return a.Path + "/" + b.Path, apath + "/" + bpath - } - return a.Path + b.Path, apath + bpath -} - -// NewReverseProxy returns a new [ReverseProxy] that routes -// URLs to the scheme, host, and base path provided in target. If the -// target's path is "/base" and the incoming request was for "/dir", -// the target request will be for /base/dir. -func NewReverseProxy(name string, target *nettypes.URL, transport http.RoundTripper) *ReverseProxy { - if transport == nil { - panic("nil transport") - } - rp := &ReverseProxy{ - Logger: log.With().Str("name", name).Logger(), - Transport: transport, - TargetName: name, - TargetURL: target, - } - rp.HandlerFunc = rp.handler - return rp -} - -func (p *ReverseProxy) rewriteRequestURL(req *http.Request) { - targetQuery := p.TargetURL.RawQuery - req.URL.Scheme = p.TargetURL.Scheme - req.URL.Host = p.TargetURL.Host - req.URL.Path, req.URL.RawPath = joinURLPath(&p.TargetURL.URL, req.URL) - if targetQuery == "" || req.URL.RawQuery == "" { - req.URL.RawQuery = targetQuery + req.URL.RawQuery - } else { - req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery - } -} - -func copyHeader(dst, src http.Header) { - for k, vv := range src { - for _, v := range vv { - dst.Add(k, v) - } - } -} - -//go:linkname errStreamClosed golang.org/x/net/http2.errStreamClosed -var errStreamClosed error - -func (p *ReverseProxy) errorHandler(rw http.ResponseWriter, r *http.Request, err error, writeHeader bool) { - reqURL := r.Host + r.URL.Path - switch { - case errors.Is(err, context.Canceled), errors.Is(err, io.EOF): - log.Trace().Err(err).Str("url", reqURL).Msg("http proxy error") - case errors.Is(err, context.DeadlineExceeded): - log.Debug().Err(err).Str("url", reqURL).Msg("http proxy error") - default: - var recordErr tls.RecordHeaderError - if errors.As(err, &recordErr) { - log.Error(). - Str("url", reqURL). - Msgf(`scheme was likely misconfigured as https, - try setting "proxy.%s.scheme" back to "http"`, p.TargetName) - log.Err(err).Msg("underlying error") - goto logged - } - if errors.Is(err, errStreamClosed) { - goto logged - } - var h2Err http2.StreamError - if errors.As(err, &h2Err) { - // ignore these errors - switch h2Err.Code { - case http2.ErrCodeStreamClosed: - goto logged - } - } - var h3Err *http3.Error - if errors.As(err, &h3Err) { - // ignore these errors - switch h3Err.ErrorCode { - case - http3.ErrCodeNoError, - http3.ErrCodeRequestCanceled: - goto logged - } - } - log.Err(err).Str("url", reqURL).Msg("http proxy error") - } - -logged: - if writeHeader { - rw.WriteHeader(http.StatusInternalServerError) - } - if p.AccessLogger != nil { - p.AccessLogger.LogError(r, err) - } -} - -// modifyResponse conditionally runs the optional ModifyResponse hook -// and reports whether the request should proceed. -func (p *ReverseProxy) modifyResponse(rw http.ResponseWriter, res *http.Response, origReq, req *http.Request) bool { - if p.ModifyResponse == nil { - return true - } - res.Request = origReq - err := p.ModifyResponse(res) - res.Request = req - if err != nil { - res.Body.Close() - p.errorHandler(rw, req, err, true) - return false - } - return true -} - -func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) { - p.HandlerFunc(rw, req) -} - -func (p *ReverseProxy) handler(rw http.ResponseWriter, req *http.Request) { - transport := p.Transport - - ctx := req.Context() - if ctx.Done() != nil { - // CloseNotifier predates context.Context, and has been - // entirely superseded by it. If the request contains - // a Context that carries a cancellation signal, don't - // bother spinning up a goroutine to watch the CloseNotify - // channel (if any). - // - // If the request Context has a nil Done channel (which - // means it is either context.Background, or a custom - // Context implementation with no cancellation signal), - // then consult the CloseNotifier if available. - } else if cn, ok := rw.(http.CloseNotifier); ok { - var cancel context.CancelFunc - ctx, cancel = context.WithCancel(ctx) - defer cancel() - notifyChan := cn.CloseNotify() - go func() { - select { - case <-notifyChan: - cancel() - case <-ctx.Done(): - } - }() - } - - outreq := req.Clone(ctx) - if req.ContentLength == 0 { - outreq.Body = nil // Issue 16036: nil Body for http.Transport retries - } - if outreq.Body != nil { - // Reading from the request body after returning from a handler is not - // allowed, and the RoundTrip goroutine that reads the Body can outlive - // this handler. This can lead to a crash if the handler panics (see - // Issue 46866). Although calling Close doesn't guarantee there isn't - // any Read in flight after the handle returns, in practice it's safe to - // read after closing it. - defer outreq.Body.Close() - } - if outreq.Header == nil { - outreq.Header = make(http.Header) // Issue 33142: historical behavior was to always allocate - } - - p.rewriteRequestURL(outreq) - outreq.Close = false - - reqUpType := httpheaders.UpgradeType(outreq.Header) - if !IsPrint(reqUpType) { - p.errorHandler(rw, req, fmt.Errorf("client tried to switch to invalid protocol %q", reqUpType), true) - return - } - - outreq.Header.Del("Forwarded") - httpheaders.RemoveHopByHopHeaders(outreq.Header) - - // Issue 21096: tell backend applications that care about trailer support - // that we support trailers. (We do, but we don't go out of our way to - // advertise that unless the incoming client request thought it was worth - // mentioning.) Note that we look at req.Header, not outreq.Header, since - // the latter has passed through removeHopByHopHeaders. - if httpguts.HeaderValuesContainsToken(req.Header["Te"], "trailers") { - outreq.Header.Set("Te", "trailers") - } - - // After stripping all the hop-by-hop connection headers above, add back any - // necessary for protocol upgrades, such as for websockets. - if reqUpType != "" { - outreq.Header.Set("Connection", "Upgrade") - outreq.Header.Set("Upgrade", reqUpType) - - if strings.EqualFold(reqUpType, "websocket") { - cleanWebsocketHeaders(outreq) - } - } - - // If we aren't the first proxy retain prior - // X-Forwarded-For information as a comma+space - // separated list and fold multiple headers into one. - prior, ok := outreq.Header[httpheaders.HeaderXForwardedFor] - omit := ok && prior == nil // Issue 38079: nil now means don't populate the header - - if !omit { - xff, _, err := net.SplitHostPort(req.RemoteAddr) - if err != nil { - xff = req.RemoteAddr - } - if len(prior) > 0 { - xff = strings.Join(prior, ", ") + ", " + xff - } - outreq.Header.Set(httpheaders.HeaderXForwardedFor, xff) - } - - var reqScheme string - if req.TLS != nil || req.Header.Get("X-Forwarded-Proto") == "https" { - reqScheme = "https" - } else { - reqScheme = "http" - } - - outreq.Header.Set(httpheaders.HeaderXForwardedMethod, req.Method) - outreq.Header.Set(httpheaders.HeaderXForwardedProto, reqScheme) - outreq.Header.Set(httpheaders.HeaderXForwardedHost, req.Host) - outreq.Header.Set(httpheaders.HeaderXForwardedURI, req.RequestURI) - - if _, ok := outreq.Header["User-Agent"]; !ok { - // If the outbound request doesn't have a User-Agent header set, - // don't send the default Go HTTP client User-Agent. - outreq.Header.Set("User-Agent", "") - } - - var ( - roundTripMutex sync.Mutex - roundTripDone bool - ) - trace := &httptrace.ClientTrace{ - Got1xxResponse: func(code int, header textproto.MIMEHeader) error { - roundTripMutex.Lock() - defer roundTripMutex.Unlock() - if roundTripDone { - // If RoundTrip has returned, don't try to further modify - // the ResponseWriter's header map. - return nil - } - h := rw.Header() - copyHeader(h, http.Header(header)) - rw.WriteHeader(code) - - // Clear headers, it's not automatically done by ResponseWriter.WriteHeader() for 1xx responses - clear(h) - return nil - }, - } - outreq = outreq.WithContext(httptrace.WithClientTrace(outreq.Context(), trace)) //nolint:contextcheck - - res, err := transport.RoundTrip(outreq) - - roundTripMutex.Lock() - roundTripDone = true - roundTripMutex.Unlock() - if err != nil { - p.errorHandler(rw, outreq, err, false) - res = &http.Response{ - Status: http.StatusText(http.StatusBadGateway), - StatusCode: http.StatusBadGateway, - Proto: req.Proto, - ProtoMajor: req.ProtoMajor, - ProtoMinor: req.ProtoMinor, - Header: http.Header{}, - Body: io.NopCloser(bytes.NewReader([]byte("Origin server is not reachable."))), - Request: req, - TLS: req.TLS, - } - } - - if p.AccessLogger != nil { - defer func() { - p.AccessLogger.Log(req, res) - }() - } - - httpheaders.RemoveServiceHeaders(res.Header) - - // Deal with 101 Switching Protocols responses: (WebSocket, h2c, etc) - if res.StatusCode == http.StatusSwitchingProtocols { - if !p.modifyResponse(rw, res, req, outreq) { - return - } - p.handleUpgradeResponse(rw, outreq, res) - return - } - - httpheaders.RemoveHopByHopHeaders(res.Header) - - if !p.modifyResponse(rw, res, req, outreq) { - return - } - - copyHeader(rw.Header(), res.Header) - - // The "Trailer" header isn't included in the Transport's response, - // at least for *http.Transport. Build it up from Trailer. - announcedTrailers := len(res.Trailer) - if announcedTrailers > 0 { - trailerKeys := make([]string, 0, len(res.Trailer)) - for k := range res.Trailer { - trailerKeys = append(trailerKeys, k) - } - rw.Header().Add("Trailer", strings.Join(trailerKeys, ", ")) - } - - rw.WriteHeader(res.StatusCode) - - err = U.CopyCloseWithContext(ctx, rw, res.Body, int(res.ContentLength)) // close now, instead of defer, to populate res.Trailer - if err != nil { - if !errors.Is(err, context.Canceled) { - p.errorHandler(rw, req, err, false) - } - return - } - - if len(res.Trailer) > 0 { - // Force chunking if we saw a response trailer. - // This prevents net/http from calculating the length for short - // bodies and adding a Content-Length. - http.NewResponseController(rw).Flush() - } - - if len(res.Trailer) == announcedTrailers { - copyHeader(rw.Header(), res.Trailer) - return - } - - for k, vv := range res.Trailer { - k = http.TrailerPrefix + k - for _, v := range vv { - rw.Header().Add(k, v) - } - } -} - -// reference: https://github.com/traefik/traefik/blob/master/pkg/proxy/httputil/proxy.go -// https://tools.ietf.org/html/rfc6455#page-20 -func cleanWebsocketHeaders(req *http.Request) { - req.Header["Sec-WebSocket-Key"] = req.Header["Sec-Websocket-Key"] - delete(req.Header, "Sec-Websocket-Key") - - req.Header["Sec-WebSocket-Extensions"] = req.Header["Sec-Websocket-Extensions"] - delete(req.Header, "Sec-Websocket-Extensions") - - req.Header["Sec-WebSocket-Accept"] = req.Header["Sec-Websocket-Accept"] - delete(req.Header, "Sec-Websocket-Accept") - - req.Header["Sec-WebSocket-Protocol"] = req.Header["Sec-Websocket-Protocol"] - delete(req.Header, "Sec-Websocket-Protocol") - - req.Header["Sec-WebSocket-Version"] = req.Header["Sec-Websocket-Version"] - delete(req.Header, "Sec-Websocket-Version") -} - -func (p *ReverseProxy) handleUpgradeResponse(rw http.ResponseWriter, req *http.Request, res *http.Response) { - reqUpType := httpheaders.UpgradeType(req.Header) - resUpType := httpheaders.UpgradeType(res.Header) - if !IsPrint(resUpType) { // We know reqUpType is ASCII, it's checked by the caller. - p.errorHandler(rw, req, fmt.Errorf("backend tried to switch to invalid protocol %q", resUpType), true) - return - } - if !strings.EqualFold(reqUpType, resUpType) { - p.errorHandler(rw, req, fmt.Errorf("backend tried to switch protocol %q when %q was requested", resUpType, reqUpType), true) - return - } - - backConn, ok := res.Body.(io.ReadWriteCloser) - if !ok { - p.errorHandler(rw, req, errors.New("internal error: 101 switching protocols response with non-writable body"), true) - return - } - - rc := http.NewResponseController(rw) - conn, brw, hijackErr := rc.Hijack() - if errors.Is(hijackErr, http.ErrNotSupported) { - p.errorHandler(rw, req, fmt.Errorf("can't switch protocols using non-Hijacker ResponseWriter type %T", rw), true) - return - } - - backConnCloseCh := make(chan bool) - go func() { - // Ensure that the cancellation of a request closes the backend. - // See issue https://golang.org/issue/35559. - select { - case <-req.Context().Done(): - case <-backConnCloseCh: - } - backConn.Close() - }() - defer close(backConnCloseCh) - - if hijackErr != nil { - p.errorHandler(rw, req, fmt.Errorf("hijack failed on protocol switch: %w", hijackErr), true) - return - } - defer conn.Close() - - copyHeader(rw.Header(), res.Header) - - res.Header = rw.Header() - res.Body = nil // so res.Write only writes the headers; we have res.Body in backConn above - if err := res.Write(brw); err != nil { - //nolint:errorlint - p.errorHandler(rw, req, fmt.Errorf("response write: %s", err), true) - return - } - if err := brw.Flush(); err != nil { - //nolint:errorlint - p.errorHandler(rw, req, fmt.Errorf("response flush: %s", err), true) - return - } - - bdp := U.NewBidirectionalPipe(req.Context(), conn, backConn) - //nolint:errcheck - bdp.Start() -} - -func IsPrint(s string) bool { - for _, r := range s { - if r < ' ' || r > '~' { - return false - } - } - return true -} diff --git a/internal/net/gphttp/reverseproxy/reverse_proxy_benchmark_test.go b/internal/net/gphttp/reverseproxy/reverse_proxy_benchmark_test.go deleted file mode 100644 index 4717fccb..00000000 --- a/internal/net/gphttp/reverseproxy/reverse_proxy_benchmark_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package reverseproxy - -import ( - "io" - "net/http" - "net/url" - "strings" - "testing" - - nettypes "github.com/yusing/godoxy/internal/net/types" -) - -type noopTransport struct{} - -func (t noopTransport) RoundTrip(req *http.Request) (*http.Response, error) { - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(strings.NewReader("Hello, world!")), - Request: req, - ContentLength: int64(len("Hello, world!")), - Header: http.Header{}, - }, nil -} - -type noopResponseWriter struct{} - -func (w noopResponseWriter) Header() http.Header { - return http.Header{} -} - -func (w noopResponseWriter) Write(b []byte) (int, error) { - return len(b), nil -} - -func (w noopResponseWriter) WriteHeader(statusCode int) { -} - -func BenchmarkReverseProxy(b *testing.B) { - var w noopResponseWriter - var req = http.Request{ - Method: "GET", - URL: &url.URL{Scheme: "http", Host: "test"}, - Body: io.NopCloser(strings.NewReader("Hello, world!")), - } - proxy := NewReverseProxy("test", nettypes.MustParseURL("http://localhost:8080"), noopTransport{}) - for b.Loop() { - proxy.ServeHTTP(w, &req) - } -} diff --git a/internal/route/provider/docker.go b/internal/route/provider/docker.go index 65d5915b..680af79f 100755 --- a/internal/route/provider/docker.go +++ b/internal/route/provider/docker.go @@ -14,7 +14,6 @@ import ( "github.com/yusing/godoxy/internal/route" "github.com/yusing/godoxy/internal/serialization" "github.com/yusing/godoxy/internal/types" - "github.com/yusing/godoxy/internal/utils/strutils" "github.com/yusing/godoxy/internal/watcher" ) @@ -157,7 +156,7 @@ func (p *DockerProvider) routesFromContainerLabels(container *types.Container) ( // check if it is an alias reference switch alias[0] { case aliasRefPrefix, aliasRefPrefixAlt: - index, err := strutils.Atoi(alias[1:]) + index, err := strconv.Atoi(alias[1:]) if err != nil { errs.Add(err) break diff --git a/internal/route/reverse_proxy.go b/internal/route/reverse_proxy.go index deae8078..a97011d1 100755 --- a/internal/route/reverse_proxy.go +++ b/internal/route/reverse_proxy.go @@ -13,13 +13,13 @@ import ( gphttp "github.com/yusing/godoxy/internal/net/gphttp" "github.com/yusing/godoxy/internal/net/gphttp/loadbalancer" "github.com/yusing/godoxy/internal/net/gphttp/middleware" - "github.com/yusing/godoxy/internal/net/gphttp/reverseproxy" nettypes "github.com/yusing/godoxy/internal/net/types" "github.com/yusing/godoxy/internal/route/routes" "github.com/yusing/godoxy/internal/task" "github.com/yusing/godoxy/internal/types" "github.com/yusing/godoxy/internal/watcher/health/monitor" "github.com/yusing/godoxy/pkg" + "github.com/yusing/goutils/http/reverseproxy" ) type ReveseProxyRoute struct { @@ -59,7 +59,7 @@ func NewReverseProxyRoute(base *Route) (*ReveseProxyRoute, gperr.Error) { } service := base.Name() - rp := reverseproxy.NewReverseProxy(service, proxyURL, trans) + rp := reverseproxy.NewReverseProxy(service, &proxyURL.URL, trans) if len(base.Middlewares) > 0 { err := middleware.PatchReverseProxy(rp, base.Middlewares) diff --git a/internal/route/route.go b/internal/route/route.go index 70e14dd8..2fafe066 100644 --- a/internal/route/route.go +++ b/internal/route/route.go @@ -19,7 +19,7 @@ import ( "github.com/yusing/godoxy/internal/proxmox" "github.com/yusing/godoxy/internal/task" "github.com/yusing/godoxy/internal/types" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" "github.com/yusing/godoxy/internal/common" config "github.com/yusing/godoxy/internal/config/types" diff --git a/internal/route/rules/do.go b/internal/route/rules/do.go index 687a18a2..4936d20d 100644 --- a/internal/route/rules/do.go +++ b/internal/route/rules/do.go @@ -8,9 +8,9 @@ import ( "github.com/yusing/godoxy/internal/gperr" gphttp "github.com/yusing/godoxy/internal/net/gphttp" - "github.com/yusing/godoxy/internal/net/gphttp/reverseproxy" nettypes "github.com/yusing/godoxy/internal/net/types" - "github.com/yusing/godoxy/internal/utils/strutils" + "github.com/yusing/goutils/http/reverseproxy" + strutils "github.com/yusing/goutils/strings" ) type ( @@ -164,7 +164,7 @@ var commands = map[string]struct { if target.Scheme == "" { target.Scheme = "http" } - rp := reverseproxy.NewReverseProxy("", target, gphttp.NewTransport()) + rp := reverseproxy.NewReverseProxy("", &target.URL, gphttp.NewTransport()) return ReturningCommand(rp.ServeHTTP) }, }, diff --git a/internal/route/rules/on.go b/internal/route/rules/on.go index e3559f23..3149588a 100644 --- a/internal/route/rules/on.go +++ b/internal/route/rules/on.go @@ -10,7 +10,7 @@ import ( "github.com/yusing/godoxy/internal/gperr" nettypes "github.com/yusing/godoxy/internal/net/types" "github.com/yusing/godoxy/internal/route/routes" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) type RuleOn struct { diff --git a/internal/route/stream/tcp_tcp.go b/internal/route/stream/tcp_tcp.go index 18d4a0ad..d32ab4ec 100644 --- a/internal/route/stream/tcp_tcp.go +++ b/internal/route/stream/tcp_tcp.go @@ -8,7 +8,7 @@ import ( "github.com/rs/zerolog" config "github.com/yusing/godoxy/internal/config/types" nettypes "github.com/yusing/godoxy/internal/net/types" - "github.com/yusing/godoxy/internal/utils" + ioutils "github.com/yusing/goutils/io" "go.uber.org/atomic" ) @@ -153,7 +153,7 @@ func (s *TCPTCPStream) handle(ctx context.Context, conn net.Conn) { } } - pipe := utils.NewBidirectionalPipe(ctx, src, dst) + pipe := ioutils.NewBidirectionalPipe(ctx, src, dst) if err := pipe.Start(); err != nil && !s.closed.Load() { logErr(s, err, "error in bidirectional pipe") } diff --git a/internal/route/stream/udp_udp.go b/internal/route/stream/udp_udp.go index 88d4701d..50afa144 100644 --- a/internal/route/stream/udp_udp.go +++ b/internal/route/stream/udp_udp.go @@ -12,7 +12,7 @@ import ( "github.com/rs/zerolog" config "github.com/yusing/godoxy/internal/config/types" nettypes "github.com/yusing/godoxy/internal/net/types" - "github.com/yusing/godoxy/internal/utils/synk" + "github.com/yusing/goutils/synk" "go.uber.org/atomic" ) diff --git a/internal/route/types/port.go b/internal/route/types/port.go index 13836d5e..8d88b82b 100644 --- a/internal/route/types/port.go +++ b/internal/route/types/port.go @@ -4,7 +4,7 @@ import ( "strconv" "github.com/yusing/godoxy/internal/gperr" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) type Port struct { diff --git a/internal/serialization/serialization.go b/internal/serialization/serialization.go index 09d300d3..99eb536d 100644 --- a/internal/serialization/serialization.go +++ b/internal/serialization/serialization.go @@ -15,7 +15,7 @@ import ( "github.com/puzpuzpuz/xsync/v4" "github.com/yusing/godoxy/internal/gperr" "github.com/yusing/godoxy/internal/utils" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) type SerializedObject = map[string]any diff --git a/internal/types/health.go b/internal/types/health.go index 0e24edad..e99192db 100644 --- a/internal/types/health.go +++ b/internal/types/health.go @@ -8,7 +8,7 @@ import ( "time" "github.com/yusing/godoxy/internal/task" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) type ( diff --git a/internal/types/loadbalancer.go b/internal/types/loadbalancer.go index 18d312ff..2dae15a5 100644 --- a/internal/types/loadbalancer.go +++ b/internal/types/loadbalancer.go @@ -4,7 +4,7 @@ import ( "net/http" nettypes "github.com/yusing/godoxy/internal/net/types" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) type ( diff --git a/internal/types/routes.go b/internal/types/routes.go index 3d3301f0..7b5e701c 100644 --- a/internal/types/routes.go +++ b/internal/types/routes.go @@ -5,10 +5,10 @@ import ( "github.com/yusing/godoxy/agent/pkg/agent" "github.com/yusing/godoxy/internal/homepage" - "github.com/yusing/godoxy/internal/net/gphttp/reverseproxy" nettypes "github.com/yusing/godoxy/internal/net/types" "github.com/yusing/godoxy/internal/task" "github.com/yusing/godoxy/internal/utils/pool" + "github.com/yusing/goutils/http/reverseproxy" ) type ( diff --git a/internal/utils/buf_writer.go b/internal/utils/buf_writer.go deleted file mode 100644 index 75df4af3..00000000 --- a/internal/utils/buf_writer.go +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Modified from bufio.Writer by yusing . -package utils - -import ( - "io" - "unicode/utf8" -) - -// buffered output - -// BufferedWriter implements buffering for an [io.BufferedWriter] object. -// If an error occurs writing to a [BufferedWriter], no more data will be -// accepted and all subsequent writes, and [BufferedWriter.Flush], will return the error. -// After all data has been written, the client should call the -// [BufferedWriter.Flush] method to guarantee all data has been forwarded to -// the underlying [io.BufferedWriter]. -type BufferedWriter struct { - err error - buf []byte - n int - wr io.Writer -} - -// NewBufferedWriter returns a new [BufferedWriter] whose buffer has at least the specified -// size. If the argument io.Writer is already a [BufferedWriter] with large enough -// size, it returns the underlying [BufferedWriter]. -func NewBufferedWriter(w io.Writer, size int) *BufferedWriter { - // Is it already a Writer? - b, ok := w.(*BufferedWriter) - if ok && len(b.buf) >= size { - return b - } - return &BufferedWriter{ - buf: bytesPool.GetSized(size), - wr: w, - } -} - -// Size returns the size of the underlying buffer in bytes. -func (b *BufferedWriter) Size() int { return len(b.buf) } - -func (b *BufferedWriter) Resize(size int) error { - err := b.Flush() - if err != nil { - return err - } - if cap(b.buf) >= size { - b.buf = b.buf[:size] - } else { - b.Release() - b.buf = bytesPool.GetSized(size) - } - b.err = nil - b.n = 0 - return nil -} - -func (b *BufferedWriter) Release() { - bytesPool.Put(b.buf) -} - -// Flush writes any buffered data to the underlying [io.Writer]. -func (b *BufferedWriter) Flush() error { - if b.err != nil { - return b.err - } - if b.n == 0 { - return nil - } - n, err := b.wr.Write(b.buf[0:b.n]) - if n < b.n && err == nil { - err = io.ErrShortWrite - } - if err != nil { - if n > 0 && n < b.n { - copy(b.buf[0:b.n-n], b.buf[n:b.n]) - } - b.n -= n - b.err = err - return err - } - b.n = 0 - return nil -} - -// Available returns how many bytes are unused in the buffer. -func (b *BufferedWriter) Available() int { return len(b.buf) - b.n } - -// AvailableBuffer returns an empty buffer with b.Available() capacity. -// This buffer is intended to be appended to and -// passed to an immediately succeeding [BufferedWriter.Write] call. -// The buffer is only valid until the next write operation on b. -func (b *BufferedWriter) AvailableBuffer() []byte { - return b.buf[b.n:][:0] -} - -// Buffered returns the number of bytes that have been written into the current buffer. -func (b *BufferedWriter) Buffered() int { return b.n } - -// Write writes the contents of p into the buffer. -// It returns the number of bytes written. -// If nn < len(p), it also returns an error explaining -// why the write is short. -func (b *BufferedWriter) Write(p []byte) (nn int, err error) { - for len(p) > b.Available() && b.err == nil { - var n int - if b.Buffered() == 0 { - // Large write, empty buffer. - // Write directly from p to avoid copy. - n, b.err = b.wr.Write(p) - } else { - n = copy(b.buf[b.n:], p) - b.n += n - b.Flush() - } - nn += n - p = p[n:] - } - if b.err != nil { - return nn, b.err - } - n := copy(b.buf[b.n:], p) - b.n += n - nn += n - return nn, nil -} - -// WriteByte writes a single byte. -func (b *BufferedWriter) WriteByte(c byte) error { - if b.err != nil { - return b.err - } - if b.Available() <= 0 && b.Flush() != nil { - return b.err - } - b.buf[b.n] = c - b.n++ - return nil -} - -// WriteRune writes a single Unicode code point, returning -// the number of bytes written and any error. -func (b *BufferedWriter) WriteRune(r rune) (size int, err error) { - // Compare as uint32 to correctly handle negative runes. - if uint32(r) < utf8.RuneSelf { - err = b.WriteByte(byte(r)) - if err != nil { - return 0, err - } - return 1, nil - } - if b.err != nil { - return 0, b.err - } - n := b.Available() - if n < utf8.UTFMax { - if b.Flush(); b.err != nil { - return 0, b.err - } - n = b.Available() - if n < utf8.UTFMax { - // Can only happen if buffer is silly small. - return b.WriteString(string(r)) - } - } - size = utf8.EncodeRune(b.buf[b.n:], r) - b.n += size - return size, nil -} - -// WriteString writes a string. -// It returns the number of bytes written. -// If the count is less than len(s), it also returns an error explaining -// why the write is short. -func (b *BufferedWriter) WriteString(s string) (int, error) { - var sw io.StringWriter - tryStringWriter := true - - nn := 0 - for len(s) > b.Available() && b.err == nil { - var n int - if b.Buffered() == 0 && sw == nil && tryStringWriter { - // Check at most once whether b.wr is a StringWriter. - sw, tryStringWriter = b.wr.(io.StringWriter) - } - if b.Buffered() == 0 && tryStringWriter { - // Large write, empty buffer, and the underlying writer supports - // WriteString: forward the write to the underlying StringWriter. - // This avoids an extra copy. - n, b.err = sw.WriteString(s) - } else { - n = copy(b.buf[b.n:], s) - b.n += n - b.Flush() - } - nn += n - s = s[n:] - } - if b.err != nil { - return nn, b.err - } - n := copy(b.buf[b.n:], s) - b.n += n - nn += n - return nn, nil -} diff --git a/internal/utils/go.mod b/internal/utils/go.mod index a19455f1..526fa67c 100644 --- a/internal/utils/go.mod +++ b/internal/utils/go.mod @@ -6,8 +6,8 @@ require ( github.com/puzpuzpuz/xsync/v4 v4.2.0 github.com/rs/zerolog v1.34.0 github.com/stretchr/testify v1.11.1 + github.com/yusing/goutils v0.2.1 go.uber.org/atomic v1.11.0 - golang.org/x/text v0.29.0 ) require ( @@ -16,5 +16,6 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect golang.org/x/sys v0.36.0 // indirect + golang.org/x/text v0.29.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/internal/utils/go.sum b/internal/utils/go.sum index a94b6af2..923171bb 100644 --- a/internal/utils/go.sum +++ b/internal/utils/go.sum @@ -19,6 +19,8 @@ github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/yusing/goutils v0.2.1 h1:KjoCrNO0otthaPCZPfQY+5GKsqs5+J77CxP+TNHYa/Y= +github.com/yusing/goutils v0.2.1/go.mod h1:v6RZsMRdzcts4udSg0vqUIFvaD0OaUMPTwYJZ4XnQYo= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/utils/io.go b/internal/utils/io.go deleted file mode 100644 index 0778c3e6..00000000 --- a/internal/utils/io.go +++ /dev/null @@ -1,259 +0,0 @@ -package utils - -import ( - "context" - "errors" - "io" - "net/http" - "sync" - "syscall" - - "github.com/yusing/godoxy/internal/utils/synk" -) - -// TODO: move to "utils/io". -type ( - FileReader struct { - Path string - } - - ContextReader struct { - ctx context.Context - io.Reader - } - - ContextWriter struct { - ctx context.Context - io.Writer - } - - Pipe struct { - r ContextReader - w ContextWriter - } - - BidirectionalPipe struct { - pSrcDst *Pipe - pDstSrc *Pipe - } - - HookCloser struct { - c io.ReadCloser - hook func() - } -) - -func NewContextReader(ctx context.Context, r io.Reader) *ContextReader { - return &ContextReader{ctx: ctx, Reader: r} -} - -func NewContextWriter(ctx context.Context, w io.Writer) *ContextWriter { - return &ContextWriter{ctx: ctx, Writer: w} -} - -func (r *ContextReader) Read(p []byte) (int, error) { - select { - case <-r.ctx.Done(): - return 0, r.ctx.Err() - default: - return r.Reader.Read(p) - } -} - -func (w *ContextWriter) Write(p []byte) (int, error) { - select { - case <-w.ctx.Done(): - return 0, w.ctx.Err() - default: - return w.Writer.Write(p) - } -} - -func NewPipe(ctx context.Context, r io.ReadCloser, w io.WriteCloser) *Pipe { - return &Pipe{ - r: ContextReader{ctx: ctx, Reader: r}, - w: ContextWriter{ctx: ctx, Writer: w}, - } -} - -func (p *Pipe) Start() (err error) { - err = CopyClose(&p.w, &p.r, 0) - switch { - case - // NOTE: ignoring broken pipe and connection reset by peer - errors.Is(err, syscall.EPIPE), - errors.Is(err, syscall.ECONNRESET): - return nil - } - return err -} - -func NewBidirectionalPipe(ctx context.Context, rw1 io.ReadWriteCloser, rw2 io.ReadWriteCloser) BidirectionalPipe { - return BidirectionalPipe{ - pSrcDst: NewPipe(ctx, rw1, rw2), - pDstSrc: NewPipe(ctx, rw2, rw1), - } -} - -func (p BidirectionalPipe) Start() error { - var wg sync.WaitGroup - var srcErr, dstErr error - wg.Go(func() { - srcErr = p.pSrcDst.Start() - }) - wg.Go(func() { - dstErr = p.pDstSrc.Start() - }) - wg.Wait() - return errors.Join(srcErr, dstErr) -} - -type flushErrorInterface interface { - FlushError() error -} - -type flusherWrapper struct { - rw http.Flusher -} - -type rwUnwrapper interface { - Unwrap() http.ResponseWriter -} - -func (f *flusherWrapper) FlushError() error { - f.rw.Flush() - return nil -} - -func getHTTPFlusher(dst io.Writer) flushErrorInterface { - // pre-unwrap the flusher to prevent unwrap and check in every loop - if rw, ok := dst.(http.ResponseWriter); ok { - for { - switch t := rw.(type) { - case flushErrorInterface: - return t - case http.Flusher: - return &flusherWrapper{rw: t} - case rwUnwrapper: - rw = t.Unwrap() - default: - return nil - } - } - } - return nil -} - -const copyBufSize = synk.SizedPoolThreshold - -var bytesPool = synk.GetBytesPool() - -// ReadAllBody reads the body of the response into a buffer and returns it and a function to release the buffer. -func ReadAllBody(resp *http.Response) (buf []byte, release func(), err error) { - if contentLength := resp.ContentLength; contentLength > 0 { - buf = bytesPool.GetSized(int(contentLength)) - _, err = io.ReadFull(resp.Body, buf) - if err != nil { - bytesPool.Put(buf) - return nil, nil, err - } - return buf, func() { bytesPool.Put(buf) }, nil - } - buf, err = io.ReadAll(resp.Body) - if err != nil { - bytesPool.Put(buf) - return nil, nil, err - } - return buf, func() { bytesPool.Put(buf) }, nil -} - -// NewHookCloser wraps a io.ReadCloser and calls the hook function when the closer is closed. -func NewHookCloser(c io.ReadCloser, hook func()) *HookCloser { - return &HookCloser{hook: hook, c: c} -} - -// Close calls the hook function and closes the underlying reader -func (r *HookCloser) Close() error { - r.hook() - return r.c.Close() -} - -// Read reads from the underlying reader. -func (r *HookCloser) Read(p []byte) (int, error) { - return r.c.Read(p) -} - -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// This is a copy of io.Copy with context and HTTP flusher handling -// Author: yusing . -func CopyClose(dst *ContextWriter, src *ContextReader, sizeHint int) (err error) { - size := copyBufSize - if l, ok := src.Reader.(*io.LimitedReader); ok { - if int64(size) > l.N { - if l.N < 1 { - size = 1 - } else { - size = int(l.N) - } - } - } else if sizeHint > 0 { - size = min(size, sizeHint) - } - buf := bytesPool.GetSized(size) - defer bytesPool.Put(buf) - // close both as soon as one of them is done - wCloser, wCanClose := dst.Writer.(io.Closer) - rCloser, rCanClose := src.Reader.(io.Closer) - if wCanClose || rCanClose { - go func() { - select { - case <-src.ctx.Done(): - case <-dst.ctx.Done(): - } - if rCanClose { - defer rCloser.Close() - } - if wCanClose { - defer wCloser.Close() - } - }() - } - flusher := getHTTPFlusher(dst.Writer) - for { - nr, er := src.Reader.Read(buf) - if nr > 0 { - nw, ew := dst.Writer.Write(buf[0:nr]) - if nw < 0 || nr < nw { - nw = 0 - if ew == nil { - ew = errors.New("invalid write result") - } - } - if ew != nil { - err = ew - return - } - if nr != nw { - err = io.ErrShortWrite - return - } - if flusher != nil { - err = flusher.FlushError() - if err != nil { - return err - } - } - } - if er != nil { - if er != io.EOF { - err = er - } - return - } - } -} - -func CopyCloseWithContext(ctx context.Context, dst io.Writer, src io.Reader, sizeHint int) (err error) { - return CopyClose(NewContextWriter(ctx, dst), NewContextReader(ctx, src), sizeHint) -} diff --git a/internal/utils/nearest_field.go b/internal/utils/nearest_field.go index 56511921..a44e7ec2 100644 --- a/internal/utils/nearest_field.go +++ b/internal/utils/nearest_field.go @@ -3,7 +3,7 @@ package utils import ( "reflect" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) func NearestField(input string, s any) string { diff --git a/internal/utils/strutils/ansi/ansi.go b/internal/utils/strutils/ansi/ansi.go deleted file mode 100644 index f450eabf..00000000 --- a/internal/utils/strutils/ansi/ansi.go +++ /dev/null @@ -1,47 +0,0 @@ -package ansi - -import ( - "regexp" -) - -var ansiRegexp = regexp.MustCompile(`\x1b\[[0-9;]*m`) - -const ( - BrightRed = "\x1b[91m" - BrightGreen = "\x1b[92m" - BrightYellow = "\x1b[93m" - BrightCyan = "\x1b[96m" - BrightWhite = "\x1b[97m" - Bold = "\x1b[1m" - Reset = "\x1b[0m" - - HighlightRed = BrightRed + Bold - HighlightGreen = BrightGreen + Bold - HighlightYellow = BrightYellow + Bold - HighlightCyan = BrightCyan + Bold - HighlightWhite = BrightWhite + Bold -) - -func Error(s string) string { - return WithANSI(s, HighlightRed) -} - -func Success(s string) string { - return WithANSI(s, HighlightGreen) -} - -func Warning(s string) string { - return WithANSI(s, HighlightYellow) -} - -func Info(s string) string { - return WithANSI(s, HighlightCyan) -} - -func WithANSI(s string, ansi string) string { - return ansi + s + Reset -} - -func StripANSI(s string) string { - return ansiRegexp.ReplaceAllString(s, "") -} diff --git a/internal/utils/strutils/filepath.go b/internal/utils/strutils/filepath.go deleted file mode 100644 index 86afe0e3..00000000 --- a/internal/utils/strutils/filepath.go +++ /dev/null @@ -1,11 +0,0 @@ -package strutils - -import "strings" - -// IsValidFilename checks if a filename is safe and doesn't contain path traversal attempts -// Returns true if the filename is valid, false otherwise -func IsValidFilename(filename string) bool { - return !strings.Contains(filename, "/") && - !strings.Contains(filename, "\\") && - !strings.Contains(filename, "..") -} diff --git a/internal/utils/strutils/format.go b/internal/utils/strutils/format.go deleted file mode 100644 index f4cec879..00000000 --- a/internal/utils/strutils/format.go +++ /dev/null @@ -1,221 +0,0 @@ -package strutils - -import ( - "fmt" - "math" - "strconv" - "time" -) - -// AppendDuration appends a duration to a buffer with the following format: -// - 1 ns -// - 1 ms -// - 1 seconds -// - 1 minutes and 1 seconds -// - 1 hours, 1 minutes and 1 seconds -// - 1 days, 1 hours and 1 minutes (ignore seconds if days >= 1) -func AppendDuration(d time.Duration, buf []byte) []byte { - if d < 0 { - buf = append(buf, '-') - d = -d - } - - if d == 0 { - return append(buf, []byte("0 Seconds")...) - } - - switch { - case d < time.Millisecond: - buf = strconv.AppendInt(buf, d.Nanoseconds(), 10) - buf = append(buf, []byte(" ns")...) - return buf - case d < time.Second: - buf = strconv.AppendInt(buf, d.Milliseconds(), 10) - buf = append(buf, []byte(" ms")...) - return buf - } - - // Get total seconds from duration - totalSeconds := int64(d.Seconds()) - - // Calculate days, hours, minutes, and seconds - days := totalSeconds / (24 * 3600) - hours := (totalSeconds % (24 * 3600)) / 3600 - minutes := (totalSeconds % 3600) / 60 - seconds := totalSeconds % 60 - - idxPartBeg := 0 - if days > 0 { - buf = strconv.AppendInt(buf, days, 10) - buf = fmt.Appendf(buf, " day%s, ", Pluralize(days)) - } - if hours > 0 { - idxPartBeg = len(buf) - 2 - buf = strconv.AppendInt(buf, hours, 10) - buf = fmt.Appendf(buf, " hour%s, ", Pluralize(hours)) - } - if minutes > 0 { - idxPartBeg = len(buf) - 2 - buf = strconv.AppendInt(buf, minutes, 10) - buf = fmt.Appendf(buf, " minute%s, ", Pluralize(minutes)) - } - if seconds > 0 && totalSeconds < 3600 { - idxPartBeg = len(buf) - 2 - buf = strconv.AppendInt(buf, seconds, 10) - buf = fmt.Appendf(buf, " second%s, ", Pluralize(seconds)) - } - // remove last comma and space - buf = buf[:len(buf)-2] - if idxPartBeg > 0 && idxPartBeg < len(buf) { - // replace last part ', ' with ' and ' in-place, alloc-free - // ', ' is 2 bytes, ' and ' is 5 bytes, so we need to make room for 3 more bytes - tailLen := len(buf) - (idxPartBeg + 2) - buf = append(buf, "000"...) // append 3 bytes for ' and ' - copy(buf[idxPartBeg+5:], buf[idxPartBeg+2:idxPartBeg+2+tailLen]) // shift tail right by 3 - copy(buf[idxPartBeg:], " and ") // overwrite ', ' with ' and ' - } - return buf -} - -func FormatDuration(d time.Duration) string { - return string(AppendDuration(d, nil)) -} - -func FormatLastSeen(t time.Time) string { - if t.IsZero() { - return "never" - } - return FormatTime(t) -} - -func appendRound(f float64, buf []byte) []byte { - return strconv.AppendInt(buf, int64(math.Round(f)), 10) -} - -func appendFloat(f float64, buf []byte) []byte { - f = math.Round(f*100) / 100 - if f == 0 { - return buf - } - return strconv.AppendFloat(buf, f, 'f', -1, 64) -} - -func AppendTime(t time.Time, buf []byte) []byte { - if t.IsZero() { - return append(buf, []byte("never")...) - } - return AppendTimeWithReference(t, time.Now(), buf) -} - -func FormatTime(t time.Time) string { - return string(AppendTime(t, nil)) -} - -func FormatUnixTime(t int64) string { - return FormatTime(time.Unix(t, 0)) -} - -func FormatTimeWithReference(t, ref time.Time) string { - return string(AppendTimeWithReference(t, ref, nil)) -} - -func AppendTimeWithReference(t, ref time.Time, buf []byte) []byte { - if t.IsZero() { - return append(buf, []byte("never")...) - } - diff := t.Sub(ref) - absDiff := diff.Abs() - switch { - case absDiff < time.Second: - return append(buf, []byte("now")...) - case absDiff < 3*time.Second: - if diff < 0 { - return append(buf, []byte("just now")...) - } - fallthrough - case absDiff < 60*time.Second: - if diff < 0 { - buf = appendRound(absDiff.Seconds(), buf) - buf = append(buf, []byte(" seconds ago")...) - } else { - buf = append(buf, []byte("in ")...) - buf = appendRound(absDiff.Seconds(), buf) - buf = append(buf, []byte(" seconds")...) - } - return buf - case absDiff < 60*time.Minute: - if diff < 0 { - buf = appendRound(absDiff.Minutes(), buf) - buf = append(buf, []byte(" minutes ago")...) - } else { - buf = append(buf, []byte("in ")...) - buf = appendRound(absDiff.Minutes(), buf) - buf = append(buf, []byte(" minutes")...) - } - return buf - case absDiff < 24*time.Hour: - if diff < 0 { - buf = appendRound(absDiff.Hours(), buf) - buf = append(buf, []byte(" hours ago")...) - } else { - buf = append(buf, []byte("in ")...) - buf = appendRound(absDiff.Hours(), buf) - buf = append(buf, []byte(" hours")...) - } - return buf - case t.Year() == ref.Year(): - return t.AppendFormat(buf, "01-02 15:04:05") - default: - return t.AppendFormat(buf, "2006-01-02 15:04:05") - } -} - -func FormatByteSize[T ~int | ~uint | ~int64 | ~uint64 | ~float64](size T) string { - return string(AppendByteSize(size, nil)) -} - -func AppendByteSize[T ~int | ~uint | ~int64 | ~uint64 | ~float64](size T, buf []byte) []byte { - const ( - _ = (1 << (10 * iota)) - kb - mb - gb - tb - pb - ) - switch { - case size < kb: - switch any(size).(type) { - case int, int64: - buf = strconv.AppendInt(buf, int64(size), 10) - case uint, uint64: - buf = strconv.AppendUint(buf, uint64(size), 10) - case float64: - buf = appendFloat(float64(size), buf) - } - buf = append(buf, []byte(" B")...) - case size < mb: - buf = appendFloat(float64(size)/kb, buf) - buf = append(buf, []byte(" KiB")...) - case size < gb: - buf = appendFloat(float64(size)/mb, buf) - buf = append(buf, []byte(" MiB")...) - case size < tb: - buf = appendFloat(float64(size)/gb, buf) - buf = append(buf, []byte(" GiB")...) - case size < pb: - buf = appendFloat(float64(size/gb)/kb, buf) - buf = append(buf, []byte(" TiB")...) - default: - buf = appendFloat(float64(size/tb)/kb, buf) - buf = append(buf, []byte(" PiB")...) - } - return buf -} - -func Pluralize(n int64) string { - if n > 1 { - return "s" - } - return "" -} diff --git a/internal/utils/strutils/format_test.go b/internal/utils/strutils/format_test.go deleted file mode 100644 index 1a2fa873..00000000 --- a/internal/utils/strutils/format_test.go +++ /dev/null @@ -1,302 +0,0 @@ -package strutils_test - -import ( - "testing" - "time" - - "github.com/stretchr/testify/require" - . "github.com/yusing/godoxy/internal/utils/strutils" -) - -func mustParseTime(t *testing.T, layout, value string) time.Time { - t.Helper() - time, err := time.Parse(layout, value) - if err != nil { - t.Fatalf("failed to parse time: %s", err) - } - return time -} - -func TestFormatTime(t *testing.T) { - now := mustParseTime(t, time.RFC3339, "2021-06-15T12:30:30Z") - - tests := []struct { - name string - time time.Time - expected string - expectedLength int - }{ - { - name: "now", - time: now.Add(100 * time.Millisecond), - expected: "now", - }, - { - name: "just now (past within 3 seconds)", - time: now.Add(-1 * time.Second), - expected: "just now", - }, - { - name: "seconds ago", - time: now.Add(-10 * time.Second), - expected: "10 seconds ago", - }, - { - name: "in seconds", - time: now.Add(10 * time.Second), - expected: "in 10 seconds", - }, - { - name: "minutes ago", - time: now.Add(-10 * time.Minute), - expected: "10 minutes ago", - }, - { - name: "in minutes", - time: now.Add(10 * time.Minute), - expected: "in 10 minutes", - }, - { - name: "hours ago", - time: now.Add(-10 * time.Hour), - expected: "10 hours ago", - }, - { - name: "in hours", - time: now.Add(10 * time.Hour), - expected: "in 10 hours", - }, - { - name: "different day", - time: now.Add(-25 * time.Hour), - expectedLength: len("01-01 15:04:05"), - }, - { - name: "same year but different month", - time: now.Add(-30 * 24 * time.Hour), - expectedLength: len("01-01 15:04:05"), - }, - { - name: "different year", - time: time.Date(now.Year()-1, 1, 1, 10, 20, 30, 0, now.Location()), - expected: time.Date(now.Year()-1, 1, 1, 10, 20, 30, 0, now.Location()).Format("2006-01-02 15:04:05"), - }, - { - name: "zero time", - time: time.Time{}, - expected: "never", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := FormatTimeWithReference(tt.time, now) - - if tt.expectedLength > 0 { - require.Len(t, result, tt.expectedLength) - } else { - require.Equal(t, tt.expected, result) - } - }) - } -} - -func TestFormatDuration(t *testing.T) { - tests := []struct { - name string - duration time.Duration - expected string - }{ - { - name: "zero duration", - duration: 0, - expected: "0 Seconds", - }, - { - name: "seconds only", - duration: 45 * time.Second, - expected: "45 seconds", - }, - { - name: "one second", - duration: 1 * time.Second, - expected: "1 second", - }, - { - name: "minutes only", - duration: 5 * time.Minute, - expected: "5 minutes", - }, - { - name: "one minute", - duration: 1 * time.Minute, - expected: "1 minute", - }, - { - name: "hours only", - duration: 3 * time.Hour, - expected: "3 hours", - }, - { - name: "one hour", - duration: 1 * time.Hour, - expected: "1 hour", - }, - { - name: "days only", - duration: 2 * 24 * time.Hour, - expected: "2 days", - }, - { - name: "one day", - duration: 24 * time.Hour, - expected: "1 day", - }, - { - name: "complex duration", - duration: 2*24*time.Hour + 3*time.Hour + 45*time.Minute + 15*time.Second, - expected: "2 days, 3 hours and 45 minutes", - }, - { - name: "hours and minutes", - duration: 2*time.Hour + 30*time.Minute, - expected: "2 hours and 30 minutes", - }, - { - name: "days and hours", - duration: 1*24*time.Hour + 12*time.Hour, - expected: "1 day and 12 hours", - }, - { - name: "days and hours and minutes", - duration: 1*24*time.Hour + 12*time.Hour + 30*time.Minute, - expected: "1 day, 12 hours and 30 minutes", - }, - { - name: "days and hours and minutes and seconds (ignore seconds)", - duration: 1*24*time.Hour + 12*time.Hour + 30*time.Minute + 15*time.Second, - expected: "1 day, 12 hours and 30 minutes", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := FormatDuration(tt.duration) - require.Equal(t, tt.expected, result) - }) - } -} - -func TestFormatLastSeen(t *testing.T) { - now := time.Now() - - tests := []struct { - name string - time time.Time - expected string - }{ - { - name: "zero time", - time: time.Time{}, - expected: "never", - }, - { - name: "non-zero time", - time: now.Add(-10 * time.Minute), - // The actual result will be handled by FormatTime, which is tested separately - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := FormatLastSeen(tt.time) - - if tt.name == "zero time" { - require.Equal(t, tt.expected, result) - } else if result == "never" { // Just make sure it's not "never", the actual formatting is tested in TestFormatTime - t.Errorf("Expected non-zero time to not return 'never', got %s", result) - } - }) - } -} - -func TestFormatByteSize(t *testing.T) { - tests := []struct { - name string - size int64 - expected string - }{ - { - name: "zero size", - size: 0, - expected: "0 B", - }, - { - name: "one byte", - size: 1, - expected: "1 B", - }, - { - name: "bytes (less than 1 KiB)", - size: 1023, - expected: "1023 B", - }, - { - name: "1 KiB", - size: 1024, - expected: "1 KiB", - }, - { - name: "KiB (less than 1 MiB)", - size: 1024 * 1023, - expected: "1023 KiB", - }, - { - name: "1 MiB", - size: 1024 * 1024, - expected: "1 MiB", - }, - { - name: "MiB (less than 1 GiB)", - size: 1024 * 1024 * 1023, - expected: "1023 MiB", - }, - { - name: "1 GiB", - size: 1024 * 1024 * 1024, - expected: "1 GiB", - }, - { - name: "GiB (less than 1 TiB)", - size: 1024 * 1024 * 1024 * 1023, - expected: "1023 GiB", - }, - { - name: "1 TiB", - size: 1024 * 1024 * 1024 * 1024, - expected: "1 TiB", - }, - { - name: "TiB (less than 1 PiB)", - size: 1024 * 1024 * 1024 * 1024 * 1023, - expected: "1023 TiB", - }, - { - name: "1 PiB", - size: 1024 * 1024 * 1024 * 1024 * 1024, - expected: "1 PiB", - }, - { - name: "PiB (large number)", - size: 1024 * 1024 * 1024 * 1024 * 1024 * 1023, - expected: "1023 PiB", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := FormatByteSize(tt.size) - require.Equal(t, tt.expected, result) - }) - } -} diff --git a/internal/utils/strutils/parser.go b/internal/utils/strutils/parser.go deleted file mode 100644 index fbf72ced..00000000 --- a/internal/utils/strutils/parser.go +++ /dev/null @@ -1,26 +0,0 @@ -package strutils - -import ( - "reflect" -) - -type Parser interface { - Parse(value string) error -} - -func Parse[T Parser](from string) (t T, err error) { - tt := reflect.TypeOf(t) - if tt.Kind() == reflect.Ptr { - t = reflect.New(tt.Elem()).Interface().(T) - } - err = t.Parse(from) - return t, err -} - -func MustParse[T Parser](from string) T { - t, err := Parse[T](from) - if err != nil { - panic("must failed: " + err.Error()) - } - return t -} diff --git a/internal/utils/strutils/split_join.go b/internal/utils/strutils/split_join.go deleted file mode 100644 index 37b2d5c5..00000000 --- a/internal/utils/strutils/split_join.go +++ /dev/null @@ -1,81 +0,0 @@ -package strutils - -import ( - "math" - "strings" -) - -// SplitRune is like strings.Split but takes a rune as separator. -func SplitRune(s string, sep rune) []string { - if sep == 0 { - return strings.Split(s, "") - } - n := strings.Count(s, string(sep)) + 1 - if n > len(s)+1 { - n = len(s) + 1 - } - a := make([]string, n) - n-- - i := 0 - for i < n { - m := strings.IndexRune(s, sep) - if m < 0 { - break - } - a[i] = s[:m] - s = s[m+1:] - i++ - } - a[i] = s - return a[:i+1] -} - -// SplitComma is a wrapper around SplitRune(s, ','). -func SplitComma(s string) []string { - return SplitRune(s, ',') -} - -// SplitLine is a wrapper around SplitRune(s, '\n'). -func SplitLine(s string) []string { - return SplitRune(s, '\n') -} - -// SplitSpace is a wrapper around SplitRune(s, ' '). -func SplitSpace(s string) []string { - return SplitRune(s, ' ') -} - -// JoinRune is like strings.Join but takes a rune as separator. -func JoinRune(elems []string, sep rune) string { - switch len(elems) { - case 0: - return "" - case 1: - return elems[0] - } - if sep == 0 { - return strings.Join(elems, "") - } - - var n int - for _, elem := range elems { - if len(elem) > math.MaxInt-n { - panic("strings: Join output length overflow") - } - n += len(elem) - } - - var b strings.Builder - b.Grow(n) - b.WriteString(elems[0]) - for _, s := range elems[1:] { - b.WriteRune(sep) - b.WriteString(s) - } - return b.String() -} - -// JoinLines is a wrapper around JoinRune(elems, '\n'). -func JoinLines(elems []string) string { - return JoinRune(elems, '\n') -} diff --git a/internal/utils/strutils/split_join_test.go b/internal/utils/strutils/split_join_test.go deleted file mode 100644 index ef52f61d..00000000 --- a/internal/utils/strutils/split_join_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package strutils_test - -import ( - "strings" - "testing" - - "github.com/stretchr/testify/require" - . "github.com/yusing/godoxy/internal/utils/strutils" -) - -var alphaNumeric = func() string { - var s strings.Builder - for i := range 'z' - 'a' + 1 { - s.WriteRune('a' + i) - s.WriteRune('A' + i) - s.WriteRune(',') - } - for i := range '9' - '0' + 1 { - s.WriteRune('0' + i) - s.WriteRune(',') - } - return s.String() -}() - -func TestSplit(t *testing.T) { - tests := map[string]rune{ - "": 0, - "1": '1', - ",": ',', - } - for sep, rsep := range tests { - t.Run(sep, func(t *testing.T) { - expected := strings.Split(alphaNumeric, sep) - require.Equal(t, expected, SplitRune(alphaNumeric, rsep)) - require.Equal(t, alphaNumeric, JoinRune(expected, rsep)) - }) - } -} - -func BenchmarkSplitRune(b *testing.B) { - for range b.N { - SplitRune(alphaNumeric, ',') - } -} - -func BenchmarkSplitRuneStdlib(b *testing.B) { - for range b.N { - strings.Split(alphaNumeric, ",") - } -} - -func BenchmarkJoinRune(b *testing.B) { - for range b.N { - JoinRune(SplitRune(alphaNumeric, ','), ',') - } -} - -func BenchmarkJoinRuneStdlib(b *testing.B) { - for range b.N { - strings.Join(SplitRune(alphaNumeric, ','), ",") - } -} diff --git a/internal/utils/strutils/strconv.go b/internal/utils/strutils/strconv.go deleted file mode 100644 index 95e539eb..00000000 --- a/internal/utils/strutils/strconv.go +++ /dev/null @@ -1,7 +0,0 @@ -package strutils - -import ( - "strconv" -) - -var Atoi = strconv.Atoi diff --git a/internal/utils/strutils/string.go b/internal/utils/strutils/string.go deleted file mode 100644 index 8fef6a18..00000000 --- a/internal/utils/strutils/string.go +++ /dev/null @@ -1,98 +0,0 @@ -package strutils - -import ( - "strings" - - "golang.org/x/text/cases" - "golang.org/x/text/language" -) - -// CommaSeperatedList returns a list of strings split by commas, -// then trim spaces from each element. -func CommaSeperatedList(s string) []string { - if s == "" { - return []string{} - } - res := SplitComma(s) - for i, part := range res { - res[i] = strings.TrimSpace(part) - } - return res -} - -var caseTitle = cases.Title(language.AmericanEnglish) - -func Title(s string) string { - return caseTitle.String(s) -} - -func ContainsFold(s, substr string) bool { - return IndexFold(s, substr) >= 0 -} - -func IndexFold(s, substr string) int { - return strings.Index(strings.ToLower(s), strings.ToLower(substr)) -} - -func ToLowerNoSnake(s string) string { - var buf strings.Builder - for _, r := range s { - if r == '_' { - continue - } - if r >= 'A' && r <= 'Z' { - r += 'a' - 'A' - } - buf.WriteRune(r) - } - return buf.String() -} - -//nolint:intrange -func LevenshteinDistance(a, b string) int { - if a == b { - return 0 - } - if len(a) == 0 { - return len(b) - } - if len(b) == 0 { - return len(a) - } - - v0 := make([]int, len(b)+1) - v1 := make([]int, len(b)+1) - - for i := 0; i <= len(b); i++ { - v0[i] = i - } - - for i := 0; i < len(a); i++ { - v1[0] = i + 1 - - for j := 0; j < len(b); j++ { - cost := 0 - if a[i] != b[j] { - cost = 1 - } - - v1[j+1] = min3(v1[j]+1, v0[j+1]+1, v0[j]+cost) - } - - for j := 0; j <= len(b); j++ { - v0[j] = v1[j] - } - } - - return v1[len(b)] -} - -func min3(a, b, c int) int { - if a < b && a < c { - return a - } - if b < a && b < c { - return b - } - return c -} diff --git a/internal/utils/strutils/url.go b/internal/utils/strutils/url.go deleted file mode 100644 index b587c245..00000000 --- a/internal/utils/strutils/url.go +++ /dev/null @@ -1,26 +0,0 @@ -package strutils - -import ( - "path" - "strings" -) - -// SanitizeURI sanitizes a URI reference to ensure it is safe -// It disallows URLs beginning with // or /\ as absolute URLs, -// cleans the URL path to remove any .. or . path elements, -// and ensures the URL starts with a / if it doesn't already -func SanitizeURI(uri string) string { - if uri == "" { - return "/" - } - if strings.HasPrefix(uri, "http://") || strings.HasPrefix(uri, "https://") { - return uri - } - if uri[0] != '/' { - uri = "/" + uri - } - if len(uri) > 1 && uri[0] == '/' && uri[1] != '/' && uri[1] != '\\' { - return path.Clean(uri) - } - return "/" -} diff --git a/internal/utils/strutils/url_test.go b/internal/utils/strutils/url_test.go deleted file mode 100644 index 8143991a..00000000 --- a/internal/utils/strutils/url_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package strutils - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestSanitizeURI(t *testing.T) { - tests := []struct { - name string - input string - expected string - }{ - { - name: "empty string", - input: "", - expected: "/", - }, - { - name: "single slash", - input: "/", - expected: "/", - }, - { - name: "normal path", - input: "/path/to/resource", - expected: "/path/to/resource", - }, - { - name: "path without leading slash", - input: "path/to/resource", - expected: "/path/to/resource", - }, - { - name: "path with dot segments", - input: "/path/./to/../resource", - expected: "/path/resource", - }, - { - name: "double slash prefix", - input: "//path/to/resource", - expected: "/", - }, - { - name: "backslash prefix", - input: "/\\path/to/resource", - expected: "/", - }, - { - name: "path with multiple slashes", - input: "/path//to///resource", - expected: "/path/to/resource", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := SanitizeURI(tt.input) - require.Equal(t, tt.expected, result) - }) - } -} diff --git a/internal/utils/synk/pool.go b/internal/utils/synk/pool.go deleted file mode 100644 index 83335443..00000000 --- a/internal/utils/synk/pool.go +++ /dev/null @@ -1,238 +0,0 @@ -package synk - -import ( - "sync/atomic" - "unsafe" - "weak" -) - -type weakBuf = weak.Pointer[[]byte] - -func makeWeak(b *[]byte) weakBuf { - return weak.Make(b) -} - -func getBufFromWeak(w weakBuf) []byte { - ptr := w.Value() - if ptr != nil { - return *ptr - } - return nil -} - -type BytesPool struct { - sizedPool chan weakBuf - unsizedPool chan weakBuf - initSize int -} - -type BytesPoolWithMemory struct { - maxAllocatedSize atomic.Uint32 - numShouldShrink atomic.Int32 - pool chan weakBuf -} - -type sliceInternal struct { - ptr unsafe.Pointer - len int - cap int -} - -func sliceStruct(b *[]byte) *sliceInternal { - return (*sliceInternal)(unsafe.Pointer(b)) -} - -func underlyingPtr(b []byte) unsafe.Pointer { - return sliceStruct(&b).ptr -} - -func setCap(b *[]byte, cap int) { - sliceStruct(b).cap = cap -} - -func setLen(b *[]byte, len int) { - sliceStruct(b).len = len -} - -const ( - kb = 1024 - mb = 1024 * kb -) - -const ( - InPoolLimit = 32 * mb - - UnsizedAvg = 4 * kb - SizedPoolThreshold = 256 * kb - DropThreshold = 4 * mb - - SizedPoolSize = InPoolLimit * 8 / 10 / SizedPoolThreshold - UnsizedPoolSize = InPoolLimit * 2 / 10 / UnsizedAvg - - ShouldShrinkThreshold = 10 -) - -var bytesPool = &BytesPool{ - sizedPool: make(chan weakBuf, SizedPoolSize), - unsizedPool: make(chan weakBuf, UnsizedPoolSize), - initSize: UnsizedAvg, -} - -var bytesPoolWithMemory = make(chan weakBuf, UnsizedPoolSize) - -func GetBytesPool() *BytesPool { - return bytesPool -} - -func GetBytesPoolWithUniqueMemory() *BytesPoolWithMemory { - return &BytesPoolWithMemory{ - pool: bytesPoolWithMemory, - } -} - -func (p *BytesPool) Get() []byte { - for { - select { - case bWeak := <-p.unsizedPool: - bPtr := getBufFromWeak(bWeak) - if bPtr == nil { - continue - } - addReused(cap(bPtr)) - return bPtr - default: - addNonPooled(p.initSize) - return make([]byte, 0, p.initSize) - } - } -} - -func (p *BytesPoolWithMemory) Get() []byte { - for { - size := int(p.maxAllocatedSize.Load()) - select { - case bWeak := <-p.pool: - bPtr := getBufFromWeak(bWeak) - if bPtr == nil { - continue - } - addReused(cap(bPtr)) - return bPtr - default: - addNonPooled(size) - return make([]byte, 0, size) - } - } -} - -func (p *BytesPool) GetSized(size int) []byte { - for { - select { - case bWeak := <-p.sizedPool: - b := getBufFromWeak(bWeak) - if b == nil { - continue - } - capB := cap(b) - - remainingSize := capB - size - if remainingSize == 0 { - addReused(capB) - return b[:size] - } - - if remainingSize > 0 { // capB > size (buffer larger than requested) - addReused(size) - - p.Put(b[size:capB]) - - // return the first part and limit the capacity to the requested size - ret := b[:size] - setLen(&ret, size) - setCap(&ret, size) - return ret - } - - // size is not enough - select { - case p.sizedPool <- bWeak: - default: - addDropped(cap(b)) - // just drop it - } - default: - } - addNonPooled(size) - return make([]byte, size) - } -} - -func (p *BytesPool) Put(b []byte) { - size := cap(b) - if size > DropThreshold { - addDropped(size) - return - } - b = b[:0] - if size >= SizedPoolThreshold { - p.put(size, makeWeak(&b), p.sizedPool) - } else { - p.put(size, makeWeak(&b), p.unsizedPool) - } -} - -func (p *BytesPoolWithMemory) Put(b []byte) { - capB := uint32(cap(b)) - - for { - current := p.maxAllocatedSize.Load() - - if capB < current { - // Potential shrink case - if p.numShouldShrink.Add(1) > ShouldShrinkThreshold { - if p.maxAllocatedSize.CompareAndSwap(current, capB) { - p.numShouldShrink.Store(0) // reset counter - break - } - p.numShouldShrink.Add(-1) // undo if CAS failed - } - break - } else if capB > current { - // Growing case - if p.maxAllocatedSize.CompareAndSwap(current, capB) { - break - } - // retry if CAS failed - } else { - // equal case - no change needed - break - } - } - - if capB > DropThreshold { - addDropped(int(capB)) - return - } - b = b[:0] - w := makeWeak(&b) - select { - case p.pool <- w: - default: - addDropped(int(capB)) - // just drop it - } -} - -//go:inline -func (p *BytesPool) put(size int, w weakBuf, pool chan weakBuf) { - select { - case pool <- w: - default: - addDropped(size) - // just drop it - } -} - -func init() { - initPoolStats() -} diff --git a/internal/utils/synk/pool_bench_test.go b/internal/utils/synk/pool_bench_test.go deleted file mode 100644 index b43a9ac2..00000000 --- a/internal/utils/synk/pool_bench_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package synk - -import ( - "slices" - "testing" -) - -var sizes = []int{1024, 4096, 16384, 65536, 32 * 1024, 128 * 1024, 512 * 1024, 1024 * 1024, 2 * 1024 * 1024} - -func BenchmarkBytesPool_GetSmall(b *testing.B) { - for b.Loop() { - bytesPool.Put(bytesPool.GetSized(1024)) - } -} - -func BenchmarkBytesPool_MakeSmall(b *testing.B) { - for b.Loop() { - _ = make([]byte, 1024) - } -} - -func BenchmarkBytesPool_GetLarge(b *testing.B) { - for b.Loop() { - buf := bytesPool.GetSized(DropThreshold / 2) - buf[0] = 1 - bytesPool.Put(buf) - } -} - -func BenchmarkBytesPool_GetLargeUnsized(b *testing.B) { - for b.Loop() { - buf := slices.Grow(bytesPool.Get(), DropThreshold/2) - buf = append(buf, 1) - bytesPool.Put(buf) - } -} - -func BenchmarkBytesPool_MakeLarge(b *testing.B) { - for b.Loop() { - buf := make([]byte, DropThreshold/2) - buf[0] = 1 - _ = buf - } -} - -func BenchmarkBytesPool_GetAll(b *testing.B) { - for i := range b.N { - bytesPool.Put(bytesPool.GetSized(sizes[i%len(sizes)])) - } -} - -func BenchmarkBytesPool_GetAllUnsized(b *testing.B) { - for i := range b.N { - bytesPool.Put(slices.Grow(bytesPool.Get(), sizes[i%len(sizes)])) - } -} - -func BenchmarkBytesPool_MakeAll(b *testing.B) { - for i := range b.N { - _ = make([]byte, sizes[i%len(sizes)]) - } -} - -func BenchmarkBytesPoolWithMemory(b *testing.B) { - pool := GetBytesPoolWithUniqueMemory() - for i := range b.N { - pool.Put(slices.Grow(pool.Get(), sizes[i%len(sizes)])) - } -} diff --git a/internal/utils/synk/pool_debug.go b/internal/utils/synk/pool_debug.go deleted file mode 100644 index 36584ccd..00000000 --- a/internal/utils/synk/pool_debug.go +++ /dev/null @@ -1,64 +0,0 @@ -//go:build !production - -package synk - -import ( - "os" - "os/signal" - "sync/atomic" - "time" - - "github.com/rs/zerolog/log" - "github.com/yusing/godoxy/internal/utils/strutils" -) - -type poolCounters struct { - num atomic.Uint64 - size atomic.Uint64 -} - -var ( - nonPooled poolCounters - dropped poolCounters - reused poolCounters -) - -func addNonPooled(size int) { - nonPooled.num.Add(1) - nonPooled.size.Add(uint64(size)) -} - -func addReused(size int) { - reused.num.Add(1) - reused.size.Add(uint64(size)) -} -func addDropped(size int) { - dropped.num.Add(1) - dropped.size.Add(uint64(size)) -} - -func initPoolStats() { - go func() { - statsTicker := time.NewTicker(5 * time.Second) - defer statsTicker.Stop() - - sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt) - - for { - select { - case <-sig: - return - case <-statsTicker.C: - log.Info(). - Uint64("numReused", reused.num.Load()). - Str("sizeReused", strutils.FormatByteSize(reused.size.Load())). - Uint64("numDropped", dropped.num.Load()). - Str("sizeDropped", strutils.FormatByteSize(dropped.size.Load())). - Uint64("numNonPooled", nonPooled.num.Load()). - Str("sizeNonPooled", strutils.FormatByteSize(nonPooled.size.Load())). - Msg("bytes pool stats") - } - } - }() -} diff --git a/internal/utils/synk/pool_prod.go b/internal/utils/synk/pool_prod.go deleted file mode 100644 index 88bc0aa6..00000000 --- a/internal/utils/synk/pool_prod.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:build production - -package synk - -func addNonPooled(size int) {} -func addDropped(size int) {} -func addReused(size int) {} -func initPoolStats() {} diff --git a/internal/utils/synk/pool_test.go b/internal/utils/synk/pool_test.go deleted file mode 100644 index 76682506..00000000 --- a/internal/utils/synk/pool_test.go +++ /dev/null @@ -1,263 +0,0 @@ -package synk - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestSized(t *testing.T) { - b := bytesPool.GetSized(2 * SizedPoolThreshold) - assert.Equal(t, cap(b), 2*SizedPoolThreshold) - bytesPool.Put(b) - assert.Equal(t, underlyingPtr(b), underlyingPtr(bytesPool.GetSized(SizedPoolThreshold))) -} - -func TestUnsized(t *testing.T) { - b := bytesPool.Get() - assert.Equal(t, cap(b), UnsizedAvg) - bytesPool.Put(b) - assert.Equal(t, underlyingPtr(b), underlyingPtr(bytesPool.Get())) -} - -func TestGetSizedExactMatch(t *testing.T) { - // Test exact size match reuse - size := SizedPoolThreshold - b1 := bytesPool.GetSized(size) - assert.Equal(t, size, len(b1)) - assert.Equal(t, size, cap(b1)) - - // Put back into pool - bytesPool.Put(b1) - - // Get same size - should reuse the same buffer - b2 := bytesPool.GetSized(size) - assert.Equal(t, size, len(b2)) - assert.Equal(t, size, cap(b2)) - assert.Equal(t, underlyingPtr(b1), underlyingPtr(b2)) -} - -func TestGetSizedBufferSplit(t *testing.T) { - // Test buffer splitting when capacity > requested size - largeSize := 2 * SizedPoolThreshold - requestedSize := SizedPoolThreshold - - // Create a large buffer and put it in pool - b1 := bytesPool.GetSized(largeSize) - assert.Equal(t, largeSize, len(b1)) - assert.Equal(t, largeSize, cap(b1)) - - bytesPool.Put(b1) - - // Request smaller size - should split the buffer - b2 := bytesPool.GetSized(requestedSize) - assert.Equal(t, requestedSize, len(b2)) - assert.Equal(t, requestedSize, cap(b2)) // capacity should remain the original - assert.Equal(t, underlyingPtr(b1), underlyingPtr(b2)) - - // The remaining part should be put back in pool - // Request the remaining size to verify - remainingSize := largeSize - requestedSize - b3 := bytesPool.GetSized(remainingSize) - assert.Equal(t, remainingSize, len(b3)) - assert.Equal(t, remainingSize, cap(b3)) - - // Verify the remaining buffer points to the correct memory location - originalPtr := underlyingPtr(b1) - remainingPtr := underlyingPtr(b3) - - // The remaining buffer should start at original + requestedSize - expectedOffset := uintptr(originalPtr) + uintptr(requestedSize) - actualOffset := uintptr(remainingPtr) - assert.Equal(t, expectedOffset, actualOffset, "Remaining buffer should point to correct offset") -} - -func TestGetSizedSmallRemainder(t *testing.T) { - // Test when remaining size is smaller than SizedPoolThreshold - poolSize := SizedPoolThreshold + 100 // Just slightly larger than threshold - requestedSize := SizedPoolThreshold - - // Create buffer and put in pool - b1 := bytesPool.GetSized(poolSize) - bytesPool.Put(b1) - - // Request size that leaves small remainder - b2 := bytesPool.GetSized(requestedSize) - assert.Equal(t, requestedSize, len(b2)) - assert.Equal(t, requestedSize, cap(b2)) - - // The small remainder (100 bytes) should NOT be put back in sized pool - // Try to get the remainder size - should create new buffer - b3 := bytesPool.GetSized(100) - assert.Equal(t, 100, len(b3)) - assert.Equal(t, 100, cap(b3)) - assert.NotEqual(t, underlyingPtr(b2), underlyingPtr(b3)) -} - -func TestGetSizedSmallBufferBypass(t *testing.T) { - // Test that small buffers (< SizedPoolThreshold) don't use sized pool - smallSize := SizedPoolThreshold - 1 - - b1 := bytesPool.GetSized(smallSize) - assert.Equal(t, smallSize, len(b1)) - assert.Equal(t, smallSize, cap(b1)) - - b2 := bytesPool.GetSized(smallSize) - assert.Equal(t, smallSize, len(b2)) - assert.Equal(t, smallSize, cap(b2)) - - // Should be different buffers (not pooled) - assert.NotEqual(t, underlyingPtr(b1), underlyingPtr(b2)) -} - -func TestGetSizedBufferTooSmall(t *testing.T) { - // Test when pool buffer is smaller than requested size - smallSize := SizedPoolThreshold - largeSize := 2 * SizedPoolThreshold - - // Put small buffer in pool - b1 := bytesPool.GetSized(smallSize) - bytesPool.Put(b1) - - // Request larger size - should create new buffer, not reuse small one - b2 := bytesPool.GetSized(largeSize) - assert.Equal(t, largeSize, len(b2)) - assert.Equal(t, largeSize, cap(b2)) - assert.NotEqual(t, underlyingPtr(b1), underlyingPtr(b2)) - - // The small buffer should still be in pool - b3 := bytesPool.GetSized(smallSize) - assert.Equal(t, underlyingPtr(b1), underlyingPtr(b3)) -} - -func TestGetSizedMultipleSplits(t *testing.T) { - // Test multiple sequential splits of the same buffer - hugeSize := 4 * SizedPoolThreshold - splitSize := SizedPoolThreshold - - // Create huge buffer - b1 := bytesPool.GetSized(hugeSize) - originalPtr := underlyingPtr(b1) - bytesPool.Put(b1) - - // Split it into smaller pieces - pieces := make([][]byte, 0, 4) - for i := range 4 { - piece := bytesPool.GetSized(splitSize) - pieces = append(pieces, piece) - - // Each piece should point to the correct offset - expectedOffset := uintptr(originalPtr) + uintptr(i*splitSize) - actualOffset := uintptr(underlyingPtr(piece)) - assert.Equal(t, expectedOffset, actualOffset, "Piece %d should point to correct offset", i) - assert.Equal(t, splitSize, len(piece)) - assert.Equal(t, splitSize, cap(piece)) - } - - // All pieces should have the same underlying capacity - for i, piece := range pieces { - assert.Equal(t, splitSize, cap(piece), "Piece %d should have correct capacity", i) - } -} - -func TestGetSizedMemorySafety(t *testing.T) { - // Test that split buffers don't interfere with each other - totalSize := 3 * SizedPoolThreshold - firstSize := SizedPoolThreshold - - // Create buffer and split it - b1 := bytesPool.GetSized(totalSize) - // Fill with test data - for i := range len(b1) { - b1[i] = byte(i % 256) - } - - bytesPool.Put(b1) - - // Get first part - first := bytesPool.GetSized(firstSize) - assert.Equal(t, firstSize, len(first)) - - // Verify data integrity - for i := range len(first) { - assert.Equal(t, byte(i%256), first[i], "Data should be preserved after split") - } - - // Get remaining part - remainingSize := totalSize - firstSize - remaining := bytesPool.GetSized(remainingSize) - assert.Equal(t, remainingSize, len(remaining)) - - // Verify remaining data - for i := range len(remaining) { - expected := byte((i + firstSize) % 256) - assert.Equal(t, expected, remaining[i], "Remaining data should be preserved") - } -} - -func TestGetSizedCapacityLimiting(t *testing.T) { - // Test that returned buffers have limited capacity to prevent overwrites - largeSize := 2 * SizedPoolThreshold - requestedSize := SizedPoolThreshold - - // Create large buffer and put in pool - b1 := bytesPool.GetSized(largeSize) - bytesPool.Put(b1) - - // Get smaller buffer from the split - b2 := bytesPool.GetSized(requestedSize) - assert.Equal(t, requestedSize, len(b2)) - assert.Equal(t, requestedSize, cap(b2), "Returned buffer should have limited capacity") - - // Try to append data - should not be able to overwrite beyond capacity - original := make([]byte, len(b2)) - copy(original, b2) - - // This append should force a new allocation since capacity is limited - b2 = append(b2, 1, 2, 3, 4, 5) - assert.Greater(t, len(b2), requestedSize, "Buffer should have grown") - - // Get the remaining buffer to verify it wasn't affected - remainingSize := largeSize - requestedSize - b3 := bytesPool.GetSized(remainingSize) - assert.Equal(t, remainingSize, len(b3)) - assert.Equal(t, remainingSize, cap(b3), "Remaining buffer should have limited capacity") -} - -func TestGetSizedAppendSafety(t *testing.T) { - // Test that appending to returned buffer doesn't affect remaining buffer - totalSize := 4 * SizedPoolThreshold - firstSize := SizedPoolThreshold - - // Create buffer with specific pattern - b1 := bytesPool.GetSized(totalSize) - for i := range len(b1) { - b1[i] = byte(100 + i%100) - } - bytesPool.Put(b1) - - // Get first part - first := bytesPool.GetSized(firstSize) - assert.Equal(t, firstSize, cap(first), "First part should have limited capacity") - - // Store original first part content - originalFirst := make([]byte, len(first)) - copy(originalFirst, first) - - // Get remaining part to establish its state - remaining := bytesPool.GetSized(SizedPoolThreshold) - - // Store original remaining content - originalRemaining := make([]byte, len(remaining)) - copy(originalRemaining, remaining) - - // Now try to append to first - this should not affect remaining buffers - // since capacity is limited - first = append(first, make([]byte, 1000)...) - - // Verify remaining buffer content is unchanged - for i := range len(originalRemaining) { - assert.Equal(t, originalRemaining[i], remaining[i], - "Remaining buffer should be unaffected by append to first buffer") - } -} diff --git a/internal/utils/trie/any.go b/internal/utils/trie/any.go deleted file mode 100644 index e98ec5e7..00000000 --- a/internal/utils/trie/any.go +++ /dev/null @@ -1,49 +0,0 @@ -package trie - -import ( - "sync/atomic" -) - -// AnyValue is a wrapper of atomic.Value -// It is used to store values in trie nodes -// And allowed to assign to empty struct value when node -// is not an end node anymore -type AnyValue struct { - v atomic.Value -} - -type zeroValue struct{} - -var zero zeroValue - -func (av *AnyValue) Store(v any) { - if v == nil { - av.v.Store(zero) - return - } - defer panicInvalidAssignment() - av.v.Store(v) -} - -func (av *AnyValue) Swap(v any) any { - defer panicInvalidAssignment() - return av.v.Swap(v) -} - -func (av *AnyValue) Load() any { - switch v := av.v.Load().(type) { - case zeroValue: - return nil - default: - return v - } -} - -func (av *AnyValue) IsNil() bool { - switch v := av.v.Load().(type) { - case zeroValue: - return true // assigned nil manually - default: - return v == nil // uninitialized - } -} diff --git a/internal/utils/trie/any_debug.go b/internal/utils/trie/any_debug.go deleted file mode 100644 index 5010ace0..00000000 --- a/internal/utils/trie/any_debug.go +++ /dev/null @@ -1,13 +0,0 @@ -//go:build debug - -package trie - -import "fmt" - -func panicInvalidAssignment() { - // assigned anything after manually assigning nil - // will panic because of type mismatch (zeroValue and v.(type)) - if r := recover(); r != nil { - panic(fmt.Errorf("attempt to assign non-nil value on edge node or assigning mismatched type: %v", r)) - } -} diff --git a/internal/utils/trie/any_prod.go b/internal/utils/trie/any_prod.go deleted file mode 100644 index 382cb040..00000000 --- a/internal/utils/trie/any_prod.go +++ /dev/null @@ -1,7 +0,0 @@ -//go:build !debug - -package trie - -func panicInvalidAssignment() { - // no-op -} diff --git a/internal/utils/trie/any_test.go b/internal/utils/trie/any_test.go deleted file mode 100644 index 3855993a..00000000 --- a/internal/utils/trie/any_test.go +++ /dev/null @@ -1,16 +0,0 @@ -package trie - -import ( - "testing" -) - -func TestStoreNil(t *testing.T) { - var v AnyValue - v.Store(nil) - if v.Load() != nil { - t.Fatal("expected nil") - } - if !v.IsNil() { - t.Fatal("expected true") - } -} diff --git a/internal/utils/trie/json.go b/internal/utils/trie/json.go deleted file mode 100644 index bc0895ee..00000000 --- a/internal/utils/trie/json.go +++ /dev/null @@ -1,21 +0,0 @@ -package trie - -import ( - "encoding/json" - "maps" -) - -func (r *Root) MarshalJSON() ([]byte, error) { - return json.Marshal(maps.Collect(r.Walk)) -} - -func (r *Root) UnmarshalJSON(data []byte) error { - var m map[string]any - if err := json.Unmarshal(data, &m); err != nil { - return err - } - for k, v := range m { - r.Store(NewKey(k), v) - } - return nil -} diff --git a/internal/utils/trie/json_test.go b/internal/utils/trie/json_test.go deleted file mode 100644 index 5e6adafb..00000000 --- a/internal/utils/trie/json_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package trie - -import ( - "encoding/json" - "testing" -) - -func TestMarshalUnmarshalJSON(t *testing.T) { - trie := NewTrie() - data := map[string]any{ - "foo.bar": 42.12, - "foo.baz": "hello", - "qwe.rt.yu.io": 123.45, - } - for k, v := range data { - trie.Store(NewKey(k), v) - } - - // MarshalJSON - bytesFromTrie, err := json.Marshal(trie) - if err != nil { - t.Fatalf("json.Marshal error: %v", err) - } - - // UnmarshalJSON - newTrie := NewTrie() - if err := json.Unmarshal(bytesFromTrie, newTrie); err != nil { - t.Fatalf("UnmarshalJSON error: %v", err) - } - for k, v := range data { - got, ok := newTrie.Get(NewKey(k)) - if !ok || got != v { - t.Errorf("UnmarshalJSON: key %q got %v, want %v", k, got, v) - } - } -} diff --git a/internal/utils/trie/key.go b/internal/utils/trie/key.go deleted file mode 100644 index e1c53162..00000000 --- a/internal/utils/trie/key.go +++ /dev/null @@ -1,80 +0,0 @@ -package trie - -import ( - "slices" - "strings" - - "github.com/yusing/godoxy/internal/utils/strutils" -) - -type Key struct { - segments []string // escaped segments - full string // unescaped original key - hasWildcard bool -} - -func Namespace(ns string) *Key { - return &Key{ - segments: []string{ns}, - full: ns, - hasWildcard: false, - } -} - -func NewKey(keyStr string) *Key { - key := &Key{ - segments: strutils.SplitRune(keyStr, '.'), - full: keyStr, - } - for _, seg := range key.segments { - if seg == "*" || seg == "**" { - key.hasWildcard = true - } - } - return key -} - -func EscapeSegment(seg string) string { - var sb strings.Builder - for _, r := range seg { - switch r { - case '.', '*': - sb.WriteString("__") - default: - sb.WriteRune(r) - } - } - return sb.String() -} - -func (ns Key) With(segment string) *Key { - ns.segments = append(ns.segments, segment) - ns.full = ns.full + "." + segment - ns.hasWildcard = ns.hasWildcard || segment == "*" || segment == "**" - return &ns -} - -func (ns Key) WithEscaped(segment string) *Key { - ns.segments = append(ns.segments, EscapeSegment(segment)) - ns.full = ns.full + "." + segment - return &ns -} - -func (ns *Key) NumSegments() int { - return len(ns.segments) -} - -func (ns *Key) HasWildcard() bool { - return ns.hasWildcard -} - -func (ns *Key) String() string { - return ns.full -} - -func (ns *Key) Clone() *Key { - clone := *ns - clone.segments = slices.Clone(ns.segments) - clone.full = strings.Clone(ns.full) - return &clone -} diff --git a/internal/utils/trie/key_test.go b/internal/utils/trie/key_test.go deleted file mode 100644 index 9079a6b2..00000000 --- a/internal/utils/trie/key_test.go +++ /dev/null @@ -1,86 +0,0 @@ -package trie - -import ( - "reflect" - "testing" -) - -func TestNamespace(t *testing.T) { - k := Namespace("foo") - if k.String() != "foo" { - t.Errorf("Namespace.String() = %q, want %q", k.String(), "foo") - } - if k.NumSegments() != 1 { - t.Errorf("Namespace.NumSegments() = %d, want 1", k.NumSegments()) - } - if k.HasWildcard() { - t.Error("Namespace.HasWildcard() = true, want false") - } -} - -func TestNewKey(t *testing.T) { - k := NewKey("a.b.c") - if !reflect.DeepEqual(k.segments, []string{"a", "b", "c"}) { - t.Errorf("NewKey.segments = %v, want [a b c]", k.segments) - } - if k.String() != "a.b.c" { - t.Errorf("NewKey.String() = %q, want %q", k.String(), "a.b.c") - } - if k.NumSegments() != 3 { - t.Errorf("NewKey.NumSegments() = %d, want 3", k.NumSegments()) - } - if k.HasWildcard() { - t.Error("NewKey.HasWildcard() = true, want false") - } - - kw := NewKey("foo.*.bar") - if !kw.HasWildcard() { - t.Error("NewKey.HasWildcard() = false, want true for wildcard") - } -} - -func TestWithAndWithEscaped(t *testing.T) { - k := Namespace("foo") - k2 := k.Clone().With("bar") - if k2.String() != "foo.bar" { - t.Errorf("With.String() = %q, want %q", k2.String(), "foo.bar") - } - if k2.NumSegments() != 2 { - t.Errorf("With.NumSegments() = %d, want 2", k2.NumSegments()) - } - - k3 := Namespace("foo").WithEscaped("b.r*") - esc := EscapeSegment("b.r*") - if k3.segments[1] != esc { - t.Errorf("WithEscaped.segment = %q, want %q", k3.segments[1], esc) - } -} - -func TestEscapeSegment(t *testing.T) { - cases := map[string]string{ - "foo": "foo", - "f.o": "f__o", - "*": "__", - "a*b.c": "a__b__c", - } - for in, want := range cases { - if got := EscapeSegment(in); got != want { - t.Errorf("EscapeSegment(%q) = %q, want %q", in, got, want) - } - } -} - -func TestClone(t *testing.T) { - k := NewKey("x.y.z") - cl := k.Clone() - if !reflect.DeepEqual(k, cl) { - t.Errorf("Clone() = %v, want %v", cl, k) - } - cl = cl.With("new") - if cl == k { - t.Error("Clone() returns same pointer") - } - if reflect.DeepEqual(k.segments, cl.segments) { - t.Error("Clone is not deep copy: segments slice is shared") - } -} diff --git a/internal/utils/trie/node.go b/internal/utils/trie/node.go deleted file mode 100644 index 254a1bfb..00000000 --- a/internal/utils/trie/node.go +++ /dev/null @@ -1,54 +0,0 @@ -package trie - -import ( - "github.com/puzpuzpuz/xsync/v4" -) - -type Node struct { - key string - children *xsync.Map[string, *Node] // lock-free map which allows concurrent access - value AnyValue // only end nodes have values -} - -func mayPrefix(key, part string) string { - if key == "" { - return part - } - return key + "." + part -} - -func (node *Node) newChild(part string) *Node { - return &Node{ - key: mayPrefix(node.key, part), - children: xsync.NewMap[string, *Node](), - } -} - -func (node *Node) Get(key *Key) (any, bool) { - for _, seg := range key.segments { - child, ok := node.children.Load(seg) - if !ok { - return nil, false - } - node = child - } - v := node.value.Load() - if v == nil { - return nil, false - } - return v, true -} - -func (node *Node) loadOrStore(key *Key, newFunc func() any) (*Node, bool) { - for i, seg := range key.segments { - child, _ := node.children.LoadOrCompute(seg, func() (*Node, bool) { - newNode := node.newChild(seg) - if i == len(key.segments)-1 { - newNode.value.Store(newFunc()) - } - return newNode, false - }) - node = child - } - return node, false -} diff --git a/internal/utils/trie/trie.go b/internal/utils/trie/trie.go deleted file mode 100644 index c031b564..00000000 --- a/internal/utils/trie/trie.go +++ /dev/null @@ -1,44 +0,0 @@ -package trie - -import "github.com/puzpuzpuz/xsync/v4" - -type Root struct { - *Node - cached *xsync.Map[string, *Node] -} - -func NewTrie() *Root { - return &Root{ - Node: &Node{ - children: xsync.NewMap[string, *Node](), - }, - cached: xsync.NewMap[string, *Node](), - } -} - -func (r *Root) getNode(key *Key, newFunc func() any) *Node { - if key.hasWildcard { - panic("should not call Load or Store on a key with any wildcard: " + key.full) - } - node, _ := r.cached.LoadOrCompute(key.full, func() (*Node, bool) { - return r.Node.loadOrStore(key, newFunc) - }) - return node -} - -// LoadOrStore loads or stores the value for the key -// Returns the value loaded/stored -func (r *Root) LoadOrStore(key *Key, newFunc func() any) any { - return r.getNode(key, newFunc).value.Load() -} - -// LoadAndStore loads or stores the value for the key -// Returns the old value if exists, nil otherwise -func (r *Root) LoadAndStore(key *Key, val any) any { - return r.getNode(key, func() any { return val }).value.Swap(val) -} - -// Store stores the value for the key -func (r *Root) Store(key *Key, val any) { - r.getNode(key, func() any { return val }).value.Store(val) -} diff --git a/internal/utils/trie/trie_test.go b/internal/utils/trie/trie_test.go deleted file mode 100644 index 2b677f40..00000000 --- a/internal/utils/trie/trie_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package trie - -import "testing" - -var nsCPU = Namespace("cpu") - -// Test functions -func TestLoadOrStore(t *testing.T) { - trie := NewTrie() - ptr := trie.LoadOrStore(nsCPU, func() any { - return new(int) - }) - if ptr == nil { - t.Fatal("expected pointer to be created") - } - if ptr != trie.LoadOrStore(nsCPU, func() any { - return new(int) - }) { - t.Fatal("expected same pointer to be returned") - } - got, ok := trie.Get(nsCPU) - if !ok || got != ptr { - t.Fatal("expected same pointer to be returned") - } -} - -func TestStore(t *testing.T) { - trie := NewTrie() - ptr := new(int) - trie.Store(nsCPU, ptr) - got, ok := trie.Get(nsCPU) - if !ok || got != ptr { - t.Fatal("expected same pointer to be returned") - } -} diff --git a/internal/utils/trie/walk.go b/internal/utils/trie/walk.go deleted file mode 100644 index 959f5bc2..00000000 --- a/internal/utils/trie/walk.go +++ /dev/null @@ -1,109 +0,0 @@ -package trie - -import ( - "maps" - "slices" -) - -type ( - YieldFunc = func(part string, value any) bool - YieldKeyFunc = func(key string) bool - Iterator = func(YieldFunc) - KeyIterator = func(YieldKeyFunc) -) - -// WalkAll walks all nodes in the trie, yields full key and series -func (node *Node) Walk(yield YieldFunc) { - node.walkAll(yield) -} - -func (node *Node) walkAll(yield YieldFunc) bool { - if !node.value.IsNil() { - return yield(node.key, node.value.Load()) - } - for _, v := range node.children.Range { - if !v.walkAll(yield) { - return false - } - } - return true -} - -func (node *Node) WalkKeys(yield YieldKeyFunc) { - node.walkKeys(yield) -} - -func (node *Node) walkKeys(yield YieldKeyFunc) bool { - if !node.value.IsNil() { - return !yield(node.key) - } - for _, v := range node.children.Range { - if !v.walkKeys(yield) { - return false - } - } - return true -} - -func (node *Node) Keys() []string { - return slices.Collect(node.WalkKeys) -} - -func (node *Node) Map() map[string]any { - return maps.Collect(node.Walk) -} - -func (tree Root) Query(key *Key) Iterator { - if !key.hasWildcard { - return func(yield YieldFunc) { - if v, ok := tree.Get(key); ok { - yield(key.full, v) - } - } - } - return func(yield YieldFunc) { - tree.walkQuery(key.segments, tree.Node, yield, false) - } -} - -func (tree Root) walkQuery(patternParts []string, node *Node, yield YieldFunc, recursive bool) bool { - if len(patternParts) == 0 { - if !node.value.IsNil() { // end - if !yield(node.key, node.value.Load()) { - return true - } - } else if recursive { - return tree.walkAll(yield) - } - return true - } - pat := patternParts[0] - - switch pat { - case "**": - // ** matches zero or more segments - // Option 1: ** matches zero segment, move to next pattern part - if !tree.walkQuery(patternParts[1:], node, yield, false) { - return false - } - // Option 2: ** matches one or more segments - for _, child := range node.children.Range { - if !tree.walkQuery(patternParts, child, yield, true) { - return false - } - } - case "*": - // * matches any single segment - for _, child := range node.children.Range { - if !tree.walkQuery(patternParts[1:], child, yield, false) { - return false - } - } - default: - // Exact match - if child, ok := node.children.Load(pat); ok { - return tree.walkQuery(patternParts[1:], child, yield, false) - } - } - return true -} diff --git a/internal/utils/trie/walk_test.go b/internal/utils/trie/walk_test.go deleted file mode 100644 index 275524a2..00000000 --- a/internal/utils/trie/walk_test.go +++ /dev/null @@ -1,113 +0,0 @@ -package trie_test - -import ( - "maps" - "slices" - "testing" - - . "github.com/yusing/godoxy/internal/utils/trie" -) - -// Test data for trie tests -var ( - testData = map[string]any{ - "routes.route1": new(int), - "routes.route2": new(int), - "routes.route3": new(int), - "system.cpu_average": new(int), - "system.mem.used": new(int), - "system.mem.percentage_used": new(int), - "system.disks.disk0.used": new(int), - "system.disks.disk0.percentage_used": new(int), - "system.disks.disk1.used": new(int), - "system.disks.disk1.percentage_used": new(int), - } - - testWalkDisksWants = []string{ - "system.disks.disk0.used", - "system.disks.disk0.percentage_used", - "system.disks.disk1.used", - "system.disks.disk1.percentage_used", - } - testWalkDisksUsedWants = []string{ - "system.disks.disk0.used", - "system.disks.disk1.used", - } - testUsedWants = []string{ - "system.mem.used", - "system.disks.disk0.used", - "system.disks.disk1.used", - } -) - -// Helper functions -func keys(m map[string]any) []string { - return slices.Sorted(maps.Keys(m)) -} - -func keysEqual(m map[string]any, want []string) bool { - slices.Sort(want) - return slices.Equal(keys(m), want) -} - -func TestWalkAll(t *testing.T) { - trie := NewTrie() - for key, series := range testData { - trie.Store(NewKey(key), series) - } - - walked := maps.Collect(trie.Walk) - for k, v := range testData { - if _, ok := walked[k]; !ok { - t.Fatalf("expected key %s not found", k) - } - if v != walked[k] { - t.Fatalf("key %s expected %v, got %v", k, v, walked[k]) - } - } -} - -func TestWalk(t *testing.T) { - trie := NewTrie() - for key, series := range testData { - trie.Store(NewKey(key), series) - } - - tests := []struct { - query string - want []string - wantEmpty bool - }{ - {"system.disks.*.used", testWalkDisksUsedWants, false}, - {"system.*.*.used", testWalkDisksUsedWants, false}, - {"*.disks.*.used", testWalkDisksUsedWants, false}, - {"*.*.*.used", testWalkDisksUsedWants, false}, - {"system.disks.**", testWalkDisksWants, false}, // note: original code uses '*' not '**' - {"system.disks", nil, true}, - {"**.used", testUsedWants, false}, - } - - for _, tc := range tests { - t.Run(tc.query, func(t *testing.T) { - got := maps.Collect(trie.Query(NewKey(tc.query))) - if tc.wantEmpty { - if len(got) != 0 { - t.Fatalf("expected empty, got %v", keys(got)) - } - return - } - if !keysEqual(got, tc.want) { - t.Fatalf("expected %v, got %v", tc.want, keys(got)) - } - for _, k := range tc.want { - want, ok := testData[k] - if !ok { - t.Fatalf("expected key %s not found", k) - } - if got[k] != want { - t.Fatalf("key %s expected %v, got %v", k, want, got[k]) - } - } - }) - } -} diff --git a/internal/watcher/health/monitor/monitor.go b/internal/watcher/health/monitor/monitor.go index 71dc34ca..3a5f019f 100644 --- a/internal/watcher/health/monitor/monitor.go +++ b/internal/watcher/health/monitor/monitor.go @@ -16,7 +16,7 @@ import ( "github.com/yusing/godoxy/internal/task" "github.com/yusing/godoxy/internal/types" "github.com/yusing/godoxy/internal/utils/atomic" - "github.com/yusing/godoxy/internal/utils/strutils" + strutils "github.com/yusing/goutils/strings" ) type ( diff --git a/socket-proxy/go.mod b/socket-proxy/go.mod index 1cef6487..67f19606 100644 --- a/socket-proxy/go.mod +++ b/socket-proxy/go.mod @@ -6,15 +6,16 @@ replace github.com/yusing/godoxy/internal/utils => ../internal/utils require ( github.com/gorilla/mux v1.8.1 - github.com/yusing/godoxy/internal/utils v0.0.0-20250922084459-f9affba9fc4e + github.com/yusing/goutils v0.2.1 golang.org/x/net v0.44.0 ) require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rs/zerolog v1.34.0 // indirect - go.uber.org/atomic v1.11.0 // indirect golang.org/x/sys v0.36.0 // indirect golang.org/x/text v0.29.0 // indirect ) diff --git a/socket-proxy/go.sum b/socket-proxy/go.sum index 49928887..e1da2185 100644 --- a/socket-proxy/go.sum +++ b/socket-proxy/go.sum @@ -19,8 +19,8 @@ github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= -go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +github.com/yusing/goutils v0.2.1 h1:KjoCrNO0otthaPCZPfQY+5GKsqs5+J77CxP+TNHYa/Y= +github.com/yusing/goutils v0.2.1/go.mod h1:v6RZsMRdzcts4udSg0vqUIFvaD0OaUMPTwYJZ4XnQYo= golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/socket-proxy/pkg/reverseproxy/reverse_proxy.go b/socket-proxy/pkg/reverseproxy/reverse_proxy.go index ae7bee82..0b90d466 100644 --- a/socket-proxy/pkg/reverseproxy/reverse_proxy.go +++ b/socket-proxy/pkg/reverseproxy/reverse_proxy.go @@ -19,7 +19,7 @@ import ( "strings" "sync" - "github.com/yusing/godoxy/internal/utils" + ioutils "github.com/yusing/goutils/io" "golang.org/x/net/http/httpguts" ) @@ -218,7 +218,7 @@ func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) { rw.WriteHeader(res.StatusCode) - err = utils.CopyCloseWithContext(ctx, rw, res.Body, int(res.ContentLength)) + err = ioutils.CopyCloseWithContext(ctx, rw, res.Body, int(res.ContentLength)) if err != nil { if !errors.Is(err, context.Canceled) { p.getErrorHandler()(rw, req, err)